From 761dd26b232c4eba78b1bbd60d9ee7330fd0d738 Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Sat, 27 Jun 2026 15:15:15 +0200 Subject: [PATCH 1/3] fix: prefix diagnostics misread prefixed names after multi-byte chars `prefix_diagnostic_helper` sliced the rope with char-indexed `get_slice` while term spans hold byte offsets. After any multi-byte char earlier on the line (e.g. an en-dash in a literal) the slice was read too far, turning `rdfs:domain` into `fs:domain` and reporting a phantom "Undefined prefix fs". Slice by bytes via `get_byte_slice` instead. Co-Authored-By: Claude Opus 4.8 --- core/src/systems/prefix_diagnostics.rs | 7 +++++- e2e/tests/prefix_diagnostics.rs | 35 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/core/src/systems/prefix_diagnostics.rs b/core/src/systems/prefix_diagnostics.rs index a44ffba174..04174c71de 100644 --- a/core/src/systems/prefix_diagnostics.rs +++ b/core/src/systems/prefix_diagnostics.rs @@ -93,7 +93,12 @@ pub fn prefix_diagnostic_helper<'a>( if span.is_empty() { continue; } - let raw = match rope.0.get_slice(span.start..span.end) { + // `span` holds *byte* offsets (matching `offset_to_position`), so we + // must slice by bytes. Using char-indexed `get_slice` here would read + // from the wrong place after any multi-byte char earlier in the line + // (e.g. an en-dash in a literal), mis-reading a prefixed name such as + // `rdfs:domain` as `fs:domain` and reporting a phantom undefined prefix. + let raw = match rope.0.get_byte_slice(span.start..span.end) { Some(s) => s.to_string(), None => continue, }; diff --git a/e2e/tests/prefix_diagnostics.rs b/e2e/tests/prefix_diagnostics.rs index 24adb52852..6bba92fb06 100644 --- a/e2e/tests/prefix_diagnostics.rs +++ b/e2e/tests/prefix_diagnostics.rs @@ -255,6 +255,41 @@ fn declared_and_used_prefix_produces_no_prefix_diagnostic() { ); } +// ─── Multi-byte characters must not shift prefix spans ─────────────────────── + +/// Regression: a multi-byte char (en-dash `–`, U+2013, 3 bytes / 1 char) inside a +/// literal must not desync the byte-based term spans from the rope slice. Before +/// the fix, the slice for `rdfs:domain` was read 2 chars too far (`fs:domain`), +/// producing a phantom "Undefined prefix fs". +#[test_log::test] +fn multibyte_literal_does_not_produce_phantom_undefined_prefix() { + let mut h = LspHarness::new(); + let src = "@prefix rdfs: .\n\ + @prefix rdfc: .\n\ + \n\ + rdfc:consistsOf rdfs:comment \"–\";\n\ + \u{0020} rdfs:domain rdfc:Pipeline."; + let file = h.open_file("file:///multibyte_prefix.ttl", "turtle", src); + + let diags = h.run_diagnostics(); + + let prefix_errors: Vec<_> = diags + .iter() + .filter(|(url, d)| { + url.as_str() == file.url && d.severity == Some(DiagnosticSeverity::ERROR) + }) + .collect(); + + assert!( + prefix_errors.is_empty(), + "All prefixes are declared; expected no undefined-prefix errors, got: {:?}", + prefix_errors + .iter() + .map(|(_, d)| &d.message) + .collect::>() + ); +} + // ─── Multiple undefined prefixes ───────────────────────────────────────────── #[test_log::test] From 0d340d1e404f29165848d125f712740011947366 Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Sat, 27 Jun 2026 15:15:25 +0200 Subject: [PATCH 2/3] test: pin LSP position encoding to UTF-16 (currently failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds e2e tests asserting that Position.character is a UTF-16 code-unit count, per the LSP default. These FAIL against the current byte-based `offset_to_position` and are the red half of the upcoming switch from ropey to a hand-rolled LineIndex: * diagnostic_column_is_utf16_after_two_byte_char (BMP: byte ≠ utf16) * diagnostic_column_is_utf16_after_surrogate_pair (astral: pins utf16 apart from both byte count and char count) Co-Authored-By: Claude Opus 4.8 --- e2e/tests/position_encoding.rs | 105 +++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 e2e/tests/position_encoding.rs diff --git a/e2e/tests/position_encoding.rs b/e2e/tests/position_encoding.rs new file mode 100644 index 0000000000..a2e6d8250a --- /dev/null +++ b/e2e/tests/position_encoding.rs @@ -0,0 +1,105 @@ +//! E2E tests pinning the LSP **position encoding** to UTF-16. +//! +//! The LSP spec defaults `positionEncoding` to UTF-16: a `Position.character` is +//! a count of UTF-16 code units from the start of the line, *not* a byte offset +//! and *not* a Unicode scalar (char) count. +//! +//! Historically SWLS emitted **byte** columns (`offset_to_position` computed +//! `column = byte_offset - line_start_byte`). That is correct only for ASCII; on +//! any line containing a multi-byte character the reported column is wrong, which +//! manifests as misplaced diagnostics, hovers, rename ranges, etc. +//! +//! These tests observe the column through diagnostic ranges (the easiest public +//! observation point) and assert it equals the UTF-16 code-unit count. + +use swls_core::lsp_types::DiagnosticSeverity; +use swls_e2e_tests::LspHarness; + +/// Find the undefined-prefix ERROR diagnostic for `prefix` and return its start +/// position `(line, character)`. +fn undefined_prefix_start( + diags: &[(swls_core::lsp_types::Url, swls_core::lsp_types::Diagnostic)], + file_url: &str, + prefix: &str, +) -> (u32, u32) { + let needle = format!("\"{}\"", prefix); + let (_, diag) = diags + .iter() + .find(|(url, d)| { + url.as_str() == file_url + && d.severity == Some(DiagnosticSeverity::ERROR) + && d.message.contains(&needle) + }) + .unwrap_or_else(|| { + panic!( + "expected an undefined-prefix ERROR for {:?}, got: {:?}", + prefix, + diags.iter().map(|(_, d)| &d.message).collect::>() + ) + }); + (diag.range.start.line, diag.range.start.character) +} + +// ─── BMP multi-byte char: byte column ≠ UTF-16 column ───────────────────────── + +/// `é` (U+00E9) is 2 bytes in UTF-8 but a single UTF-16 code unit. A token after +/// it on the same line must be reported at its UTF-16 column, not its byte column. +#[test_log::test] +fn diagnostic_column_is_utf16_after_two_byte_char() { + let mut h = LspHarness::new(); + // line 0: prefix decl line 1: `é` literal then an UNDECLARED prefix `bad` + let line1 = "ex:s ex:p \"é\" , bad:obj ."; + let src = format!("@prefix ex: .\n{line1}"); + let file = h.open_file("file:///utf16_bmp.ttl", "turtle", &src); + + let diags = h.run_diagnostics(); + let (line, character) = undefined_prefix_start(&diags, &file.url, "bad"); + + let byte_off = line1.find("bad").unwrap(); + let expected_utf16 = line1[..byte_off].encode_utf16().count() as u32; + let byte_col = byte_off as u32; // what the old byte-based code returned + + assert_eq!(line, 1, "diagnostic should be on line 1"); + assert_ne!( + expected_utf16, byte_col, + "test is only meaningful if the byte column differs from the UTF-16 column" + ); + assert_eq!( + character, expected_utf16, + "Position.character must be the UTF-16 column ({expected_utf16}), not the byte column ({byte_col})" + ); +} + +// ─── Astral char (surrogate pair): UTF-16 ≠ char count ≠ byte count ─────────── + +/// `😀` (U+1F600) is 4 bytes in UTF-8, **2** UTF-16 code units, and **1** Unicode +/// scalar. A token after it pins the encoding to UTF-16 specifically: the expected +/// column differs from the byte column (regression we're fixing) *and* from a +/// naive `chars().count()` implementation (regression we must not introduce). +#[test_log::test] +fn diagnostic_column_is_utf16_after_surrogate_pair() { + let mut h = LspHarness::new(); + let line1 = "ex:s ex:p \"😀\" , bad:obj ."; + let src = format!("@prefix ex: .\n{line1}"); + let file = h.open_file("file:///utf16_astral.ttl", "turtle", &src); + + let diags = h.run_diagnostics(); + let (line, character) = undefined_prefix_start(&diags, &file.url, "bad"); + + let byte_off = line1.find("bad").unwrap(); + let expected_utf16 = line1[..byte_off].encode_utf16().count() as u32; + let char_count = line1[..byte_off].chars().count() as u32; + let byte_col = byte_off as u32; + + // All three must be distinct so the assertion truly pins UTF-16. + assert_eq!(line, 1, "diagnostic should be on line 1"); + assert!( + expected_utf16 != byte_col && expected_utf16 != char_count, + "expected UTF-16 ({expected_utf16}) to differ from byte ({byte_col}) and char ({char_count}) counts" + ); + assert_eq!( + character, expected_utf16, + "Position.character must be the UTF-16 column ({expected_utf16}); \ + got {character} (byte col = {byte_col}, char count = {char_count})" + ); +} From 2f7269741f5773fc5a9bd548496b7c4c882d9cb4 Mon Sep 17 00:00:00 2001 From: ajuvercr Date: Sat, 27 Jun 2026 15:28:18 +0200 Subject: [PATCH 3/3] refactor: replace ropey with a hand-rolled LineIndex (UTF-16 positions) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We only ever used ropey for index conversion, never for what it's for: the server uses full-document sync and rebuilds the buffer from a String on every change (no incremental edits, no snapshots). Worse, ropey hid the one decision that actually matters for correctness — which encoding a column is measured in — and we got it wrong, emitting byte columns where the LSP default is UTF-16. Replace `RopeC(ropey::Rope)` with `RopeC(LineIndex)`: a String plus a line-start table, where every conversion takes an explicit `PositionEncoding`. `util::{offset_to_position, position_to_offset, ...}` now route through it with a single `ENCODING` constant (UTF-16). This makes the red `position_encoding` tests pass and also fixes the byte/char mixups found along the way: * rename: sliced by char index + mixed byte spans with char counts * prefix decl range: built char offsets, fed them to a byte-based converter * inlay hints: `get_char` (char-indexed) called with a byte offset * semantic tokens: char/byte length mixups; now emits UTF-16 columns *and* lengths via a single linear pass (no O(line^2) on minified single-line JSON-LD) ropey is dropped from every crate. LineIndex carries unit tests for line counting, CRLF, trailing newline, BMP/astral columns and round-trips. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 25 --- Cargo.toml | 1 - core/Cargo.toml | 1 - core/src/backend.rs | 6 +- core/src/components/document.rs | 7 +- core/src/feature/rename.rs | 17 +- core/src/feature/semantic.rs | 66 +++++-- core/src/lang.rs | 10 +- core/src/lib.rs | 2 + core/src/prelude.rs | 4 +- core/src/systems/lov/extractor.rs | 2 +- core/src/systems/lov/fetch.rs | 2 +- core/src/systems/prefix_diagnostics.rs | 31 ++- core/src/text.rs | 263 +++++++++++++++++++++++++ core/src/util/mod.rs | 48 ++--- e2e/Cargo.toml | 1 - e2e/src/lib.rs | 4 +- lang-jsonld/Cargo.toml | 1 - lang-jsonld/src/ecs/completion.rs | 10 +- lang-jsonld/src/ecs/mod.rs | 2 +- lang-jsonld/src/lib.rs | 6 +- lang-n3/Cargo.toml | 1 - lang-sparql/Cargo.toml | 1 - lang-trig/Cargo.toml | 1 - lang-turtle/Cargo.toml | 1 - lang-turtle/src/config.rs | 2 +- lang-turtle/src/ecs/completion.rs | 12 +- lang-turtle/src/ecs/mod.rs | 4 +- lang-turtle/src/lang/formatter.rs | 30 +-- swls/Cargo.toml | 1 - test-utils/Cargo.toml | 1 - test-utils/src/lib.rs | 3 +- 32 files changed, 411 insertions(+), 155 deletions(-) create mode 100644 core/src/text.rs diff --git a/Cargo.lock b/Cargo.lock index 0537f332f1..0f081701be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2764,16 +2764,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "ropey" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5" -dependencies = [ - "smallvec", - "str_indices", -] - [[package]] name = "rowan" version = "0.16.1" @@ -3486,12 +3476,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "str_indices" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" - [[package]] name = "strsim" version = "0.11.1" @@ -3514,7 +3498,6 @@ dependencies = [ "futures", "glob", "reqwest", - "ropey", "serde", "serde_json", "sophia_api", @@ -3565,7 +3548,6 @@ dependencies = [ "oxrdf", "prefixmap", "rdf-parsers", - "ropey", "rowan", "rudof_rdf", "serde", @@ -3586,7 +3568,6 @@ version = "0.1.0" dependencies = [ "bevy_ecs", "futures", - "ropey", "swls-core", "swls-lang-jsonld", "swls-lang-sparql", @@ -3607,7 +3588,6 @@ dependencies = [ "futures", "oxigraph", "rdf-parsers", - "ropey", "rowan", "serde_json", "swls-core", @@ -3627,7 +3607,6 @@ dependencies = [ "bevy_ecs", "futures", "rdf-parsers", - "ropey", "rowan", "sophia_iri", "swls-core", @@ -3658,7 +3637,6 @@ dependencies = [ "futures", "lazy_static", "rdf-parsers", - "ropey", "rowan", "serde_json", "sophia_api", @@ -3680,7 +3658,6 @@ dependencies = [ "bevy_ecs", "futures", "rdf-parsers", - "ropey", "rowan", "sophia_iri", "swls-core", @@ -3700,7 +3677,6 @@ dependencies = [ "bevy_ecs", "futures", "rdf-parsers", - "ropey", "rowan", "serde", "serde_json", @@ -3732,7 +3708,6 @@ dependencies = [ "chumsky", "futures", "glob", - "ropey", "serde_json", "sophia_api", "swls-core", diff --git a/Cargo.toml b/Cargo.toml index 2cb4cc01e8..af12c9d85a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ futures = { version = "0.3.31", features = ["alloc", "executor"], default-featur lazy_static = "1.5" logos = "0.15.1" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } -ropey = "1.6.1" # rdf-parsers = "0.1.8" rdf-parsers = { version = "0.1.15" } serde = { version = "1.0", features = ["derive"] } diff --git a/core/Cargo.toml b/core/Cargo.toml index f700d6e5c8..be6a613faf 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -24,7 +24,6 @@ bevy_ecs.workspace = true chumsky.workspace = true derive_more.workspace = true futures.workspace = true -ropey.workspace = true serde.workspace = true serde_json.workspace = true sophia_api.workspace = true diff --git a/core/src/backend.rs b/core/src/backend.rs index 2371ab7a89..7c70cfc32a 100644 --- a/core/src/backend.rs +++ b/core/src/backend.rs @@ -12,7 +12,7 @@ use futures::lock::Mutex; use goto_type::GotoTypeRequest; use references::ReferencesRequest; use request::{GotoTypeDefinitionParams, GotoTypeDefinitionResponse}; -use ropey::Rope; +use crate::text::LineIndex; use tracing::{debug, error, info, instrument}; use crate::{ @@ -532,7 +532,7 @@ impl Backend { ( Source(item.text.clone()), Label(item.uri.clone()), - RopeC(Rope::from_str(&item.text)), + RopeC(LineIndex::new(&item.text)), Wrapped(item), DocumentLinks(Vec::new()), Open, @@ -576,7 +576,7 @@ impl Backend { }; self.run(move |world| { - let rope_c = RopeC(Rope::from_str(&change.text)); + let rope_c = RopeC(LineIndex::new(&change.text)); world .entity_mut(entity) .insert((Source(change.text), rope_c)); diff --git a/core/src/components/document.rs b/core/src/components/document.rs index 982490e0b5..9546882513 100644 --- a/core/src/components/document.rs +++ b/core/src/components/document.rs @@ -29,9 +29,10 @@ pub struct Errors(pub Vec); #[derive(Component, AsRef, Deref, AsMut, DerefMut, Debug)] pub struct Source(pub String); -/// [`Component`] containing the current source code as [`ropey::Rope`] -#[derive(Component, AsRef, Deref, AsMut, DerefMut, Debug)] -pub struct RopeC(pub ropey::Rope); +/// [`Component`] containing the current source code as a [`LineIndex`], used for +/// LSP byte ↔ position conversions. +#[derive(Component, AsRef, Deref, AsMut, DerefMut, Debug, Default)] +pub struct RopeC(pub LineIndex); /// [`Component`] that allows for language specific implementation for certain things, reducing /// code duplication. diff --git a/core/src/feature/rename.rs b/core/src/feature/rename.rs index 42f387b054..dd156ebb8e 100644 --- a/core/src/feature/rename.rs +++ b/core/src/feature/rename.rs @@ -67,7 +67,10 @@ pub fn prepare_rename( TripleTarget::Graph => continue, }; - let raw: String = rope.0.slice(span.start..span.end).to_string(); + let Some(raw) = rope.0.byte_slice(span.start..span.end) else { + continue; + }; + let raw = raw.to_string(); let inner = lang.0.rename_placeholder(&raw); // Guard: inner must be non-empty @@ -75,13 +78,13 @@ pub fn prepare_rename( continue; } - // Compute how many chars are stripped from the front. + // `inner` is a sub-slice of `raw`; work entirely in *byte* offsets + // so the math stays consistent with `span` (also bytes) and with + // `range_to_range`. Mixing in char counts here previously produced + // wrong ranges after multi-byte characters. let prefix_offset = inner.as_ptr() as usize - raw.as_ptr() as usize; - let prefix_chars = raw[..prefix_offset].chars().count(); - let inner_char_len = inner.chars().count(); - - let inner_start = span.start + prefix_chars; - let inner_end = inner_start + inner_char_len; + let inner_start = span.start + prefix_offset; + let inner_end = inner_start + inner.len(); if inner_start >= inner_end || inner_end > span.end { continue; diff --git a/core/src/feature/semantic.rs b/core/src/feature/semantic.rs index e7e5c327e5..b7d65530e1 100644 --- a/core/src/feature/semantic.rs +++ b/core/src/feature/semantic.rs @@ -61,7 +61,8 @@ pub fn semantic_tokens_system( tracing::debug!("semantic_tokens_system called"); for (rope, types, mut req) in &mut query { let rope = &rope.0; - let mut ts: Vec> = Vec::with_capacity(rope.len_chars()); + // `ts` is indexed by *byte* offset; spans (`r`) are byte ranges. + let mut ts: Vec> = Vec::with_capacity(rope.len_bytes()); ts.resize(rope.len_bytes(), None); types.iter().for_each(|Spanned(ty, r)| { r.clone().for_each(|j| { @@ -69,10 +70,9 @@ pub fn semantic_tokens_system( ts[j] = Some(ty.clone()) } else { tracing::error!( - "Semantic tokens type {} (index={}) falls outside of rope size (chars: {} bytes: {})", + "Semantic tokens type {} (index={}) falls outside of document size ({} bytes)", ty.as_str(), j, - rope.len_chars(), rope.len_bytes() ); } @@ -100,35 +100,63 @@ pub fn semantic_tokens_system( if let Some(t) = last { out_tokens.push(TokenHelper { start, - length: rope.len_chars() - start, + length: rope.len_bytes() - start, // byte length, like the branch above ty: res.get(&t).cloned().unwrap_or(0), }); } - let mut pre_line = 0; - let mut pre_start = 0; + // Emit LSP semantic tokens. `start`/`length` on each `TokenHelper` are byte + // offsets/lengths; LSP wants UTF-16 columns and lengths. We walk the source + // once with a monotonically advancing cursor (tokens are sorted, non- + // overlapping), so this is O(n) overall — crucially avoiding an O(line²) + // blow-up on minified single-line documents that a per-token byte→UTF-16 + // conversion would incur. + let text = rope.as_str(); + let mut cur_byte = 0usize; + let mut cur_line = 0u32; + let mut cur_col = 0u32; // UTF-16 column within the current line + let mut pre_line = 0u32; + let mut pre_col = 0u32; req.0 = out_tokens .into_iter() - .flat_map(|token| { - let line = rope.try_byte_to_line(token.start as usize).ok()? as u32; - let first = rope.try_line_to_char(line as usize).ok()? as u32; - let start = rope.try_byte_to_char(token.start as usize).ok()? as u32 - first; - let delta_line = line - pre_line; + .map(|token| { + let advance = |from: usize, to: usize, line: &mut u32, col: &mut u32| -> u32 { + let mut utf16 = 0u32; + for ch in text.get(from..to).unwrap_or("").chars() { + let w = ch.len_utf16() as u32; + utf16 += w; + if ch == '\n' { + *line += 1; + *col = 0; + } else { + *col += w; + } + } + utf16 + }; + + advance(cur_byte, token.start, &mut cur_line, &mut cur_col); + let start_line = cur_line; + let start_col = cur_col; + let length = advance(token.start, token.start + token.length, &mut cur_line, &mut cur_col); + cur_byte = token.start + token.length; + + let delta_line = start_line - pre_line; let delta_start = if delta_line == 0 { - start - pre_start + start_col - pre_col } else { - start + start_col }; - let ret = Some(SemanticToken { + pre_line = start_line; + pre_col = start_col; + + SemanticToken { delta_line, delta_start, - length: token.length as u32, + length, token_type: token.ty as u32, token_modifiers_bitset: 0, - }); - pre_line = line; - pre_start = start; - ret + } }) .collect(); } diff --git a/core/src/lang.rs b/core/src/lang.rs index 015006e778..0b9b069bc7 100644 --- a/core/src/lang.rs +++ b/core/src/lang.rs @@ -1,6 +1,8 @@ use std::{borrow::Cow, ops::Range}; -use crate::{lsp_types::SemanticTokenType, prelude::TripleTarget, util::offset_to_position}; +use crate::{ + lsp_types::SemanticTokenType, prelude::TripleTarget, text::LineIndex, util::offset_to_position, +}; pub fn head() -> crate::lsp_types::Range { let start = crate::lsp_types::Position { @@ -89,7 +91,7 @@ pub trait LangHelper: std::fmt::Debug { fn prefix_edits( &self, _source: &str, - _rope: &ropey::Rope, + _rope: &LineIndex, name: &str, namespace: &str, format: crate::components::PrefixFormat, @@ -217,7 +219,7 @@ pub trait LangHelper: std::fmt::Debug { fn inlay_types_hint( &self, subject: &Range, - rope: &ropey::Rope, + rope: &LineIndex, last_type: Option<&Range>, types: Vec>, ) -> Option { @@ -229,7 +231,7 @@ pub trait LangHelper: std::fmt::Debug { return None; } } else { - let offset = if rope.get_char(subject.start) == Some('[') { + let offset = if rope.char_at_byte(subject.start) == Some('[') { subject.start + 1 } else { subject.end diff --git a/core/src/lib.rs b/core/src/lib.rs index cb385f853c..5718128486 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -97,6 +97,8 @@ pub mod backend; /// Handle platform specific implementations for fetching and spawning tasks. pub mod client; pub mod store; +/// Minimal text indexing for LSP position math (replaces ropey). +pub mod text; /// Common utils /// /// Includes range transformations between [`std::ops::Range`] and [`swls_core::lsp_types::Range`]. diff --git a/core/src/prelude.rs b/core/src/prelude.rs index 7394332f18..8e9a926e11 100644 --- a/core/src/prelude.rs +++ b/core/src/prelude.rs @@ -16,7 +16,9 @@ pub use crate::{ *, }, lang::{Lang, LangHelper}, - setup_schedule_labels, systems, + setup_schedule_labels, + text::{LineIndex, PositionEncoding}, + systems, systems::prefix::{Prefix, Prefixes}, systems::spawn_or_insert, util::{ diff --git a/core/src/systems/lov/extractor.rs b/core/src/systems/lov/extractor.rs index 3eedff9f7f..33d024c7a7 100644 --- a/core/src/systems/lov/extractor.rs +++ b/core/src/systems/lov/extractor.rs @@ -37,7 +37,7 @@ pub fn init_ontology_extractor(mut commands: Commands, fs: Res) { url.clone(), ( Source(local.content.to_string()), - RopeC(ropey::Rope::from_str(&local.content)), + RopeC(LineIndex::new(&*local.content)), Label(url), Wrapped(item), Types(HashMap::new()), diff --git a/core/src/systems/lov/fetch.rs b/core/src/systems/lov/fetch.rs index 701efd0f77..88c6a3fcd1 100644 --- a/core/src/systems/lov/fetch.rs +++ b/core/src/systems/lov/fetch.rs @@ -187,7 +187,7 @@ pub fn spawn_document( let spawn = spawn_or_insert( url.clone(), ( - RopeC(ropey::Rope::from_str(&content)), + RopeC(LineIndex::new(&content)), Source(content.clone()), Label(url.clone()), Wrapped(item), diff --git a/core/src/systems/prefix_diagnostics.rs b/core/src/systems/prefix_diagnostics.rs index 04174c71de..191e14e247 100644 --- a/core/src/systems/prefix_diagnostics.rs +++ b/core/src/systems/prefix_diagnostics.rs @@ -93,12 +93,12 @@ pub fn prefix_diagnostic_helper<'a>( if span.is_empty() { continue; } - // `span` holds *byte* offsets (matching `offset_to_position`), so we - // must slice by bytes. Using char-indexed `get_slice` here would read - // from the wrong place after any multi-byte char earlier in the line - // (e.g. an en-dash in a literal), mis-reading a prefixed name such as - // `rdfs:domain` as `fs:domain` and reporting a phantom undefined prefix. - let raw = match rope.0.get_byte_slice(span.start..span.end) { + // `span` holds *byte* offsets, so slice by bytes. Slicing by chars + // here would read from the wrong place after any multi-byte char + // earlier in the line (e.g. an en-dash in a literal), mis-reading a + // prefixed name such as `rdfs:domain` as `fs:domain` and reporting a + // phantom undefined prefix. + let raw = match rope.0.byte_slice(span.start..span.end) { Some(s) => s.to_string(), None => continue, }; @@ -236,7 +236,7 @@ pub fn prefix_diagnostic_helper<'a>( /// - Turtle/TriG: `@prefix foaf: <…>.` /// - SPARQL: `PREFIX foaf: <…>` /// - JSON-LD: `"foaf": "http://…"` (inside `@context`) -fn find_prefix_declaration_range(rope: &ropey::Rope, prefix_name: &str) -> (Position, Position) { +fn find_prefix_declaration_range(rope: &LineIndex, prefix_name: &str) -> (Position, Position) { // Patterns that identify a declaration for this prefix, paired with the byte // offset (within the match) at which the highlighted key/name begins. let turtle_needle = format!("@prefix {}:", prefix_name); @@ -254,18 +254,17 @@ fn find_prefix_declaration_range(rope: &ropey::Rope, prefix_name: &str) -> (Posi (&jsonld_needle, 0, prefix_name.len() + 2), ]; - let candidate = rope.lines().enumerate().find_map(|(line_idx, line_slice)| { - let line = line_slice.to_string(); - let line_start = rope.line_to_char(line_idx); + let candidate = (0..rope.len_lines()).find_map(|line_idx| { + let line = rope.line_str(line_idx)?; + let line_start = rope.line_start(line_idx)?; for (needle, key_off, key_len) in patterns.iter() { if let Some(idx) = line.find(needle.as_str()) { - let key_byte = idx + key_off; - // Convert byte indices within the line to char offsets. - let key_start_char = line[..key_byte].chars().count(); - let key_end_char = line[..key_byte + key_len].chars().count(); - let start = offset_to_position(line_start + key_start_char, rope)?; - let end = offset_to_position(line_start + key_end_char, rope)?; + // Byte offsets within the whole document; `offset_to_position` + // converts them (and the encoding) for us. + let key_byte = line_start + idx + key_off; + let start = offset_to_position(key_byte, rope)?; + let end = offset_to_position(key_byte + key_len, rope)?; return Some((start, end)); } } diff --git a/core/src/text.rs b/core/src/text.rs new file mode 100644 index 0000000000..0905d1b6e3 --- /dev/null +++ b/core/src/text.rs @@ -0,0 +1,263 @@ +//! Minimal text indexing for LSP position math. +//! +//! Replaces our previous use of [`ropey`]. We never actually used ropey for what +//! ropey is *for* — incremental edits and cheap snapshots — because the server +//! uses full-document sync and rebuilds the buffer from a `String` on every +//! change. The only thing we used was index conversion, and ropey hid the one +//! decision that actually matters for correctness: **which encoding a column is +//! measured in**. +//! +//! [`LineIndex`] makes that explicit. It owns the source text plus a table of +//! line-start byte offsets, and every conversion takes a [`PositionEncoding`] so +//! the byte/UTF-16 choice is visible at the call site instead of buried in a +//! dependency. +//! +//! ## Encodings +//! +//! An LSP `Position.character` is a count *from the start of its line*. The unit +//! depends on the negotiated [`PositionEncoding`]: +//! +//! * [`PositionEncoding::Utf16`] — UTF-16 code units. This is the LSP **default** +//! and what clients assume unless the server negotiates otherwise. +//! * [`PositionEncoding::Utf8`] — bytes. Cheapest, and what this codebase used to +//! emit unconditionally (correct only for ASCII). +//! +//! Internally everything is a UTF-8 **byte** offset; encodings only ever affect +//! the in-line column number. + +use crate::lsp_types::Position; +use std::ops::Range; + +/// The unit in which an LSP `Position.character` column is measured. +/// +/// Defaults to [`Utf16`](PositionEncoding::Utf16), matching the LSP spec default. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum PositionEncoding { + /// UTF-16 code units (LSP default). + #[default] + Utf16, + /// Raw UTF-8 bytes. + Utf8, +} + +/// Source text plus a line-start index, supporting byte ↔ [`Position`] conversion. +/// +/// Owns the text so it can serve slices and per-line lookups without the caller +/// also threading a `&str` through. This is the same memory profile as the old +/// `RopeC(Rope)` (which already duplicated the `Source` string). +#[derive(Clone, Debug, Default)] +pub struct LineIndex { + text: String, + /// Byte offset of the start of each line. Always begins with `0`; its length + /// is the number of lines (a trailing `\n` yields a final empty line). + line_starts: Vec, +} + +impl LineIndex { + /// Build an index over `text`, scanning once for line breaks (`O(n)`). + pub fn new(text: impl Into) -> Self { + let text = text.into(); + let mut line_starts = vec![0]; + line_starts.extend( + text.bytes() + .enumerate() + .filter(|&(_, b)| b == b'\n') + .map(|(i, _)| i + 1), + ); + Self { text, line_starts } + } + + /// The full source text. + #[inline] + pub fn as_str(&self) -> &str { + &self.text + } + + /// Total length in bytes. + #[inline] + pub fn len_bytes(&self) -> usize { + self.text.len() + } + + /// Number of lines (a trailing newline counts as starting one more line). + #[inline] + pub fn len_lines(&self) -> usize { + self.line_starts.len() + } + + /// Byte offset at which `line` starts, or `None` if out of range. A + /// one-past-the-end `line` is *not* accepted (use [`len_bytes`] for that). + #[inline] + pub fn line_start(&self, line: usize) -> Option { + self.line_starts.get(line).copied() + } + + /// Byte range `[start, end)` of `line`, including its trailing newline (if any). + pub fn line_byte_range(&self, line: usize) -> Option> { + let start = self.line_start(line)?; + let end = self.line_starts.get(line + 1).copied().unwrap_or(self.text.len()); + Some(start..end) + } + + /// Text of `line`, including its trailing newline (if any). + pub fn line_str(&self, line: usize) -> Option<&str> { + let r = self.line_byte_range(line)?; + self.text.get(r) + } + + /// Sub-slice by **byte** range. Returns `None` for out-of-bounds ranges or + /// ranges that do not fall on `char` boundaries. + #[inline] + pub fn byte_slice(&self, range: Range) -> Option<&str> { + self.text.get(range) + } + + /// The `char` starting at byte offset `byte`, if `byte` is a char boundary + /// within the text. + pub fn char_at_byte(&self, byte: usize) -> Option { + self.text.get(byte..)?.chars().next() + } + + /// Convert a byte offset to an LSP [`Position`] in the given encoding. + /// + /// Accepts `byte == len_bytes()` (end of document). Returns `None` if `byte` + /// is out of range or not on a char boundary. + pub fn byte_to_position(&self, byte: usize, encoding: PositionEncoding) -> Option { + if byte > self.text.len() { + return None; + } + let line = match self.line_starts.binary_search(&byte) { + Ok(line) => line, + Err(next) => next - 1, // next >= 1 because line_starts[0] == 0 <= byte + }; + let line_start = self.line_starts[line]; + let segment = self.text.get(line_start..byte)?; // None if not a char boundary + let character = match encoding { + PositionEncoding::Utf16 => segment.encode_utf16().count(), + PositionEncoding::Utf8 => segment.len(), + }; + Some(Position::new(line as u32, character as u32)) + } + + /// Convert an LSP [`Position`] (interpreted in `encoding`) to a byte offset. + /// + /// A `character` past the end of its line is clamped to the line's end (which + /// keeps the lenient behaviour editors rely on when the cursor sits after the + /// last character). Returns `None` only if the line itself is out of range. + pub fn position_to_byte(&self, position: Position, encoding: PositionEncoding) -> Option { + let line = position.line as usize; + let line_range = self.line_byte_range(line)?; + let line_text = self.text.get(line_range.clone())?; + let target = position.character as usize; + + let col_bytes = match encoding { + PositionEncoding::Utf8 => target.min(line_text.len()), + PositionEncoding::Utf16 => { + let mut utf16 = 0usize; + let mut found = None; + for (byte_off, ch) in line_text.char_indices() { + if utf16 >= target { + found = Some(byte_off); + break; + } + utf16 += ch.len_utf16(); + } + found.unwrap_or(line_text.len()) + } + }; + Some(line_range.start + col_bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const U16: PositionEncoding = PositionEncoding::Utf16; + const U8: PositionEncoding = PositionEncoding::Utf8; + + fn pos(l: u32, c: u32) -> Position { + Position::new(l, c) + } + + #[test] + fn line_counting_matches_ropey_conventions() { + assert_eq!(LineIndex::new("").len_lines(), 1); + assert_eq!(LineIndex::new("abc").len_lines(), 1); + assert_eq!(LineIndex::new("abc\n").len_lines(), 2); // trailing newline → empty last line + assert_eq!(LineIndex::new("a\nb\nc").len_lines(), 3); + } + + #[test] + fn ascii_byte_to_position() { + let idx = LineIndex::new("ab\ncde"); + assert_eq!(idx.byte_to_position(0, U16), Some(pos(0, 0))); + assert_eq!(idx.byte_to_position(2, U16), Some(pos(0, 2))); // before '\n' + assert_eq!(idx.byte_to_position(3, U16), Some(pos(1, 0))); // start of line 1 + assert_eq!(idx.byte_to_position(6, U16), Some(pos(1, 3))); // end of document + assert_eq!(idx.byte_to_position(7, U16), None); // past end + } + + #[test] + fn two_byte_char_column_differs_by_encoding() { + // 'é' is 2 bytes, 1 UTF-16 unit. "xé" then 'y' at byte 3. + let idx = LineIndex::new("xéy"); + let y_byte = "xé".len(); // 3 + assert_eq!(idx.byte_to_position(y_byte, U16), Some(pos(0, 2))); // x, é + assert_eq!(idx.byte_to_position(y_byte, U8), Some(pos(0, 3))); // x, é(2 bytes) + } + + #[test] + fn astral_char_is_two_utf16_units_one_char() { + // '😀' is 4 bytes, 2 UTF-16 units, 1 scalar. 'z' follows. + let idx = LineIndex::new("😀z"); + let z_byte = "😀".len(); // 4 + assert_eq!(idx.byte_to_position(z_byte, U16), Some(pos(0, 2))); // surrogate pair + assert_eq!(idx.byte_to_position(z_byte, U8), Some(pos(0, 4))); + } + + #[test] + fn position_to_byte_roundtrips_utf16() { + for text in ["", "abc", "ab\ncde", "xéy\nfoo", "😀z\n€uro", "a\r\nb"] { + let idx = LineIndex::new(text); + // Every char boundary should round-trip byte → position → byte. + for (byte, _) in text.char_indices().chain(std::iter::once((text.len(), ' '))) { + let p = idx.byte_to_position(byte, U16).unwrap(); + assert_eq!( + idx.position_to_byte(p, U16), + Some(byte), + "roundtrip failed for {text:?} at byte {byte} (pos {p:?})" + ); + } + } + } + + #[test] + fn position_to_byte_clamps_past_end_of_line() { + let idx = LineIndex::new("ab\ncd"); + // character way past the line length clamps to end of that line (incl. '\n'). + assert_eq!(idx.position_to_byte(pos(0, 99), U16), Some(3)); // "ab\n" -> byte 3 + assert_eq!(idx.position_to_byte(pos(1, 99), U16), Some(5)); // end of doc + assert_eq!(idx.position_to_byte(pos(9, 0), U16), None); // line out of range + } + + #[test] + fn crlf_line_starts() { + let idx = LineIndex::new("a\r\nb"); + assert_eq!(idx.len_lines(), 2); + // '\r' is part of line 0; line 1 starts after '\n'. + assert_eq!(idx.byte_to_position(3, U16), Some(pos(1, 0))); + assert_eq!(idx.line_str(0), Some("a\r\n")); + assert_eq!(idx.line_str(1), Some("b")); + } + + #[test] + fn byte_slice_and_char_at_byte() { + let idx = LineIndex::new("xéy"); + assert_eq!(idx.byte_slice(0..1), Some("x")); + assert_eq!(idx.byte_slice(0..2), None); // splits 'é' + assert_eq!(idx.char_at_byte(1), Some('é')); + assert_eq!(idx.char_at_byte(2), None); // mid-'é' + assert_eq!(idx.char_at_byte(3), Some('y')); + } +} diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index 4365bfbf85..1940f2b4db 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -1,10 +1,16 @@ -use ropey::Rope; - use crate::{ lsp_types::{Location, Position, Range}, + text::{LineIndex, PositionEncoding}, Label, }; +/// Encoding used for all LSP positions emitted/consumed by these helpers. +/// +/// The LSP default is UTF-16; until we negotiate `positionEncoding` at +/// `initialize`, everything goes through this single constant so the choice is +/// explicit and changeable in one place. +const ENCODING: PositionEncoding = PositionEncoding::Utf16; + pub mod fs; /// Commonly used RDF prefixes pub mod ns; @@ -25,7 +31,7 @@ pub use rdf_parsers::{spanned, Spanned}; // crate::lsp_types::Url::parse(&url).ok() // } -pub fn range_to_range(range: &std::ops::Range, rope: &Rope) -> Option { +pub fn range_to_range(range: &std::ops::Range, rope: &LineIndex) -> Option { let start = offset_to_position(range.start, rope)?; let end = offset_to_position(range.end, rope)?; Range::new(start, end).into() @@ -33,38 +39,20 @@ pub fn range_to_range(range: &std::ops::Range, rope: &Rope) -> Option Option> { - if range.start.line as usize >= rope.len_lines() || range.end.line as usize >= rope.len_lines() - { - return None; - } - - let start = rope.line_to_byte(range.start.line as usize) + range.start.character as usize; - let end = rope.line_to_byte(range.end.line as usize) + range.end.character as usize; - + let start = position_to_offset(range.start, rope)?; + let end = position_to_offset(range.end, rope)?; Some(start..end) } -pub fn offset_to_position(offset: usize, rope: &Rope) -> Option { - let line = rope.try_byte_to_line(offset).ok()?; - let line_start = rope.line_to_byte(line); - let column = offset - line_start; - Some(Position::new(line as u32, column as u32)) +pub fn offset_to_position(offset: usize, rope: &LineIndex) -> Option { + rope.byte_to_position(offset, ENCODING) } -pub fn position_to_offset(position: Position, rope: &Rope) -> Option { - let line_start = rope.try_line_to_byte(position.line as usize).ok()?; - let line_length = rope.get_line(position.line as usize)?.len_bytes(); - - // Allow cursor at exactly line_length (end of line / end of file). This is a - // valid LSP position that editors send when the cursor is after the last character. - if (position.character as usize) <= line_length { - Some(line_start + position.character as usize) - } else { - None - } +pub fn position_to_offset(position: Position, rope: &LineIndex) -> Option { + rope.position_to_byte(position, ENCODING) } -pub fn offsets_to_range(start: usize, end: usize, rope: &Rope) -> Option { +pub fn offsets_to_range(start: usize, end: usize, rope: &LineIndex) -> Option { let start = offset_to_position(start, rope)?; let end = offset_to_position(end, rope)?; Some(Range { start, end }) @@ -73,7 +61,7 @@ pub fn offsets_to_range(start: usize, end: usize, rope: &Rope) -> Option pub fn token_to_location( token: &std::ops::Range, label: &Label, - rope: &Rope, + rope: &LineIndex, ) -> Option { let range = range_to_range(token, rope)?; Some(Location { diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml index 8724cf480f..2d45e34fb4 100644 --- a/e2e/Cargo.toml +++ b/e2e/Cargo.toml @@ -13,7 +13,6 @@ homepage.workspace = true [dependencies] bevy_ecs.workspace = true futures.workspace = true -ropey.workspace = true tracing.workspace = true swls-core = { workspace = true } diff --git a/e2e/src/lib.rs b/e2e/src/lib.rs index 8adcd6db12..8468096796 100644 --- a/e2e/src/lib.rs +++ b/e2e/src/lib.rs @@ -34,7 +34,7 @@ use std::collections::HashMap; use bevy_ecs::{prelude::*, world::World}; use futures::{channel::mpsc::UnboundedReceiver, executor::block_on}; -use ropey::Rope; +use swls_core::text::LineIndex; use swls_core::{ feature::{ code_action::{CodeActionRequest, Label as CodeActionLabel}, @@ -129,7 +129,7 @@ impl LspHarness { pub fn update_file(&mut self, handle: &FileHandle, new_content: &str) { self.world .entity_mut(handle.entity) - .insert((Source(new_content.to_string()), RopeC(Rope::from_str(new_content)))); + .insert((Source(new_content.to_string()), RopeC(LineIndex::new(new_content)))); self.world.run_schedule(ParseLabel); } diff --git a/lang-jsonld/Cargo.toml b/lang-jsonld/Cargo.toml index eb5c554b33..b681c9d038 100644 --- a/lang-jsonld/Cargo.toml +++ b/lang-jsonld/Cargo.toml @@ -21,7 +21,6 @@ agnostic = ["swls-core/agnostic"] bevy_ecs.workspace = true futures.workspace = true rdf-parsers.workspace = true -ropey.workspace = true tracing.workspace = true rowan = "0.16.1" diff --git a/lang-jsonld/src/ecs/completion.rs b/lang-jsonld/src/ecs/completion.rs index bb55ab530e..1ff0685bcb 100644 --- a/lang-jsonld/src/ecs/completion.rs +++ b/lang-jsonld/src/ecs/completion.rs @@ -172,7 +172,7 @@ fn new_context_value(entry: &ContextEntry<'_>) -> String { /// opening `{` of the top-level JSON object. pub fn add_to_context( source: &str, - rope: &ropey::Rope, + rope: &LineIndex, entry: ContextEntry<'_>, ) -> Option> { match find_context_value_span(source) { @@ -491,13 +491,13 @@ mod tests { #[test] fn add_to_context_inserts_when_absent() { - use ropey::Rope; + use swls_core::text::LineIndex; use crate::ecs::completion::{add_to_context, ContextEntry}; // Document without any @context. let src = "{\n \"@id\": \"http://example.com/me\"\n}"; - let rope = Rope::from_str(src); + let rope = LineIndex::new(src); let edits = add_to_context( src, @@ -523,13 +523,13 @@ mod tests { #[test] fn add_to_context_no_duplicate() { - use ropey::Rope; + use swls_core::text::LineIndex; use crate::ecs::completion::{add_to_context, ContextEntry}; // Document where "foaf" is already declared. let src = "{\n \"@context\": {\"foaf\": \"http://xmlns.com/foaf/0.1/\"},\n \"@id\": \"http://example.com/me\"\n}"; - let rope = Rope::from_str(src); + let rope = LineIndex::new(src); let edits = add_to_context( src, diff --git a/lang-jsonld/src/ecs/mod.rs b/lang-jsonld/src/ecs/mod.rs index 56eec3fcc3..f4c4d49c73 100644 --- a/lang-jsonld/src/ecs/mod.rs +++ b/lang-jsonld/src/ecs/mod.rs @@ -272,7 +272,7 @@ fn fetch_from_web( }; let e = world .spawn(( - RopeC(ropey::Rope::from_str(&resp)), + RopeC(LineIndex::new(&resp)), Source(resp), Label(label.clone()), Wrapped(item), diff --git a/lang-jsonld/src/lib.rs b/lang-jsonld/src/lib.rs index 55e2052a94..14200f150d 100644 --- a/lang-jsonld/src/lib.rs +++ b/lang-jsonld/src/lib.rs @@ -120,7 +120,7 @@ impl LangHelper for JsonLdHelper { fn prefix_edits( &self, source: &str, - rope: &ropey::Rope, + rope: &LineIndex, name: &str, namespace: &str, _format: swls_core::components::PrefixFormat, @@ -135,7 +135,7 @@ impl LangHelper for JsonLdHelper { fn inlay_types_hint( &self, subject: &Range, - rope: &ropey::Rope, + rope: &LineIndex, last_type: Option<&Range>, types: Vec>, ) -> Option { @@ -147,7 +147,7 @@ impl LangHelper for JsonLdHelper { return None; } } else { - let offset = if rope.get_char(subject.start) == Some('[') { + let offset = if rope.char_at_byte(subject.start) == Some('[') { subject.start + 1 } else { subject.end diff --git a/lang-n3/Cargo.toml b/lang-n3/Cargo.toml index f5a36b572d..cca8ed5b3e 100644 --- a/lang-n3/Cargo.toml +++ b/lang-n3/Cargo.toml @@ -21,7 +21,6 @@ agnostic = ["swls-core/agnostic"] async-trait.workspace = true bevy_ecs.workspace = true futures.workspace = true -ropey.workspace = true sophia_iri.workspace = true tracing.workspace = true rowan = "0.16.1" diff --git a/lang-sparql/Cargo.toml b/lang-sparql/Cargo.toml index 47d68622c6..c77e050a1f 100644 --- a/lang-sparql/Cargo.toml +++ b/lang-sparql/Cargo.toml @@ -22,7 +22,6 @@ async-trait.workspace = true bevy_ecs.workspace = true futures.workspace = true lazy_static.workspace = true -ropey.workspace = true serde_json.workspace = true sophia_api.workspace = true sophia_iri.workspace = true diff --git a/lang-trig/Cargo.toml b/lang-trig/Cargo.toml index 624f6169aa..8505f05543 100644 --- a/lang-trig/Cargo.toml +++ b/lang-trig/Cargo.toml @@ -21,7 +21,6 @@ agnostic = ["swls-core/agnostic"] async-trait.workspace = true bevy_ecs.workspace = true futures.workspace = true -ropey.workspace = true sophia_iri.workspace = true tracing.workspace = true rowan = "0.16.1" diff --git a/lang-turtle/Cargo.toml b/lang-turtle/Cargo.toml index 8f8685d9f3..90ae7d5e99 100644 --- a/lang-turtle/Cargo.toml +++ b/lang-turtle/Cargo.toml @@ -21,7 +21,6 @@ agnostic = ["swls-core/agnostic"] async-trait.workspace = true bevy_ecs.workspace = true futures.workspace = true -ropey.workspace = true serde.workspace = true serde_json.workspace = true similar.workspace = true diff --git a/lang-turtle/src/config.rs b/lang-turtle/src/config.rs index 354e6c278f..733f1022cc 100644 --- a/lang-turtle/src/config.rs +++ b/lang-turtle/src/config.rs @@ -38,7 +38,7 @@ pub fn extract_known_shapes_from_config. world.entity_mut(entity).insert(( Source(t1_2.to_string()), - RopeC(Rope::from_str(t1_2)), + RopeC(LineIndex::new(t1_2)), Open, )); world.run_schedule(ParseLabel); @@ -311,7 +311,7 @@ foaf:me foaf:friend <#me>. world .entity_mut(entity) - .insert((Source(t2.to_string()), RopeC(Rope::from_str(t2)), Open)); + .insert((Source(t2.to_string()), RopeC(LineIndex::new(t2)), Open)); world.run_schedule(ParseLabel); block_on(c.await_futures(|| world.run_schedule(Tasks))); @@ -477,7 +477,7 @@ foaf:me foaf:friend <#me>. world .entity_mut(entity) - .insert((Source(t2.to_string()), RopeC(Rope::from_str(t2)), Open)); + .insert((Source(t2.to_string()), RopeC(LineIndex::new(t2)), Open)); world.run_schedule(ParseLabel); block_on(c.await_futures(|| world.run_schedule(Tasks))); @@ -521,7 +521,7 @@ foaf:me foaf:friend <#me>. world .entity_mut(entity) - .insert((Source(t2.to_string()), RopeC(Rope::from_str(t2)), Open)); + .insert((Source(t2.to_string()), RopeC(LineIndex::new(t2)), Open)); world.run_schedule(ParseLabel); block_on(c.await_futures(|| world.run_schedule(Tasks))); diff --git a/lang-turtle/src/ecs/mod.rs b/lang-turtle/src/ecs/mod.rs index 789df44cf8..6977ebe9f1 100644 --- a/lang-turtle/src/ecs/mod.rs +++ b/lang-turtle/src/ecs/mod.rs @@ -52,7 +52,7 @@ pub fn setup_completion(world: &mut World) { #[cfg(test)] mod tests { use futures::executor::block_on; - use ropey::Rope; + use swls_core::text::LineIndex; use swls_core::{ components::*, prelude::{diagnostics::DiagnosticItem, *}, @@ -94,7 +94,7 @@ mod tests { // t3: 'foa' is an invalid token → syntax errors world .entity_mut(entity) - .insert((Source(t3.to_string()), RopeC(Rope::from_str(t3)))); + .insert((Source(t3.to_string()), RopeC(LineIndex::new(t3)))); world.run_schedule(ParseLabel); let diags = last_diags(); diff --git a/lang-turtle/src/lang/formatter.rs b/lang-turtle/src/lang/formatter.rs index ffbe65c03c..41f0d993dc 100644 --- a/lang-turtle/src/lang/formatter.rs +++ b/lang-turtle/src/lang/formatter.rs @@ -3,7 +3,7 @@ use std::{ ops::Range, }; -use ropey::Rope; +use swls_core::text::LineIndex; use swls_core::{ lsp_types::FormattingOptions, prelude::{spanned, Spanned}, @@ -29,16 +29,18 @@ impl<'a> FormatState<'a> { options: FormattingOptions, buf: Buf, comments: &'a [Spanned], - source: &'a Rope, + source: &'a LineIndex, ) -> Self { let mut indent = String::new(); for _ in 0..options.tab_size { indent.push(' '); } + // Sentinel span past the end of the document, so the trailing comment + // bucket sorts after every real span. let tail = spanned( String::new(), - source.len_chars() + 1..source.len_chars() + 1, + source.len_bytes() + 1..source.len_bytes() + 1, ); Self { tail, @@ -314,7 +316,7 @@ pub fn format_turtle( turtle: &Turtle, config: FormattingOptions, comments: &[Spanned], - source: &Rope, + source: &LineIndex, ) -> Option { let buf: Buf = Cursor::new(Vec::new()); let mut state = FormatState::new(config, buf, comments, source); @@ -334,7 +336,7 @@ mod tests { use std::str::FromStr; use rdf_parsers::turtle::SyntaxKind; - use ropey::Rope; + use swls_core::text::LineIndex; use rowan::NodeOrToken; use swls_core::prelude::Spanned; @@ -392,7 +394,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -421,7 +423,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -458,7 +460,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -487,7 +489,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -512,7 +514,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -542,7 +544,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -577,7 +579,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -611,7 +613,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); @@ -657,7 +659,7 @@ mod tests { ..Default::default() }, &comments, - &Rope::from_str(txt), + &LineIndex::new(txt), ) .expect("formatting"); assert_eq!(formatted, expected); diff --git a/swls/Cargo.toml b/swls/Cargo.toml index 2ba3d0627f..e2181d0e66 100644 --- a/swls/Cargo.toml +++ b/swls/Cargo.toml @@ -17,7 +17,6 @@ bevy_ecs.workspace = true chumsky.workspace = true futures.workspace = true reqwest.workspace = true -ropey.workspace = true serde_json.workspace = true serde.workspace = true sophia_api.workspace = true diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 403374d2a5..b2254b612f 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -15,7 +15,6 @@ homepage.workspace = true bevy_ecs.workspace = true chumsky.workspace = true futures.workspace = true -ropey.workspace = true serde_json.workspace = true sophia_api.workspace = true async-trait.workspace = true diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 94610f1b2e..c0bf9b9d10 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -35,6 +35,7 @@ use swls_core::{ }, setup_schedule_labels, systems::{handle_tasks, spawn_or_insert}, + text::LineIndex, Startup, }; @@ -296,7 +297,7 @@ pub fn create_file( url.clone(), ( Source(content.to_string()), - RopeC(ropey::Rope::from_str(content)), + RopeC(LineIndex::new(content)), Label(url), // this might crash Wrapped(item), Types(HashMap::new()),