diff --git a/crates/pgls_statement_splitter/src/diagnostics.rs b/crates/pgls_statement_splitter/src/diagnostics.rs index c929ad3b6..7f7c37ba8 100644 --- a/crates/pgls_statement_splitter/src/diagnostics.rs +++ b/crates/pgls_statement_splitter/src/diagnostics.rs @@ -27,6 +27,17 @@ impl SplitDiagnostic { } } +impl SplitDiagnostic { + pub(crate) fn span(&self) -> Option { + self.span + } + + pub(crate) fn with_span(mut self, span: TextRange) -> Self { + self.span = Some(span); + self + } +} + impl From for SplitDiagnostic { fn from(lex_diagnostic: LexDiagnostic) -> Self { Self { diff --git a/crates/pgls_statement_splitter/src/lib.rs b/crates/pgls_statement_splitter/src/lib.rs index d745a93d8..21efb2e1c 100644 --- a/crates/pgls_statement_splitter/src/lib.rs +++ b/crates/pgls_statement_splitter/src/lib.rs @@ -6,12 +6,16 @@ mod splitter; use diagnostics::SplitDiagnostic; use pgls_lexer::{Lexed, Lexer, SyntaxKind, convert_to_positional_params}; -use pgls_text_size::TextRange; +use pgls_text_size::{TextRange, TextSize}; use splitter::{SplitResult as PassResult, Splitter, source}; +#[derive(Debug, Clone, PartialEq)] pub struct SplitResult { pub ranges: Vec, + /// The errors of the lexer, ordered by position, then those of the splitter, by statement. pub errors: Vec, + /// How many of the `errors` come from the lexer. + lex_errors: usize, } /// Splits `sql` into the ranges of the statements it contains. @@ -46,6 +50,7 @@ pub fn split(sql: &str) -> SplitResult { let mut ranges = Vec::with_capacity(coarse.ranges.len()); let mut errors: Vec = lexed.errors().into_iter().map(Into::into).collect(); + let lex_errors = errors.len(); for range in coarse.ranges { let contains_blank_line = blank_lines.as_mut().is_some_and(|blank_lines| { @@ -96,7 +101,224 @@ pub fn split(sql: &str) -> SplitResult { ); } - SplitResult { ranges, errors } + SplitResult { + ranges, + errors, + lex_errors, + } +} + +/// The statements around a change that are split again by [`split_edited`], on each side of it. +/// The ones farthest from the change have to come out as before, which shows that the split +/// around the change doesn't depend on the text outside of it. +const EDIT_CONTEXT_STATEMENTS: usize = 2; + +/// Bytes that can open or close a string, a quoted identifier, a comment or a psql command. A +/// change that touches one can change how everything after it is lexed. +const LEXER_STATE_BYTES: &[u8] = b"$'\"/*-\\"; + +/// Splits `new` like [`split`], reusing `previous`, the split of `old`. +/// +/// Only the statements around the part of the text that changed are split again: from +/// [`EDIT_CONTEXT_STATEMENTS`] statements before it to as many after it. The rest is taken from +/// `previous`, shifted by the change in length. The result is the same as `split(new)`. It +/// falls back to splitting all of `new` when the change can affect statements far from it: +/// when it touches a quote, a comment or a psql command, or when the statements at the edges of +/// the region don't come out as before. +pub fn split_edited(old: &str, previous: &SplitResult, new: &str) -> SplitResult { + split_region(old, previous, new).unwrap_or_else(|| split(new)) +} + +/// Compared in blocks first, which is much faster than byte by byte. +const COMPARE_BLOCK_BYTES: usize = 64; + +/// The length of the longest common prefix of `a` and `b`. +fn common_prefix(a: &[u8], b: &[u8]) -> usize { + let len = a.len().min(b.len()); + let mut equal = 0; + while equal + COMPARE_BLOCK_BYTES <= len + && a[equal..equal + COMPARE_BLOCK_BYTES] == b[equal..equal + COMPARE_BLOCK_BYTES] + { + equal += COMPARE_BLOCK_BYTES; + } + equal + + a[equal..len] + .iter() + .zip(&b[equal..len]) + .take_while(|(a, b)| a == b) + .count() +} + +/// The length of the longest common suffix of `a` and `b`, which have the same length. +fn common_suffix(a: &[u8], b: &[u8]) -> usize { + let len = a.len(); + let mut equal = 0; + while equal + COMPARE_BLOCK_BYTES <= len + && a[len - equal - COMPARE_BLOCK_BYTES..len - equal] + == b[len - equal - COMPARE_BLOCK_BYTES..len - equal] + { + equal += COMPARE_BLOCK_BYTES; + } + equal + + a[..len - equal] + .iter() + .rev() + .zip(b[..len - equal].iter().rev()) + .take_while(|(a, b)| a == b) + .count() +} + +fn split_region(old: &str, previous: &SplitResult, new: &str) -> Option { + let (old_bytes, new_bytes) = (old.as_bytes(), new.as_bytes()); + let mut prefix = common_prefix(old_bytes, new_bytes); + while !old.is_char_boundary(prefix) || !new.is_char_boundary(prefix) { + prefix -= 1; + } + let max_suffix = old.len().min(new.len()) - prefix; + let mut suffix = common_suffix( + &old_bytes[old.len() - max_suffix..], + &new_bytes[new.len() - max_suffix..], + ); + while !old.is_char_boundary(old.len() - suffix) || !new.is_char_boundary(new.len() - suffix) { + suffix -= 1; + } + let (old_end, new_end) = (old.len() - suffix, new.len() - suffix); + if old_end == prefix && new_end == prefix { + return Some(previous.clone()); + } + + // The change, with a byte of context on each side, because two bytes can form a token like + // `/*` or `--` together. + let touches_lexer_state = |text: &[u8], end: usize| { + text[prefix.saturating_sub(1)..(end + 1).min(text.len())] + .iter() + .any(|byte| LEXER_STATE_BYTES.contains(byte)) + }; + if touches_lexer_state(old_bytes, old_end) || touches_lexer_state(new_bytes, new_end) { + return None; + } + + let ranges = &previous.ranges; + let to_size = |offset: usize| TextSize::try_from(offset).ok(); + let (prefix_size, old_end_size) = (to_size(prefix)?, to_size(old_end)?); + // The statements from `first` to `last` (exclusive) are split again: the ones that the + // change touches, and `EDIT_CONTEXT_STATEMENTS` on each side. + let touched = ranges.partition_point(|range| range.end() < prefix_size); + let after = ranges.partition_point(|range| range.start() < old_end_size); + let first = touched.saturating_sub(EDIT_CONTEXT_STATEMENTS); + let last = (after + EDIT_CONTEXT_STATEMENTS).min(ranges.len()); + + let start = if first == 0 { + 0 + } else { + usize::from(ranges[first].start()) + }; + let new_region_end = if last == ranges.len() { + new.len() + } else { + (usize::from(ranges[last - 1].end()) + new.len()).checked_sub(old.len())? + }; + if start > prefix || new_region_end < new_end { + return None; + } + let offset = to_size(start)?; + let shift = |range: TextRange| -> Option { + let start = (usize::from(range.start()) + new.len()).checked_sub(old.len())?; + let end = (usize::from(range.end()) + new.len()).checked_sub(old.len())?; + Some(TextRange::new(to_size(start)?, to_size(end)?)) + }; + + let region = split(&new[start..new_region_end]); + let region_ranges: Vec = region.ranges.iter().map(|range| *range + offset).collect(); + + // The statements before and after the change come out as before, so the region doesn't + // depend on the text around it. + let before = touched - first; + if first > 0 + && (region_ranges.len() < before || region_ranges[..before] != ranges[first..touched]) + { + return None; + } + if last < ranges.len() { + let after_count = last - after; + if region_ranges.len() < after_count { + return None; + } + let expected = ranges[after..last] + .iter() + .map(|range| shift(*range)) + .collect::>>()?; + if region_ranges[region_ranges.len() - after_count..] != expected[..] { + return None; + } + } + + // The first and the last statement of the region only show that the region lines up with + // the statements around it: the first one is split without the statements before it, and + // the last one runs into the end of the region. Both are unchanged, so they are taken from + // `previous`, like the statements outside the region. + let low = if first > 0 { + ranges[first].end() + } else { + TextSize::from(0) + }; + let (old_high, new_high) = if last < ranges.len() { + let old_high = ranges[last - 1].start(); + (old_high, shift(TextRange::empty(old_high))?.start()) + } else { + (to_size(old.len())?, to_size(new.len())?) + }; + let kept_before = if first > 0 { first + 1 } else { 0 }; + let kept_after = if last < ranges.len() { + last - 1 + } else { + ranges.len() + }; + + let mut result_ranges = Vec::with_capacity(ranges.len() + region_ranges.len()); + result_ranges.extend_from_slice(&ranges[..kept_before]); + result_ranges.extend( + region_ranges + .into_iter() + .filter(|range| range.start() >= low && range.start() < new_high), + ); + for range in &ranges[kept_after..] { + result_ranges.push(shift(*range)?); + } + + // The same for the errors. The errors of the lexer come first, as in `split`. + let place = |errors: &[SplitDiagnostic], region: &[SplitDiagnostic]| { + let mut placed = Vec::new(); + for error in errors { + if error.span()?.end() <= low { + placed.push(error.clone()); + } + } + for error in region { + let span = error.span()? + offset; + if span.start() >= low && span.end() <= new_high { + placed.push(error.clone().with_span(span)); + } + } + for error in errors { + let span = error.span()?; + if span.start() >= old_high { + placed.push(error.clone().with_span(shift(span)?)); + } + } + Some(placed) + }; + let (old_lex, old_split) = previous.errors.split_at(previous.lex_errors); + let (region_lex, region_split) = region.errors.split_at(region.lex_errors); + let mut errors = place(old_lex, region_lex)?; + let lex_errors = errors.len(); + errors.extend(place(old_split, region_split)?); + + Some(SplitResult { + ranges: result_ranges, + errors, + lex_errors, + }) } fn split_pass(lexed: &Lexed, blank_line_is_boundary: bool) -> PassResult { @@ -1184,4 +1406,136 @@ VALUES assert_eq!(split("").ranges.len(), 0); assert_eq!(split(" \n \n ").ranges.len(), 0); } + + /// A small deterministic random number generator (xorshift), so that a failure reproduces. + struct Rng(u64); + + impl Rng { + fn below(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % n.max(1) as u64) as usize + } + } + + /// Statements of the kinds the splitter treats differently. + const STATEMENTS: &[&str] = &[ + "create table t (id bigint primary key, name text not null);", + "create index t_name_idx on t (name);", + "create function f(a int) returns int language sql as $$ select a + 1 $$;", + "create function g(a int) returns int language sql begin atomic select a; select a + 1; end;", + "select c.relname from pg_class c where c.relnatts = 1 order by 1;", + "insert into t (id, name) values (1, 'x') on conflict (id) do update set name = 'y';", + "update t set name = 'z' where id = 1", + "delete from t where id in (select id from t)", + "with x as (select 1) select * from x;", + "-- a comment\nselect 1;", + "/* a block\ncomment */ select 2;", + "select 'a;b', \"c;d\";", + "alter table t add column c int;", + "begin;", + "commit;", + "select 1\n\nfrom t;", + "selec 1;", + "grant select on t to public;", + ]; + + /// Text inserted by the edits: whole statements, and fragments that can merge, cut or quote + /// statements. + const SNIPPETS: &[&str] = &[ + ";", + "\n", + "\n\n", + " ", + "x", + "select 1", + "select", + "from t", + "begin atomic", + "end;", + "(", + ")", + "'", + "$$", + "/*", + "*/", + "--", + ",", + "with", + "delete", + "create table q (a int);", + "\\d t\n", + "é", + "insert into", + "case when true then 1 end", + ]; + + fn document(rng: &mut Rng, statements: usize) -> String { + let mut sql = String::new(); + for _ in 0..statements { + sql.push_str(STATEMENTS[rng.below(STATEMENTS.len())]); + sql.push_str(["\n", "\n\n", " "][rng.below(3)]); + } + sql + } + + fn char_boundary(sql: &str, mut offset: usize) -> usize { + while !sql.is_char_boundary(offset) { + offset -= 1; + } + offset + } + + fn edit(rng: &mut Rng, sql: &str) -> String { + let at = char_boundary(sql, rng.below(sql.len() + 1)); + let end = char_boundary(sql, (at + rng.below(40)).min(sql.len())); + let inserted = match rng.below(4) { + 0 => "", + 1 => STATEMENTS[rng.below(STATEMENTS.len())], + _ => SNIPPETS[rng.below(SNIPPETS.len())], + }; + let end = if inserted.is_empty() || rng.below(2) == 0 { + end + } else { + at + }; + format!("{}{inserted}{}", &sql[..at], &sql[end..]) + } + + #[test] + fn split_edited_is_split() { + let seed = + std::env::var("SPLIT_SEED").map_or(0x9e37_79b9_7f4a_7c15, |seed| seed.parse().unwrap()); + let edits: usize = + std::env::var("SPLIT_EDITS").map_or(3_000, |edits| edits.parse().unwrap()); + let mut rng = Rng(seed); + let mut incremental_edits = 0; + for document_index in 0..10 { + let mut sql = document(&mut rng, 60); + let mut previous = split(&sql); + for _ in 0..edits / 10 { + // Sometimes two edits at once, which the region has to cover together. + let mut new = edit(&mut rng, &sql); + if rng.below(4) == 0 { + new = edit(&mut rng, &new); + } + let expected = split(&new); + let edited = split_edited(&sql, &previous, &new); + assert_eq!( + edited, expected, + "document {document_index}, from\n{sql}\nto\n{new}" + ); + incremental_edits += usize::from(split_region(&sql, &previous, &new).is_some()); + sql = new; + previous = edited; + } + } + // Most edits are split incrementally; the others fall back to a full split. + println!("{incremental_edits} of {edits} edits split incrementally"); + assert!( + incremental_edits > edits / 3, + "{incremental_edits} of {edits}" + ); + } } diff --git a/crates/pgls_suppressions/src/line_index.rs b/crates/pgls_suppressions/src/line_index.rs index ca71e235d..ba0558195 100644 --- a/crates/pgls_suppressions/src/line_index.rs +++ b/crates/pgls_suppressions/src/line_index.rs @@ -28,16 +28,9 @@ impl LineIndex { } pub fn line_for_offset(&self, offset: TextSize) -> Option { + // The offsets are ordered: the last line that starts at or before `offset`. self.line_offset - .iter() - .enumerate() - .filter_map(|(i, line_offset)| { - if offset >= *line_offset { - Some(i) - } else { - None - } - }) - .next_back() + .partition_point(|line_offset| *line_offset <= offset) + .checked_sub(1) } } diff --git a/crates/pgls_suppressions/src/parser.rs b/crates/pgls_suppressions/src/parser.rs index 7110db79d..e19b02c1b 100644 --- a/crates/pgls_suppressions/src/parser.rs +++ b/crates/pgls_suppressions/src/parser.rs @@ -40,6 +40,16 @@ impl<'a> SuppressionsParser<'a> { } pub fn parse(doc: &str) -> Suppressions { + // Every suppression comment contains `-ignore`. Most documents have none, and finding + // that out is much faster than looking at each line. Without line suppressions, the + // line index isn't needed either. + if !doc.contains("-ignore") { + return Suppressions::default(); + } + Self::parse_lines(doc) + } + + fn parse_lines(doc: &str) -> Suppressions { let mut parser = SuppressionsParser::new(doc); parser.parse_file_suppressions(); @@ -177,6 +187,33 @@ impl<'a> SuppressionsParser<'a> { #[cfg(test)] mod tests { + use super::SuppressionsParser; + + #[test] + fn documents_without_suppressions_skip_the_lines() { + for doc in [ + "", + "select 1;", + "-- a comment\nselect 1;\r\n\r\nselect 2;\n", + "-- pgls\n--ignore\nselect 1;", + ] { + let fast = SuppressionsParser::parse(doc); + let slow = SuppressionsParser::parse_lines(doc); + assert!(slow.file_suppressions.is_empty(), "{doc}"); + assert!(slow.line_suppressions.is_empty(), "{doc}"); + assert!(slow.range_suppressions.is_empty(), "{doc}"); + assert!(slow.diagnostics.is_empty(), "{doc}"); + if !doc.contains("-ignore") { + assert_eq!( + format!("{fast:?}"), + format!("{:?}", super::Suppressions::default()) + ); + } else { + assert_eq!(format!("{fast:?}"), format!("{slow:?}")); + } + } + } + use super::*; use crate::suppression::{RuleSpecifier, SuppressionKind}; diff --git a/crates/pgls_workspace/src/workspace/server/document.rs b/crates/pgls_workspace/src/workspace/server/document.rs index 0841817a6..ea9e74cc3 100644 --- a/crates/pgls_workspace/src/workspace/server/document.rs +++ b/crates/pgls_workspace/src/workspace/server/document.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use pgls_analyser::{AnalysableStatement, UnparsableStatement}; use pgls_diagnostics::{Diagnostic, DiagnosticExt, serde::Diagnostic as SDiagnostic}; use pgls_query_ext::{diagnostics::SyntaxDiagnostic, function_body::OffsetMap}; +use pgls_statement_splitter::SplitResult; use pgls_suppressions::Suppressions; use pgls_text_size::{TextRange, TextSize}; use rustc_hash::FxHashSet; @@ -15,7 +16,8 @@ use super::{ pub struct Document { content: String, version: i32, - ranges: Vec, + /// The statements of `content`, kept to split the next version of it incrementally. + split: SplitResult, diagnostics: Vec, cache: DocumentCache, suppressions: Suppressions, @@ -25,11 +27,13 @@ pub struct Document { impl Document { pub fn new(content: String, version: i32) -> Document { - let (ranges, diagnostics) = split_with_diagnostics(&content, None); - let suppressions = Suppressions::from(content.as_str()).with_statements(ranges.clone()); + let split = pgls_statement_splitter::split(&content); + let diagnostics = split_diagnostics(&split); + let suppressions = + Suppressions::from(content.as_str()).with_statements(split.ranges.clone()); Document { - ranges, + split, diagnostics, content, version, @@ -43,21 +47,24 @@ impl Document { /// for the caller to drop, see [`StaleEntries::drop_in_background`]. pub fn update_content(&mut self, content: String, version: i32) -> StaleEntries { let unchanged = common_prefix(&self.content, &content); + // Only the statements around the change are split again. + let split = pgls_statement_splitter::split_edited(&self.content, &self.split, &content); + let ranges = &split.ranges; self.content = content; self.version = version; - let (ranges, diagnostics) = split_with_diagnostics(&self.content, None); - // The statements before the change keep what the walks cached for them. The splitter // looks ahead, so a statement before the change can still end elsewhere. let first_changed_statement = self + .split .ranges .iter() - .zip(&ranges) + .zip(ranges) .find(|(old, new)| old != new || old.end() > unchanged) .map_or_else( || { - self.ranges + self.split + .ranges .get(ranges.len()) .map_or(TextSize::from(u32::MAX), |range| range.start()) }, @@ -67,11 +74,12 @@ impl Document { self.suppressions = Suppressions::from(self.content.as_str()).with_statements(ranges.clone()); - self.ranges = ranges; - self.diagnostics = diagnostics; + self.diagnostics = split_diagnostics(&split); + self.split = split; // Keep what is cached for the statements that are still in the document. The bodies // of SQL functions are kept with their function. let statements: FxHashSet<&str> = self + .split .ranges .iter() .map(|range| &self.content[*range]) @@ -98,6 +106,7 @@ impl Document { /// What the caches of the document hold, for tests of their memory. It doesn't fill them. pub(crate) fn cache_stats(&self) -> DocumentCacheStats { let statements: FxHashSet = self + .split .ranges .iter() .map(|range| StatementId::new(&self.content[*range])) @@ -111,7 +120,7 @@ impl Document { /// The ranges of the top-level statements, in file order. pub fn statement_ranges(&self) -> &[TextRange] { - &self.ranges + &self.split.ranges } pub fn get_document_content(&self) -> &str { @@ -157,8 +166,11 @@ impl Document { { // The ranges are ordered and don't overlap, so only the first one that ends after // `position` can contain it. - let first = self.ranges.partition_point(|range| range.end() <= position); - let ranges = &self.ranges[first..self.ranges.len().min(first + 1)]; + let first = self + .split + .ranges + .partition_point(|range| range.end() <= position); + let ranges = &self.split.ranges[first..self.split.ranges.len().min(first + 1)]; ParseIterator { ranges: ranges.iter(), ..ParseIterator::new(self, mapper, CursorPositionFilter::new(position)) @@ -170,9 +182,12 @@ impl Document { where M: StatementMapper<'a>, { - let first = self.ranges.partition_point(|range| range.start() < offset); + let first = self + .split + .ranges + .partition_point(|range| range.start() < offset); ParseIterator { - ranges: self.ranges[first..].iter(), + ranges: self.split.ranges[first..].iter(), ..ParseIterator::new(self, mapper, NoFilter) } } @@ -189,7 +204,7 @@ impl Document { { // The filter accepts a statement up to two characters after its end. The bodies of SQL // functions are inside their top-level statement. - let ranges = &self.ranges; + let ranges = &self.split.ranges; let first = ranges.partition_point(|range| range.end() + TextSize::from(2) <= position); let last = first + ranges[first..].partition_point(|range| range.start() <= position); ParseIterator { @@ -267,7 +282,7 @@ impl<'a, M, F> ParseIterator<'a, M, F> { parser, mapper, filter, - ranges: parser.ranges.iter(), + ranges: parser.split.ranges.iter(), pending_embedded: Vec::new(), } } @@ -545,28 +560,13 @@ fn common_prefix(a: &str, b: &str) -> TextSize { TextSize::try_from(equal).unwrap_or_default() } -/// Helper function that wraps the statement splitter and returns the ranges with unified -/// diagnostics -pub(crate) fn split_with_diagnostics( - content: &str, - offset: Option, -) -> (Vec, Vec) { - let o = offset.unwrap_or_else(|| 0.into()); - let result = pgls_statement_splitter::split(content); - - ( - result.ranges, - result - .errors - .into_iter() - .map(|err| { - SDiagnostic::new( - err.clone() - .with_file_span(err.location().span.map(|r| r + o)), - ) - }) - .collect(), - ) +/// The errors of the statement splitter as diagnostics. +fn split_diagnostics(split: &SplitResult) -> Vec { + split + .errors + .iter() + .map(|err| SDiagnostic::new(err.clone().with_file_span(err.location().span))) + .collect() } #[cfg(test)] @@ -976,4 +976,58 @@ $$ LANGUAGE plpgsql;"; let result = d.find(fake_id, DefaultMapper); assert!(result.is_none()); } + + /// Each change splits only the statements around it, with the same result as splitting the + /// whole document. + #[test] + fn updates_split_like_the_whole_document() { + let statements = [ + "create table t (id bigint primary key, name text);", + "create function f(a int) returns int language sql as $$ select a $$;", + "select c.relname from pg_class c where c.relnatts = 1;", + "insert into t values (1, 'x');", + "-- comment\nselect 1;", + "update t set name = 'y'", + ]; + let snippets = [ + "x", + " ", + "\n", + "\n\n", + ";", + "select 2;", + "'", + "$$", + "(", + ")", + "", + ]; + let mut seed: u64 = 0x2545_f491_4f6c_dd1d; + let mut below = |n: usize| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed % n as u64) as usize + }; + let mut content: String = (0..200) + .map(|i| statements[i % statements.len()]) + .collect::>() + .join("\n"); + let mut d = Document::new(content.clone(), 0); + for version in 1..1_000 { + let at = below(content.len() + 1); + let end = (at + below(20)).min(content.len()); + let inserted = snippets[below(snippets.len())]; + content = format!("{}{inserted}{}", &content[..at], &content[end..]); + d.update_content(content.clone(), version) + .drop_in_background(); + + let whole = pgls_statement_splitter::split(&content); + assert_eq!(d.split, whole, "{content}"); + assert_eq!( + format!("{:?}", d.diagnostics), + format!("{:?}", split_diagnostics(&whole)) + ); + } + } }