From 45c22d206c0c8f0b4c1ebb261407ca29bcee0a38 Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Wed, 15 Jul 2026 15:41:38 +0300 Subject: [PATCH 1/4] Extend lexer to support Newline, Space for formatting * refactor lexer.rs * reduce value ignoring when we pass tokens to parser stage after lexer * add `fmt` feature --- Cargo.toml | 1 + src/lexer.rs | 396 ++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 342 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 61fe2642..9abba208 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ default = [ "serde" ] serde = ["dep:serde", "dep:serde_json"] docs = [] external-jets = [] +fmt = [] [dependencies] base64 = "0.21.2" diff --git a/src/lexer.rs b/src/lexer.rs index fb6e8616..5cd5f69c 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,7 +11,11 @@ use crate::version::SIMC_STR; pub type Spanned = (T, SimpleSpan); pub type Tokens<'src> = Vec<(Token<'src>, crate::error::Span)>; +#[cfg(feature = "fmt")] +pub type FmtTokens<'src> = Vec<(FmtToken<'src>, crate::error::Span)>; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Token<'src> { // Keywords Pub, @@ -66,12 +70,22 @@ pub enum Token<'src> { // Built-in functions Macro(&'src str), +} - // Comments and block comments - // - // We would discard them for the compiler, but they are needed, for example, for the formatter. - Comment, +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum TriviaKind { + LineComment, BlockComment, + Newline, + Whitespace, +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum FmtToken<'src> { + Token(Token<'src>), + Trivia(TriviaKind), } impl<'src> fmt::Display for Token<'src> { @@ -118,9 +132,23 @@ impl<'src> fmt::Display for Token<'src> { Token::Param(s) => write!(f, "param::{}", s), Token::Bool(b) => write!(f, "{}", b), + } + } +} - Token::Comment => write!(f, "comment"), - Token::BlockComment => write!(f, "block_comment"), +#[cfg(feature = "fmt")] +impl<'src> fmt::Display for FmtToken<'src> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FmtToken::Token(t) => { + write!(f, "{}", t) + } + FmtToken::Trivia(t) => match t { + TriviaKind::LineComment => write!(f, "comment"), + TriviaKind::BlockComment => write!(f, "block_comment"), + TriviaKind::Newline => writeln!(f), + TriviaKind::Whitespace => write!(f, " "), + }, } } } @@ -128,11 +156,42 @@ impl<'src> fmt::Display for Token<'src> { /// Recognizer for a `// ...` line comment. fn line_comment<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + let newline = newline_choice(); + just("//") - .then(any().and_is(just('\n').not()).repeated()) + .ignore_then(any().and_is(newline.not()).repeated()) + .ignored() +} + +/// Recognizer for different kinds of newline (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +/// Returns the exact string slice that was matched. +fn newline_choice<'src>( +) -> impl Parser<'src, &'src str, &'src str, extra::Err>> + Clone { + choice((just("\r\n"), just("\n"), just("\r"))) +} + +/// Recognizer for different kinds of newline (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +fn newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + newline_choice().ignored() +} + +/// Recognizer for whitespace. +fn whitespace<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace() && *c != '\n' && *c != '\r') + .repeated() + .at_least(1) .ignored() } +/// Recognizer for whitespace or a newline. +fn whitespace_or_newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any().filter(|c: &char| c.is_whitespace()).ignored() +} + /// Recognizer for a (possibly nested) `/* ... */` block comment; an unterminated /// comment is reported and swallows the rest of the input. fn block_comment<'src>( @@ -155,18 +214,13 @@ fn block_comment<'src>( /// syntax. pub(crate) fn trivia<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> { - choice(( - line_comment(), - block_comment(), - any().filter(|c: &char| c.is_whitespace()).ignored(), - )) - .repeated() - .ignored() + choice((line_comment(), block_comment(), whitespace_or_newline())) + .repeated() + .ignored() } -pub fn lexer<'src>( -) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> -{ +fn to_token<'src>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { let digits_with_underscore = |radix: u32| { any() .filter(move |c: &char| c.is_digit(radix)) @@ -240,25 +294,32 @@ pub fn lexer<'src>( just(">").to(Token::RAngle), )); - let comment = line_comment().to(Token::Comment); - let block_comment = block_comment().to(Token::BlockComment); - let token = choice(( - comment, - block_comment, - jet, - witness, - param, - macros, - keyword, - hex, - bin, - num, - op, - )); + choice((jet, witness, param, macros, keyword, hex, bin, num, op)) +} - token +pub fn lexer<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token() .map_with(|tok, e| (tok, e.span())) - .padded() + .recover_with(skip_then_retry_until(any().ignored(), end())); + + trivia().ignore_then(token.then_ignore(trivia()).repeated().collect()) +} + +#[cfg(feature = "fmt")] +pub fn lexer_lossless<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token().map(FmtToken::Token); + + let newline = newline().to(FmtToken::Trivia(TriviaKind::Newline)); + let whitespace = whitespace().to(FmtToken::Trivia(TriviaKind::Whitespace)); + let line_comment = line_comment().to(FmtToken::Trivia(TriviaKind::LineComment)); + let block_comment = block_comment().to(FmtToken::Trivia(TriviaKind::BlockComment)); + + choice((line_comment, block_comment, newline, whitespace, token)) + .map_with(|lexeme, e| (lexeme, e.span())) .recover_with(skip_then_retry_until(any().ignored(), end())) .repeated() .collect() @@ -268,43 +329,99 @@ pub fn lexer<'src>( /// offset `start` — the end of the version directive per `SimcDirective::prescan`, /// or `0`. Spans are reported relative to the full input. /// -/// All comments in the input code are discarded. +/// All comments, newlines, and spaces in the input code are discarded. pub fn lex( file_id: usize, input: &str, start: usize, ) -> (Option>, Vec) { let (tokens, lex_errors) = lexer().parse(&input[start..]).into_output_errors(); - let shift = |span: SimpleSpan| Span::new(file_id, span.start + start..span.end + start); + let shift = |span: SimpleSpan| simple_span_to_span(span, file_id, start); // The reserved-keyword errors come first: a stray directive also produces // follow-up errors for its `"";` remnant, and the sentinel is their cause. let mut errors: Vec = Vec::new(); let tokens = tokens.map(|vec| { vec.into_iter() - .filter_map(|(tok, span)| match tok { - Token::Comment | Token::BlockComment => None, - // The reserved keyword is a sentinel: the prescan consumed the one - // legitimate directive before lexing, so any occurrence is misplaced. - Token::Simc => { - errors.push(RichError::new(Error::ReservedSimcKeyword, shift(span))); - None - } - tok => Some((tok, shift(span))), + .filter_map(|(tok, span)| filter_token(tok, span, &mut errors, shift)) + .collect() + }); + + finalize_lex_errors(lex_errors, &mut errors, shift); + + (tokens, errors) +} + +/// Lexes an input string into a lossles stream of tokens with spans, beginning at byte +/// offset `start` — the end of the version directive per `SimcDirective::prescan`, +/// or `0`. Spans are reported relative to the full input. +/// +/// All comments, newlines, and spaces in the input code are remained. +#[cfg(feature = "fmt")] +pub fn lex_lossless( + file_id: usize, + input: &str, + start: usize, +) -> (Option>, Vec) { + let (tokens, lex_errors) = lexer_lossless().parse(&input[start..]).into_output_errors(); + let shift = |span: SimpleSpan| simple_span_to_span(span, file_id, start); + + // The reserved-keyword errors come first: a stray directive also produces + // follow-up errors for its `"";` remnant, and the sentinel is their cause. + let mut errors: Vec = Vec::new(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .filter_map(|(fmt_tok, span)| match fmt_tok { + FmtToken::Token(tok) => filter_token(tok, span, &mut errors, shift) + .map(|(t, s)| (FmtToken::Token(t), s)), + FmtToken::Trivia(t) => Some((FmtToken::Trivia(t), shift(span))), }) .collect() }); + finalize_lex_errors(lex_errors, &mut errors, shift); + + (tokens, errors) +} + +fn finalize_lex_errors Span>( + lex_errors: Vec>, + errors: &mut Vec, + convert_span: F, +) { errors.extend(lex_errors.into_iter().map(|err| { RichError::new( Error::CannotParse { msg: err.reason().to_string(), }, - shift(*err.span()), + convert_span(*err.span()), ) })); +} - (tokens, errors) +#[inline] +fn simple_span_to_span(span: SimpleSpan, file_id: usize, start: usize) -> Span { + Span::new(file_id, span.start + start..span.end + start) +} + +fn filter_token<'src, F: Fn(SimpleSpan) -> Span>( + tok: Token<'src>, + span: SimpleSpan, + errors: &mut Vec, + convert_span: F, +) -> Option<(Token<'src>, Span)> { + match tok { + // The reserved keyword is a sentinel: the prescan consumed the one + // legitimate directive before lexing, so any occurrence is misplaced. + Token::Simc => { + errors.push(RichError::new( + Error::ReservedSimcKeyword, + convert_span(span), + )); + None + } + tok => Some((tok, convert_span(span))), + } } /// A list of all reserved keywords. @@ -328,7 +445,27 @@ mod tests { input: &'src str, ) -> (Option>>, Vec>) { let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| vec.iter().map(|(tok, _)| tok.clone()).collect::>()); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + + #[cfg(feature = "fmt")] + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); (tokens, errors) } @@ -340,7 +477,21 @@ mod tests { assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); assert_eq!( tokens, - Some(vec![Token::BlockComment]), + Some(vec![]), + "Should produce a single block comment token" + ); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]), "Should produce a single block comment token" ); } @@ -351,7 +502,20 @@ mod tests { let (tokens, errors) = lex(input); assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -360,7 +524,20 @@ mod tests { let (tokens, errors) = lex(input); assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -369,7 +546,20 @@ mod tests { let (tokens, errors) = lex(input); assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -384,7 +574,26 @@ mod tests { assert_eq!(err.span().end, 2); assert_eq!(err.to_string(), "Unclosed block comment"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -394,7 +603,21 @@ mod tests { assert_eq!(errors.len(), 1); assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -410,7 +633,27 @@ mod tests { assert_eq!(errors[1].span().start, 0); assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[cfg(feature = "fmt")] + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); } #[test] @@ -438,6 +681,33 @@ mod tests { assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); } + #[cfg(feature = "fmt")] + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "unexpected: {errors:?}"); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } + #[test] fn lexer_test() { use chumsky::prelude::*; @@ -450,4 +720,20 @@ mod tests { assert!(lex_errs.is_empty()); } + + #[cfg(feature = "fmt")] + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); + + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } + + // TODO: add more tests for fmt lexer (\n\r, \r, newline spans, spaces) } From 979559f325e5113fd10f37fc174406c9bdaa8ad4 Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Wed, 15 Jul 2026 22:30:04 +0300 Subject: [PATCH 2/4] add additional utils --- src/lexer.rs | 21 +++- src/parse.rs | 265 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 275 insertions(+), 11 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 5cd5f69c..f1f82903 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -147,7 +147,7 @@ impl<'src> fmt::Display for FmtToken<'src> { TriviaKind::LineComment => write!(f, "comment"), TriviaKind::BlockComment => write!(f, "block_comment"), TriviaKind::Newline => writeln!(f), - TriviaKind::Whitespace => write!(f, " "), + TriviaKind::Whitespace => write!(f, "< >"), }, } } @@ -496,6 +496,23 @@ mod tests { ); } + #[cfg(feature = "fmt")] + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| matches!(token, FmtToken::Trivia(TriviaKind::Newline))) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } + #[test] fn test_block_comment_nested() { let input = "/* outer /* inner */ outer */"; @@ -734,6 +751,4 @@ mod tests { assert!(lex_errs.is_empty()); } - - // TODO: add more tests for fmt lexer (\n\r, \r, newline spans, spaces) } diff --git a/src/parse.rs b/src/parse.rs index 61c20360..86e29f6f 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -21,7 +21,10 @@ use crate::driver::{CRATE_STR, MAIN_MODULE}; use crate::error::ErrorCollector; use crate::error::{Error, RichError, Span}; use crate::impl_eq_hash; +#[cfg(not(feature = "fmt"))] use crate::lexer::Token; +#[cfg(feature = "fmt")] +use crate::lexer::{FmtToken, FmtTokens, Token}; use crate::num::NonZeroPow2Usize; use crate::pattern::Pattern; use crate::source::SourceFile; @@ -40,6 +43,40 @@ pub struct Program { span: Span, } +/// Source-aware parse result used by formatters. +/// +/// The compiler keeps using [`Program`] directly. Formatters need the same +/// semantic tree plus the lossless trivia stream that the grammar intentionally +/// does not consume. +#[cfg(feature = "fmt")] +#[derive(Clone, Debug)] +pub struct ParsedSource<'src> { + program: Program, + tokens: FmtTokens<'src>, + prefix: Span, +} + +#[cfg(feature = "fmt")] +impl<'src> ParsedSource<'src> { + /// Access the semantic program used for formatting. + pub fn program(&self) -> &Program { + &self.program + } + + /// Access the lossless token stream in source order. + pub fn tokens(&self) -> &FmtTokens<'src> { + &self.tokens + } + + /// Access the source prefix skipped by the version-directive prescan. + /// + /// A formatter should preserve this range verbatim until the directive gains + /// its own formatting grammar. + pub fn prefix(&self) -> Span { + self.prefix + } +} + impl Program { // Need for driver usage pub(crate) fn new(items: &[Item], span: Span) -> Self { @@ -53,6 +90,72 @@ impl Program { pub fn items(&self) -> &[Item] { &self.items } + + /// Parse source for formatting while retaining all comments and whitespace. + #[cfg(feature = "fmt")] + pub fn parse_for_formatting<'src>( + file_id: usize, + input: &'src str, + unstable_features: &UnstableFeatures, + ) -> Result, Vec> { + let source = SourceFile::anonymous(Arc::from(input)); + let start = match SimcDirective::prescan(input, file_id) { + Ok(start) => start, + Err((error, span)) => { + return Err(vec![RichError::new(error, span).with_source(source)]) + } + }; + + let (tokens, mut errors) = crate::lexer::lex_lossless(file_id, input, start); + let Some(tokens) = tokens else { + return Err(errors + .into_iter() + .map(|error| error.with_source(source.clone())) + .collect()); + }; + + let semantic_tokens = tokens + .iter() + .filter_map(|(token, span)| match token { + FmtToken::Token(token) => Some((token.clone(), *span)), + FmtToken::Trivia(_) => None, + }) + .collect::>(); + + let eoi = Span::eof(file_id, input.len()); + let (program, parse_errors) = Program::parser() + .parse( + semantic_tokens + .as_slice() + .map(eoi, |(token, span)| (token, span)), + ) + .into_output_errors(); + errors.extend(parse_errors); + + let Some(program) = program else { + return Err(errors + .into_iter() + .map(|error| error.with_source(source.clone())) + .collect()); + }; + + let mut feature_errors = ErrorCollector::new(); + unstable_features.check_program(&program, &source, &mut feature_errors); + errors.extend(feature_errors.get().iter().cloned()); + + if errors.is_empty() { + Ok(ParsedSource { + program, + tokens, + prefix: Span::new(file_id, 0..start), + }) + } else { + Err(errors + .into_iter() + .map(|error| error.with_source(source.clone())) + .collect()) + } + } } impl_eq_hash!(Program; items); @@ -78,6 +181,21 @@ pub enum Item { Ignored, } +impl Item { + /// Access the source span when this item was parsed successfully. + /// + /// Error-recovery placeholders have no source node to decorate. + pub fn span(&self) -> Option<&Span> { + match self { + Self::TypeAlias(alias) => Some(alias.span()), + Self::Function(function) => Some(function.span()), + Self::Use(use_decl) => Some(use_decl.span()), + Self::Module(module) => Some(module.span()), + Self::Ignored => None, + } + } +} + impl_require_feature!(Item { variants: TypeAlias(alias), @@ -274,11 +392,12 @@ impl_eq_hash!(Function; visibility, name, params, ret, body); impl_require_feature!(Function { recurse: params, ret, body; }); /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct FunctionParam { identifier: Identifier, ty: AliasedType, + span: Span, } impl FunctionParam { @@ -291,8 +410,15 @@ impl FunctionParam { pub fn ty(&self) -> &AliasedType { &self.ty } + + /// Access the source span of the complete parameter. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + impl_require_feature!(FunctionParam { recurse: ty; }); /// A statement is a component of a block expression. @@ -304,6 +430,19 @@ pub enum Statement { Expression(Expression), } +impl Statement { + /// Access the span of the statement contents. + /// + /// The terminating semicolon is deliberately not part of this span: it is a + /// separator owned by the containing block's layout. + pub fn span(&self) -> &Span { + match self { + Self::Assignment(assignment) => assignment.span(), + Self::Expression(expression) => expression.span(), + } + } +} + impl_require_feature!(Statement { variants: Assignment(assignment), @@ -650,10 +789,12 @@ impl_eq_hash!(Match; scrutinee, left, right); impl_require_feature!(Match {recurse: scrutinee, left, right; }); /// Arm of a match expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, + comma_span: Option, } impl MatchArm { @@ -666,8 +807,20 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } + + /// Access the span of the arm's trailing comma, when present. + pub fn comma_span(&self) -> Option<&Span> { + self.comma_span.as_ref() + } } +impl_eq_hash!(MatchArm; pattern, expression); + impl_require_feature!(MatchArm {recurse: pattern, expression; }); /// Pattern of a match arm. @@ -1673,7 +1826,11 @@ impl ChumskyParse for FunctionParam { identifier .then_ignore(just(Token::Colon)) .then(ty) - .map(|(identifier, ty)| Self { identifier, ty }) + .map_with(|(identifier, ty), e| Self { + identifier, + ty, + span: e.span(), + }) } } @@ -2136,7 +2293,7 @@ impl MatchArm { MatchPattern::parser() .then_ignore(just(Token::FatArrow)) .then(expr.map(Arc::new)) - .then(just(Token::Comma).or_not()) + .then(just(Token::Comma).to_span().or_not()) .validate(|((pattern, expression), comma), e, emitter| { let is_block = matches!(expression.as_ref().inner, ExpressionInner::Block(_, _)); @@ -2150,10 +2307,13 @@ impl MatchArm { ); } - Self { - pattern, - expression, - } + (pattern, expression, comma) + }) + .map_with(|(pattern, expression, comma_span), e| Self { + pattern, + expression, + span: e.span(), + comma_span, }) } } @@ -2233,6 +2393,8 @@ impl Match { let match_arm_fallback = MatchArm { expression: Arc::new(Expression::empty(Span::DUMMY)), pattern: MatchPattern::False, + span: Span::DUMMY, + comma_span: None, }; let (left, right) = ( @@ -2308,6 +2470,12 @@ impl AsRef for Function { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Assignment { fn as_ref(&self) -> &Span { &self.span @@ -2344,6 +2512,12 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for UseDecl { fn as_ref(&self) -> &Span { &self.span @@ -2647,10 +2821,14 @@ impl crate::ArbitraryRec for Match { left: MatchArm { pattern: pat_l, expression: expr_l, + span: Span::DUMMY, + comma_span: None, }, right: MatchArm { pattern: pat_r, expression: expr_r, + span: Span::DUMMY, + comma_span: None, }, span: Span::DUMMY, }) @@ -2826,4 +3004,75 @@ fn main() { assert_eq!(program.to_string(), format!("{input}\n")); } } + + #[test] + fn parsed_nodes_keep_source_spans() { + let input = "fn choose(x: u8) -> u8 { match x { false => 0, true => 1, } }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + assert_eq!( + program.items()[0].span().unwrap().to_slice(input), + Some(input) + ); + assert_eq!(function.params()[0].span().to_slice(input), Some("x: u8")); + + let ExpressionInner::Block(_, Some(final_expr)) = function.body().inner() else { + panic!("expected a block with a final expression"); + }; + let ExpressionInner::Single(single) = final_expr.inner() else { + panic!("expected a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a match expression"); + }; + + assert_eq!(match_.left().span().to_slice(input), Some("false => 0,")); + assert_eq!( + match_.left().comma_span().unwrap().to_slice(input), + Some(",") + ); + assert_eq!(match_.right().span().to_slice(input), Some("true => 1,")); + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + #[cfg(feature = "fmt")] + #[test] + fn formatting_parse_keeps_trivia_and_directive_prefix() { + let input = "// header\nsimc \"*\";\n// body\nfn main() {}"; + let parsed = Program::parse_for_formatting(0, input, &UnstableFeatures::none()) + .expect("formatting parse succeeds"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\nsimc \"*\";") + ); + assert!(parsed.tokens().iter().any(|(token, span)| { + matches!( + token, + FmtToken::Trivia(crate::lexer::TriviaKind::LineComment) + ) && span.to_slice(input) == Some("// body") + })); + assert_eq!(parsed.program().items().len(), 1); + } } From 71b4d7fd6c894a1f900db1ef200dcc2b04306b0c Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Thu, 16 Jul 2026 19:54:20 +0300 Subject: [PATCH 3/4] minor changes --- src/lexer.rs | 549 ++++++++++++++++++++++++++----------------------- src/parse.rs | 149 ++++++++++---- src/source.rs | 6 + src/version.rs | 1 + 4 files changed, 403 insertions(+), 302 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index f1f82903..e13eec3f 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -441,314 +441,347 @@ mod tests { use super::*; - fn lex<'src>( - input: &'src str, - ) -> (Option>>, Vec>) { - let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| { - vec.into_iter() - .map(|(tok, _)| tok.clone()) - .collect::>() - }); - (tokens, errors) - } - - #[cfg(feature = "fmt")] - fn lex_lossless<'src>( - input: &'src str, - ) -> ( - Option>>, - Vec>, - ) { - let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| { - vec.into_iter() - .map(|(fmt_tok, _)| fmt_tok) - .collect::>() - }); - (tokens, errors) - } + mod lexer { + use super::*; + + fn lex<'src>( + input: &'src str, + ) -> (Option>>, Vec>) { + let (tokens, errors) = lexer().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + #[test] + fn test_block_comment_simple() { + let input = "/* hello world */"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![]), + "Should produce a single block comment token" + ); + } - #[test] - fn test_block_comment_simple() { - let input = "/* hello world */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![]), - "Should produce a single block comment token" - ); - } + #[test] + fn test_block_comment_nested() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex(input); - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_simple_fmt() { - let input = "/* hello world */"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]), - "Should produce a single block comment token" - ); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - #[cfg(feature = "fmt")] - #[test] - fn lossless_lexer_keeps_each_newline_kind_with_its_span() { - let input = "first\r\nsecond\rthird\nfourth"; - let (tokens, errors) = super::lex_lossless(0, input, 0); - - assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); - let tokens = tokens.expect("lossless lexing succeeds"); - let newlines: Vec<_> = tokens - .iter() - .filter(|(token, _)| matches!(token, FmtToken::Trivia(TriviaKind::Newline))) - .map(|(_, span)| span.to_slice(input)) - .collect(); - - assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); - } + #[test] + fn test_block_comment_deeply_nested() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_nested() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![])); - } + #[test] + fn test_block_comment_multiline() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex(input); - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_nested_fmt() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_deeply_nested() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_unclosed() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex(input); - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_deeply_nested_fmt() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - #[test] - fn test_block_comment_multiline() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex(input); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![])); - } + #[test] + fn test_block_comment_partial_nesting_unclosed() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex(input); - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_multiline_fmt() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex_lossless(input); - - assert!(errors.is_empty()); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_unclosed() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex(input); + #[test] + fn test_block_comment_double_unclosed() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex(input); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + assert_eq!(errors.len(), 2); - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - assert_eq!(tokens, Some(vec![])); - } + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_unclosed_fmt() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex_lossless(input); - - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); - - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + assert_eq!(tokens, Some(vec![])); + } - #[test] - fn test_block_comment_partial_nesting_unclosed() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex(input); + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![])); - } + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_partial_nesting_unclosed_fmt() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex_lossless(input); - - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + #[test] + fn simc_is_reserved() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, Token::Simc)), + "the sentinel must not reach the token stream for {src:?}" + ); + } - #[test] - fn test_block_comment_double_unclosed() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex(input); + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); + } - assert_eq!(errors.len(), 2); + #[test] + fn lexer_test() { + use chumsky::prelude::*; - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert_eq!(tokens, Some(vec![])); + assert!(lex_errs.is_empty()); + } } #[cfg(feature = "fmt")] - #[test] - fn test_block_comment_double_unclosed_fmt() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex_lossless(input); + mod fmt_lexer { + use super::*; + + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } - assert_eq!(errors.len(), 2); + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]), + "Should produce a single block comment token" + ); + } - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); - assert_eq!( - tokens, - Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) - ); - } + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| matches!(token, FmtToken::Trivia(TriviaKind::Newline))) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } - #[test] - fn simc_is_reserved() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) ); - assert!( - tokens - .expect("recovery keeps the stream") - .iter() - .all(|(tok, _)| !matches!(tok, Token::Simc)), - "the sentinel must not reach the token stream for {src:?}" + } + + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) ); } - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); - assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); - } + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); - #[cfg(feature = "fmt")] - #[test] - fn simc_is_reserved_fmt() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex_lossless(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } + + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) ); + } + + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); - assert!( - tokens - .expect("recovery keeps the stream") - .iter() - .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), - "the sentinel must not reach the token stream for {src:?}" + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) ); } - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex_lossless("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); - assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); - } + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); - #[test] - fn lexer_test() { - use chumsky::prelude::*; + assert_eq!(errors.len(), 2); - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert!(lex_errs.is_empty()); - } + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } - #[cfg(feature = "fmt")] - #[test] - fn lossless_lexer_test() { - use chumsky::prelude::*; + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Whitespace,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + ]) + ); + } - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } - let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } - assert!(lex_errs.is_empty()); + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); + + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } } } diff --git a/src/parse.rs b/src/parse.rs index 86e29f6f..b0863c6f 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -23,6 +23,7 @@ use crate::error::{Error, RichError, Span}; use crate::impl_eq_hash; #[cfg(not(feature = "fmt"))] use crate::lexer::Token; +use crate::lexer::Tokens; #[cfg(feature = "fmt")] use crate::lexer::{FmtToken, FmtTokens, Token}; use crate::num::NonZeroPow2Usize; @@ -33,7 +34,7 @@ use crate::str::{ SymbolName, WitnessName, }; use crate::types::{AliasedType, BuiltinAlias, TypeConstructible, UIntType}; -use crate::unstable::{impl_require_feature, UnstableFeature, UnstableFeatures}; +use crate::unstable::{impl_require_feature, RequireFeature, UnstableFeature, UnstableFeatures}; use crate::version::SimcDirective; /// A program is a sequence of items. @@ -51,6 +52,7 @@ pub struct Program { #[cfg(feature = "fmt")] #[derive(Clone, Debug)] pub struct ParsedSource<'src> { + simc_directive: SimcDirective, program: Program, tokens: FmtTokens<'src>, prefix: Span, @@ -95,11 +97,13 @@ impl Program { #[cfg(feature = "fmt")] pub fn parse_for_formatting<'src>( file_id: usize, - input: &'src str, + source: impl Into, unstable_features: &UnstableFeatures, ) -> Result, Vec> { - let source = SourceFile::anonymous(Arc::from(input)); - let start = match SimcDirective::prescan(input, file_id) { + let source = source.into(); + let input = source.content(); + + let start = match SimcDirective::prescan(&input, file_id) { Ok(start) => start, Err((error, span)) => { return Err(vec![RichError::new(error, span).with_source(source)]) @@ -122,7 +126,7 @@ impl Program { }) .collect::>(); - let eoi = Span::eof(file_id, input.len()); + let eoi = Span::eof(file_id, input.as_ref().len()); let (program, parse_errors) = Program::parser() .parse( semantic_tokens @@ -1327,6 +1331,10 @@ impl_parse_wrapped_string!(WitnessName, "witness name"); impl_parse_wrapped_string!(AliasName, "alias name"); impl_parse_wrapped_string!(ModuleName, "module name"); +pub trait AstNode: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + +impl AstNode for T where T: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + /// Copy of [`FromStr`] that internally uses the `chumsky` parser. pub trait ParseFromStr: Sized { /// Parse a value from the string `s`. @@ -1345,6 +1353,71 @@ pub trait ParseFromStrWithErrors: Sized { unstable_features: &UnstableFeatures, handler: &mut ErrorCollector, ) -> Option; + + /// Handle the `simc` directive before lexing: an incompatible or malformed + /// directive is reported as the only diagnostic (the rest is noise), and + /// lexing starts right after a valid one, so the lexer and grammar never + /// see it. + fn directive_prescan( + src_file: &SourceFile, + file_id: usize, + err_collector: &mut ErrorCollector, + ) -> Option { + match SimcDirective::prescan(&src_file.content(), file_id) { + Ok(start) => Some(start), + Err((err, span)) => { + err_collector.push(RichError::new(err, span).with_source(src_file.clone())); + None + } + } + } + + fn lex( + file_id: usize, + src_file: &SourceFile, + start: usize, + ) -> (Option>, Vec) { + crate::lexer::lex(file_id, src_file.as_ref(), start) + } + + fn is_lex_ok( + src: &SourceFile, + mut lex_errs: Vec, + err_sink: &mut ErrorCollector, + ) -> bool { + // A stray `simc` makes every other diagnostic noise — its `"";` remnant + // does not lex — so the reserved-keyword errors are reported alone. + if lex_errs + .iter() + .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) + { + lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); + let has_errs = lex_errs.is_empty(); + + err_sink.extend(src.clone(), lex_errs); + has_errs + } else { + lex_errs.is_empty() + } + } + + fn parse_ast( + file_id: usize, + src: &SourceFile, + tokens: Tokens<'_>, + err_sink: &mut ErrorCollector, + ) -> (Option, bool); + + fn post_check( + src_file: &SourceFile, + unstable_features: &UnstableFeatures, + program: &Option, + handler: &mut ErrorCollector, + ) { + if let Some(ast) = program { + unstable_features.check_program(program, &src_file, handler); + } + } } /// Trait for generating parsers of themselves. @@ -1397,64 +1470,52 @@ impl ParseFromStr for A { } } -impl ParseFromStrWithErrors - for A -{ +impl ParseFromStrWithErrors for A { fn parse_from_str_with_errors( file_id: usize, source: impl Into, unstable_features: &UnstableFeatures, - handler: &mut ErrorCollector, + err_sink: &mut ErrorCollector, ) -> Option { let source: SourceFile = source.into(); - let content = source.content(); - // Handle the `simc` directive before lexing: an incompatible or malformed - // directive is reported as the only diagnostic (the rest is noise), and - // lexing starts right after a valid one, so the lexer and grammar never - // see it. - let start = match SimcDirective::prescan(&content, file_id) { - Ok(start) => start, - Err((err, span)) => { - handler.push(RichError::new(err, span).with_source(source)); - return None; - } - }; + let start = Self::directive_prescan(&source, file_id, err_sink).unwrap_or_default(); - let (tokens, mut lex_errs) = crate::lexer::lex(file_id, &content, start); - // A stray `simc` makes every other diagnostic noise — its `"";` remnant - // does not lex — so the reserved-keyword errors are reported alone. - if lex_errs - .iter() - .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) - { - lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - handler.extend(source, lex_errs); - return None; - } - let lex_ok = lex_errs.is_empty(); - handler.extend(source.clone(), lex_errs); + let (tokens, mut lex_errs) = Self::lex(file_id, &source, start); + + let lex_ok = Self::is_lex_ok(&source, lex_errs, err_sink); let tokens = tokens?; - let eoi = Span::eof(file_id, content.len()); - let (ast, parse_errs) = A::parser() - .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) - .into_output_errors(); - let parse_ok = parse_errs.is_empty(); - handler.extend(source.clone(), parse_errs); + let (ast, parse_status) = Self::parse_ast(file_id, &source, tokens, err_sink); - if let (Some(ast), true) = (&ast, lex_ok && parse_ok) { - unstable_features.check_program(ast, &source, handler); + if parse_status && lex_ok { + let _ = Self::post_check(&source, unstable_features, &ast, err_sink); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module // is not handling poisoned tree right now, we don't return parsed result - if handler.has_errors() { + if err_sink.has_errors() { None } else { ast } } + + fn parse_ast( + file_id: usize, + src: &SourceFile, + tokens: Tokens<'_>, + err_sink: &mut ErrorCollector, + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.as_ref().len()); + let (ast, parse_errs) = Self::parser() + .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) + .into_output_errors(); + + let parse_ok = parse_errs.is_empty(); + err_sink.extend(src.clone(), parse_errs); + (ast, parse_ok) + } } /// Parse a token, and, if not found, place itself in place of missing one. @@ -3000,7 +3061,7 @@ fn main() { "use lib::A::foo as bar;", "use lib::A::{foo, bar as baz};", ] { - let program = parse::Program::parse_from_str(input).expect("parsing works"); + let program = parse::Program::parse_from_str(input).expect("program parses"); assert_eq!(program.to_string(), format!("{input}\n")); } } diff --git a/src/source.rs b/src/source.rs index 49a70bc7..4826b4c6 100644 --- a/src/source.rs +++ b/src/source.rs @@ -22,6 +22,12 @@ impl From for SourceFile { } } +impl AsRef for SourceFile{ + fn as_ref(&self) -> &str { + self.content.as_ref() + } +} + impl SourceFile { /// Creates a standard `SourceFile` from a file path and its content. pub fn new(name: &Path, content: Arc) -> Self { diff --git a/src/version.rs b/src/version.rs index 7dd7bdf6..8ae2178c 100644 --- a/src/version.rs +++ b/src/version.rs @@ -41,6 +41,7 @@ enum DirectiveScan<'a> { } /// Operations on the `simc "";` compiler-version directive. +#[derive(Clone, Debug)] pub struct SimcDirective; impl SimcDirective { From e71710cc953fcd30e27ab7b33e40bc575fee120c Mon Sep 17 00:00:00 2001 From: Illia Kripaka Date: Thu, 16 Jul 2026 20:21:13 +0300 Subject: [PATCH 4/4] refactor parsing, reuse functions --- src/parse.rs | 609 +++++++++++++++++++++++++-------------------------- 1 file changed, 304 insertions(+), 305 deletions(-) diff --git a/src/parse.rs b/src/parse.rs index b0863c6f..eea6f88f 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -52,7 +52,7 @@ pub struct Program { #[cfg(feature = "fmt")] #[derive(Clone, Debug)] pub struct ParsedSource<'src> { - simc_directive: SimcDirective, + simc_directive: String, program: Program, tokens: FmtTokens<'src>, prefix: Span, @@ -97,25 +97,26 @@ impl Program { #[cfg(feature = "fmt")] pub fn parse_for_formatting<'src>( file_id: usize, - source: impl Into, + source: &'src SourceFile, unstable_features: &UnstableFeatures, ) -> Result, Vec> { - let source = source.into(); - let input = source.content(); + let input = source.as_ref(); + let mut errors = ErrorCollector::new(); - let start = match SimcDirective::prescan(&input, file_id) { - Ok(start) => start, - Err((error, span)) => { - return Err(vec![RichError::new(error, span).with_source(source)]) - } + let Some(start) = pipeline::directive_prescan(source, file_id, &mut errors) else { + return Err(errors.get().to_vec()); }; + let simc_directive = source.content()[0..start].to_string(); - let (tokens, mut errors) = crate::lexer::lex_lossless(file_id, input, start); + let (tokens, lex_errors) = crate::lexer::lex_lossless(file_id, input, start); + let lex_ok = match pipeline::is_lex_ok(source, lex_errors, &mut errors) { + None => { + return Err(errors.get().to_vec()); + } + Some(x) => x, + }; let Some(tokens) = tokens else { - return Err(errors - .into_iter() - .map(|error| error.with_source(source.clone())) - .collect()); + return Err(errors.get().to_vec()); }; let semantic_tokens = tokens @@ -126,38 +127,29 @@ impl Program { }) .collect::>(); - let eoi = Span::eof(file_id, input.as_ref().len()); - let (program, parse_errors) = Program::parser() - .parse( - semantic_tokens - .as_slice() - .map(eoi, |(token, span)| (token, span)), - ) - .into_output_errors(); - errors.extend(parse_errors); + let (program, parse_ok) = + pipeline::parse_ast(file_id, source, semantic_tokens, &mut errors); - let Some(program) = program else { - return Err(errors - .into_iter() - .map(|error| error.with_source(source.clone())) - .collect()); - }; + if parse_ok && lex_ok { + pipeline::post_check(source, unstable_features, &program, &mut errors); + } - let mut feature_errors = ErrorCollector::new(); - unstable_features.check_program(&program, &source, &mut feature_errors); - errors.extend(feature_errors.get().iter().cloned()); + if errors.has_errors() { + Err(errors.get().to_vec()) + } else { + let Some(program) = program else { + return Err(vec![RichError::parsing_error( + "Empty AST without an error.", + ) + .with_source(source.clone())]); + }; - if errors.is_empty() { Ok(ParsedSource { + simc_directive, program, tokens, prefix: Span::new(file_id, 0..start), }) - } else { - Err(errors - .into_iter() - .map(|error| error.with_source(source.clone())) - .collect()) } } } @@ -1353,12 +1345,66 @@ pub trait ParseFromStrWithErrors: Sized { unstable_features: &UnstableFeatures, handler: &mut ErrorCollector, ) -> Option; +} + +/// Trait for generating parsers of themselves. +/// +/// Replacement for previous `PestParse` trait. +pub trait ChumskyParse: Sized { + fn parser<'tokens, 'src: 'tokens, I>() -> impl Parser<'tokens, I, Self, ParseError<'src>> + Clone + where + I: ValueInput<'tokens, Token = Token<'src>, Span = Span>; +} + +type ParseError<'src> = extra::Err; + +/// This implementation only returns first encountered error. +impl ParseFromStr for A { + fn parse_from_str(s: &str) -> Result { + let (tokens, mut lex_errs) = crate::lexer::lex(MAIN_MODULE, s, 0); + + // The `simc` directive is source-file syntax, so fragments have no prescan + // and `simc` lexes as a reserved keyword. Its `"";` remnant does not + // lex either, so the first reserved-keyword error is reported alone. + if let Some(err) = lex_errs + .iter() + .find(|e| matches!(e.error(), Error::ReservedSimcKeyword)) + { + return Err(err.clone()); + } + + let Some(tokens) = tokens else { + return Err(lex_errs.pop().unwrap_or(RichError::parsing_error( + "Empty token stream without an error.", + ))); + }; + + let (ast, parse_errs) = A::parser() + .map_with(|parsed, _| parsed) + .parse( + tokens + .as_slice() + .map(Span::eof(MAIN_MODULE, s.len()), |(t, s)| (t, s)), + ) + .into_output_errors(); + + if parse_errs.is_empty() { + Ok(ast.ok_or(RichError::parsing_error("Empty AST without an error."))?) + } else { + let err = parse_errs.first().unwrap().clone(); + Err(err) + } + } +} + +mod pipeline { + use super::*; /// Handle the `simc` directive before lexing: an incompatible or malformed /// directive is reported as the only diagnostic (the rest is noise), and /// lexing starts right after a valid one, so the lexer and grammar never /// see it. - fn directive_prescan( + pub fn directive_prescan( src_file: &SourceFile, file_id: usize, err_collector: &mut ErrorCollector, @@ -1372,7 +1418,7 @@ pub trait ParseFromStrWithErrors: Sized { } } - fn lex( + pub fn lex( file_id: usize, src_file: &SourceFile, start: usize, @@ -1380,11 +1426,11 @@ pub trait ParseFromStrWithErrors: Sized { crate::lexer::lex(file_id, src_file.as_ref(), start) } - fn is_lex_ok( + pub fn is_lex_ok( src: &SourceFile, mut lex_errs: Vec, err_sink: &mut ErrorCollector, - ) -> bool { + ) -> Option { // A stray `simc` makes every other diagnostic noise — its `"";` remnant // does not lex — so the reserved-keyword errors are reported alone. if lex_errs @@ -1392,80 +1438,39 @@ pub trait ParseFromStrWithErrors: Sized { .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) { lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - let has_errs = lex_errs.is_empty(); - err_sink.extend(src.clone(), lex_errs); - has_errs - } else { - lex_errs.is_empty() + return None; } + + let lex_ok = lex_errs.is_empty(); + err_sink.extend(src.clone(), lex_errs); + Some(lex_ok) } - fn parse_ast( + pub fn parse_ast( file_id: usize, src: &SourceFile, tokens: Tokens<'_>, err_sink: &mut ErrorCollector, - ) -> (Option, bool); + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.as_ref().len()); + let (ast, parse_errs) = T::parser() + .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) + .into_output_errors(); - fn post_check( + let parse_ok = parse_errs.is_empty(); + err_sink.extend(src.clone(), parse_errs); + (ast, parse_ok) + } + + pub fn post_check( src_file: &SourceFile, unstable_features: &UnstableFeatures, - program: &Option, + program: &Option, handler: &mut ErrorCollector, ) { if let Some(ast) = program { - unstable_features.check_program(program, &src_file, handler); - } - } -} - -/// Trait for generating parsers of themselves. -/// -/// Replacement for previous `PestParse` trait. -pub trait ChumskyParse: Sized { - fn parser<'tokens, 'src: 'tokens, I>() -> impl Parser<'tokens, I, Self, ParseError<'src>> + Clone - where - I: ValueInput<'tokens, Token = Token<'src>, Span = Span>; -} - -type ParseError<'src> = extra::Err; - -/// This implementation only returns first encountered error. -impl ParseFromStr for A { - fn parse_from_str(s: &str) -> Result { - let (tokens, mut lex_errs) = crate::lexer::lex(MAIN_MODULE, s, 0); - - // The `simc` directive is source-file syntax, so fragments have no prescan - // and `simc` lexes as a reserved keyword. Its `"";` remnant does not - // lex either, so the first reserved-keyword error is reported alone. - if let Some(err) = lex_errs - .iter() - .find(|e| matches!(e.error(), Error::ReservedSimcKeyword)) - { - return Err(err.clone()); - } - - let Some(tokens) = tokens else { - return Err(lex_errs.pop().unwrap_or(RichError::parsing_error( - "Empty token stream without an error.", - ))); - }; - - let (ast, parse_errs) = A::parser() - .map_with(|parsed, _| parsed) - .parse( - tokens - .as_slice() - .map(Span::eof(MAIN_MODULE, s.len()), |(t, s)| (t, s)), - ) - .into_output_errors(); - - if parse_errs.is_empty() { - Ok(ast.ok_or(RichError::parsing_error("Empty AST without an error."))?) - } else { - let err = parse_errs.first().unwrap().clone(); - Err(err) + unstable_features.check_program(ast, &src_file, handler); } } } @@ -1479,17 +1484,17 @@ impl ParseFromStrWithErrors for A { ) -> Option { let source: SourceFile = source.into(); - let start = Self::directive_prescan(&source, file_id, err_sink).unwrap_or_default(); + let start = pipeline::directive_prescan(&source, file_id, err_sink).unwrap_or_default(); - let (tokens, mut lex_errs) = Self::lex(file_id, &source, start); + let (tokens, lex_errs) = pipeline::lex(file_id, &source, start); - let lex_ok = Self::is_lex_ok(&source, lex_errs, err_sink); + let lex_ok = pipeline::is_lex_ok(&source, lex_errs, err_sink)?; let tokens = tokens?; - let (ast, parse_status) = Self::parse_ast(file_id, &source, tokens, err_sink); + let (ast, parse_status) = pipeline::parse_ast(file_id, &source, tokens, err_sink); if parse_status && lex_ok { - let _ = Self::post_check(&source, unstable_features, &ast, err_sink); + let _ = pipeline::post_check(&source, unstable_features, &ast, err_sink); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module @@ -1500,22 +1505,6 @@ impl ParseFromStrWithErrors for A { ast } } - - fn parse_ast( - file_id: usize, - src: &SourceFile, - tokens: Tokens<'_>, - err_sink: &mut ErrorCollector, - ) -> (Option, bool) { - let eoi = Span::eof(file_id, src.as_ref().len()); - let (ast, parse_errs) = Self::parser() - .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) - .into_output_errors(); - - let parse_ok = parse_errs.is_empty(); - err_sink.extend(src.clone(), parse_errs); - (ast, parse_ok) - } } /// Parse a token, and, if not found, place itself in place of missing one. @@ -2914,98 +2903,101 @@ mod test { } } - #[test] - fn test_reject_redefined_builtin_type() { - let ty = TypeAlias::parse_from_str("type Ctx8 = u32") - .expect_err("Redefining built-in alias should be rejected"); + mod regular_parsing { + use super::*; - assert!(ty - .error() - .to_string() - .contains("Type alias `Ctx8` is already exists as built-in alias")); - } + #[test] + fn test_reject_redefined_builtin_type() { + let ty = TypeAlias::parse_from_str("type Ctx8 = u32") + .expect_err("Redefining built-in alias should be rejected"); - #[test] - fn test_fragment_rejects_version_directive() { - let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") - .expect_err("a version directive must not be accepted in a fragment"); + assert!(ty + .error() + .to_string() + .contains("Type alias `Ctx8` is already exists as built-in alias")); + } - assert!(matches!(err.error(), Error::ReservedSimcKeyword)); - } + #[test] + fn test_fragment_rejects_version_directive() { + let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") + .expect_err("a version directive must not be accepted in a fragment"); - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let source = SourceFile::anonymous(Arc::from(input)); - let mut error_handler = ErrorCollector::new(); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - source, - &UnstableFeatures::all(), - &mut error_handler, - ); + assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + } - assert!(parsed_program.is_none()); - assert!(ErrorCollector::to_string(&error_handler).contains("Expected '::', found ':'")); - } + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + let source = SourceFile::anonymous(Arc::from(input)); + let mut error_handler = ErrorCollector::new(); + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + source, + &UnstableFeatures::all(), + &mut error_handler, + ); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - let source = SourceFile::anonymous(Arc::from(input)); - let mut error_handler = ErrorCollector::new(); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - source, - &UnstableFeatures::all(), - &mut error_handler, - ); + assert!(parsed_program.is_none()); + assert!(ErrorCollector::to_string(&error_handler).contains("Expected '::', found ':'")); + } - assert!(parsed_program.is_none()); - assert!(ErrorCollector::to_string(&error_handler).contains("Expected ';', found '::'")); - } + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let source = SourceFile::anonymous(Arc::from(input)); + let mut error_handler = ErrorCollector::new(); + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + source, + &UnstableFeatures::all(), + &mut error_handler, + ); - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let source = SourceFile::anonymous(Arc::from(input)); - let mut handler = ErrorCollector::new(); - let program = - Program::parse_from_str_with_errors(MAIN_MODULE, source, features, &mut handler); - (program.is_none(), ErrorCollector::to_string(&handler)) - } + assert!(parsed_program.is_none()); + assert!(ErrorCollector::to_string(&error_handler).contains("Expected ';', found '::'")); + } - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let source = SourceFile::anonymous(Arc::from(input)); + let mut handler = ErrorCollector::new(); + let program = + Program::parse_from_str_with_errors(MAIN_MODULE, source, features, &mut handler); + (program.is_none(), ErrorCollector::to_string(&handler)) + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ just stabilized, delete this test):\n{input}" - ); - assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" - ); + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); - assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" - ); + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } } - } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -3018,122 +3010,129 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let program = parse::Program::parse_from_str(input).expect("program parses"); - assert_eq!(program.to_string(), format!("{input}\n")); + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let program = parse::Program::parse_from_str(input).expect("program parses"); + assert_eq!(program.to_string(), format!("{input}\n")); + } } - } - #[test] - fn parsed_nodes_keep_source_spans() { - let input = "fn choose(x: u8) -> u8 { match x { false => 0, true => 1, } }"; - let program = Program::parse_from_str(input).expect("program parses"); + #[test] + fn parsed_nodes_keep_source_spans() { + let input = "fn choose(x: u8) -> u8 { match x { false => 0, true => 1, } }"; + let program = Program::parse_from_str(input).expect("program parses"); - let Item::Function(function) = &program.items()[0] else { - panic!("expected a function item"); - }; - assert_eq!( - program.items()[0].span().unwrap().to_slice(input), - Some(input) - ); - assert_eq!(function.params()[0].span().to_slice(input), Some("x: u8")); + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + assert_eq!( + program.items()[0].span().unwrap().to_slice(input), + Some(input) + ); + assert_eq!(function.params()[0].span().to_slice(input), Some("x: u8")); - let ExpressionInner::Block(_, Some(final_expr)) = function.body().inner() else { - panic!("expected a block with a final expression"); - }; - let ExpressionInner::Single(single) = final_expr.inner() else { - panic!("expected a single expression"); - }; - let SingleExpressionInner::Match(match_) = single.inner() else { - panic!("expected a match expression"); - }; + let ExpressionInner::Block(_, Some(final_expr)) = function.body().inner() else { + panic!("expected a block with a final expression"); + }; + let ExpressionInner::Single(single) = final_expr.inner() else { + panic!("expected a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a match expression"); + }; - assert_eq!(match_.left().span().to_slice(input), Some("false => 0,")); - assert_eq!( - match_.left().comma_span().unwrap().to_slice(input), - Some(",") - ); - assert_eq!(match_.right().span().to_slice(input), Some("true => 1,")); - } + assert_eq!(match_.left().span().to_slice(input), Some("false => 0,")); + assert_eq!( + match_.left().comma_span().unwrap().to_slice(input), + Some(",") + ); + assert_eq!(match_.right().span().to_slice(input), Some("true => 1,")); + } - #[test] - fn statement_spans_exclude_terminating_semicolons() { - let input = "fn main() { let value: u8 = 0; value; }"; - let program = Program::parse_from_str(input).expect("program parses"); + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); - let Item::Function(function) = &program.items()[0] else { - panic!("expected a function item"); - }; - let ExpressionInner::Block(statements, None) = function.body().inner() else { - panic!("expected a block containing only statements"); - }; + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; - assert_eq!( - statements[0].span().to_slice(input), - Some("let value: u8 = 0") - ); - assert_eq!(statements[1].span().to_slice(input), Some("value")); + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } } #[cfg(feature = "fmt")] - #[test] - fn formatting_parse_keeps_trivia_and_directive_prefix() { - let input = "// header\nsimc \"*\";\n// body\nfn main() {}"; - let parsed = Program::parse_for_formatting(0, input, &UnstableFeatures::none()) - .expect("formatting parse succeeds"); - - assert_eq!( - parsed.prefix().to_slice(input), - Some("// header\nsimc \"*\";") - ); - assert!(parsed.tokens().iter().any(|(token, span)| { - matches!( - token, - FmtToken::Trivia(crate::lexer::TriviaKind::LineComment) - ) && span.to_slice(input) == Some("// body") - })); - assert_eq!(parsed.program().items().len(), 1); + mod fmt_parsing { + use super::*; + + #[test] + fn formatting_parse_keeps_trivia_and_directive_prefix() { + let input = "// header\nsimc \"*\";\n// body\nfn main() {}"; + let source = SourceFile::anonymous(Arc::from(input)); + let parsed = + Program::parse_for_formatting(0, &source, &UnstableFeatures::none()).unwrap(); + // .expect("formatting parse succeeds"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\nsimc \"*\";") + ); + assert!(parsed.tokens().iter().any(|(token, span)| { + matches!( + token, + FmtToken::Trivia(crate::lexer::TriviaKind::LineComment) + ) && span.to_slice(input) == Some("// body") + })); + assert_eq!(parsed.program().items().len(), 1); + } } }