diff --git a/Cargo.toml b/Cargo.toml index 61fe2642..9abba208 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ default = [ "serde" ] serde = ["dep:serde", "dep:serde_json"] docs = [] external-jets = [] +fmt = [] [dependencies] base64 = "0.21.2" diff --git a/src/lexer.rs b/src/lexer.rs index fb6e8616..e13eec3f 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,7 +11,11 @@ use crate::version::SIMC_STR; pub type Spanned = (T, SimpleSpan); pub type Tokens<'src> = Vec<(Token<'src>, crate::error::Span)>; +#[cfg(feature = "fmt")] +pub type FmtTokens<'src> = Vec<(FmtToken<'src>, crate::error::Span)>; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Token<'src> { // Keywords Pub, @@ -66,12 +70,22 @@ pub enum Token<'src> { // Built-in functions Macro(&'src str), +} - // Comments and block comments - // - // We would discard them for the compiler, but they are needed, for example, for the formatter. - Comment, +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum TriviaKind { + LineComment, BlockComment, + Newline, + Whitespace, +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum FmtToken<'src> { + Token(Token<'src>), + Trivia(TriviaKind), } impl<'src> fmt::Display for Token<'src> { @@ -118,9 +132,23 @@ impl<'src> fmt::Display for Token<'src> { Token::Param(s) => write!(f, "param::{}", s), Token::Bool(b) => write!(f, "{}", b), + } + } +} - Token::Comment => write!(f, "comment"), - Token::BlockComment => write!(f, "block_comment"), +#[cfg(feature = "fmt")] +impl<'src> fmt::Display for FmtToken<'src> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FmtToken::Token(t) => { + write!(f, "{}", t) + } + FmtToken::Trivia(t) => match t { + TriviaKind::LineComment => write!(f, "comment"), + TriviaKind::BlockComment => write!(f, "block_comment"), + TriviaKind::Newline => writeln!(f), + TriviaKind::Whitespace => write!(f, "< >"), + }, } } } @@ -128,11 +156,42 @@ impl<'src> fmt::Display for Token<'src> { /// Recognizer for a `// ...` line comment. fn line_comment<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + let newline = newline_choice(); + just("//") - .then(any().and_is(just('\n').not()).repeated()) + .ignore_then(any().and_is(newline.not()).repeated()) .ignored() } +/// Recognizer for different kinds of newline (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +/// Returns the exact string slice that was matched. +fn newline_choice<'src>( +) -> impl Parser<'src, &'src str, &'src str, extra::Err>> + Clone { + choice((just("\r\n"), just("\n"), just("\r"))) +} + +/// Recognizer for different kinds of newline (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +fn newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + newline_choice().ignored() +} + +/// Recognizer for whitespace. +fn whitespace<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace() && *c != '\n' && *c != '\r') + .repeated() + .at_least(1) + .ignored() +} + +/// Recognizer for whitespace or a newline. +fn whitespace_or_newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any().filter(|c: &char| c.is_whitespace()).ignored() +} + /// Recognizer for a (possibly nested) `/* ... */` block comment; an unterminated /// comment is reported and swallows the rest of the input. fn block_comment<'src>( @@ -155,18 +214,13 @@ fn block_comment<'src>( /// syntax. pub(crate) fn trivia<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> { - choice(( - line_comment(), - block_comment(), - any().filter(|c: &char| c.is_whitespace()).ignored(), - )) - .repeated() - .ignored() + choice((line_comment(), block_comment(), whitespace_or_newline())) + .repeated() + .ignored() } -pub fn lexer<'src>( -) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> -{ +fn to_token<'src>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { let digits_with_underscore = |radix: u32| { any() .filter(move |c: &char| c.is_digit(radix)) @@ -240,25 +294,32 @@ pub fn lexer<'src>( just(">").to(Token::RAngle), )); - let comment = line_comment().to(Token::Comment); - let block_comment = block_comment().to(Token::BlockComment); - let token = choice(( - comment, - block_comment, - jet, - witness, - param, - macros, - keyword, - hex, - bin, - num, - op, - )); + choice((jet, witness, param, macros, keyword, hex, bin, num, op)) +} - token +pub fn lexer<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token() .map_with(|tok, e| (tok, e.span())) - .padded() + .recover_with(skip_then_retry_until(any().ignored(), end())); + + trivia().ignore_then(token.then_ignore(trivia()).repeated().collect()) +} + +#[cfg(feature = "fmt")] +pub fn lexer_lossless<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + let token = to_token().map(FmtToken::Token); + + let newline = newline().to(FmtToken::Trivia(TriviaKind::Newline)); + let whitespace = whitespace().to(FmtToken::Trivia(TriviaKind::Whitespace)); + let line_comment = line_comment().to(FmtToken::Trivia(TriviaKind::LineComment)); + let block_comment = block_comment().to(FmtToken::Trivia(TriviaKind::BlockComment)); + + choice((line_comment, block_comment, newline, whitespace, token)) + .map_with(|lexeme, e| (lexeme, e.span())) .recover_with(skip_then_retry_until(any().ignored(), end())) .repeated() .collect() @@ -268,43 +329,99 @@ pub fn lexer<'src>( /// offset `start` — the end of the version directive per `SimcDirective::prescan`, /// or `0`. Spans are reported relative to the full input. /// -/// All comments in the input code are discarded. +/// All comments, newlines, and spaces in the input code are discarded. pub fn lex( file_id: usize, input: &str, start: usize, ) -> (Option>, Vec) { let (tokens, lex_errors) = lexer().parse(&input[start..]).into_output_errors(); - let shift = |span: SimpleSpan| Span::new(file_id, span.start + start..span.end + start); + let shift = |span: SimpleSpan| simple_span_to_span(span, file_id, start); // The reserved-keyword errors come first: a stray directive also produces // follow-up errors for its `"";` remnant, and the sentinel is their cause. let mut errors: Vec = Vec::new(); let tokens = tokens.map(|vec| { vec.into_iter() - .filter_map(|(tok, span)| match tok { - Token::Comment | Token::BlockComment => None, - // The reserved keyword is a sentinel: the prescan consumed the one - // legitimate directive before lexing, so any occurrence is misplaced. - Token::Simc => { - errors.push(RichError::new(Error::ReservedSimcKeyword, shift(span))); - None - } - tok => Some((tok, shift(span))), + .filter_map(|(tok, span)| filter_token(tok, span, &mut errors, shift)) + .collect() + }); + + finalize_lex_errors(lex_errors, &mut errors, shift); + + (tokens, errors) +} + +/// Lexes an input string into a lossles stream of tokens with spans, beginning at byte +/// offset `start` — the end of the version directive per `SimcDirective::prescan`, +/// or `0`. Spans are reported relative to the full input. +/// +/// All comments, newlines, and spaces in the input code are remained. +#[cfg(feature = "fmt")] +pub fn lex_lossless( + file_id: usize, + input: &str, + start: usize, +) -> (Option>, Vec) { + let (tokens, lex_errors) = lexer_lossless().parse(&input[start..]).into_output_errors(); + let shift = |span: SimpleSpan| simple_span_to_span(span, file_id, start); + + // The reserved-keyword errors come first: a stray directive also produces + // follow-up errors for its `"";` remnant, and the sentinel is their cause. + let mut errors: Vec = Vec::new(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .filter_map(|(fmt_tok, span)| match fmt_tok { + FmtToken::Token(tok) => filter_token(tok, span, &mut errors, shift) + .map(|(t, s)| (FmtToken::Token(t), s)), + FmtToken::Trivia(t) => Some((FmtToken::Trivia(t), shift(span))), }) .collect() }); + finalize_lex_errors(lex_errors, &mut errors, shift); + + (tokens, errors) +} + +fn finalize_lex_errors Span>( + lex_errors: Vec>, + errors: &mut Vec, + convert_span: F, +) { errors.extend(lex_errors.into_iter().map(|err| { RichError::new( Error::CannotParse { msg: err.reason().to_string(), }, - shift(*err.span()), + convert_span(*err.span()), ) })); +} - (tokens, errors) +#[inline] +fn simple_span_to_span(span: SimpleSpan, file_id: usize, start: usize) -> Span { + Span::new(file_id, span.start + start..span.end + start) +} + +fn filter_token<'src, F: Fn(SimpleSpan) -> Span>( + tok: Token<'src>, + span: SimpleSpan, + errors: &mut Vec, + convert_span: F, +) -> Option<(Token<'src>, Span)> { + match tok { + // The reserved keyword is a sentinel: the prescan consumed the one + // legitimate directive before lexing, so any occurrence is misplaced. + Token::Simc => { + errors.push(RichError::new( + Error::ReservedSimcKeyword, + convert_span(span), + )); + None + } + tok => Some((tok, convert_span(span))), + } } /// A list of all reserved keywords. @@ -324,130 +441,347 @@ mod tests { use super::*; - fn lex<'src>( - input: &'src str, - ) -> (Option>>, Vec>) { - let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| vec.iter().map(|(tok, _)| tok.clone()).collect::>()); - (tokens, errors) - } + mod lexer { + use super::*; + + fn lex<'src>( + input: &'src str, + ) -> (Option>>, Vec>) { + let (tokens, errors) = lexer().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); + (tokens, errors) + } + #[test] + fn test_block_comment_simple() { + let input = "/* hello world */"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![]), + "Should produce a single block comment token" + ); + } - #[test] - fn test_block_comment_simple() { - let input = "/* hello world */"; - let (tokens, errors) = lex(input); - - assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); - assert_eq!( - tokens, - Some(vec![Token::BlockComment]), - "Should produce a single block comment token" - ); - } + #[test] + fn test_block_comment_nested() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_nested() { - let input = "/* outer /* inner */ outer */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_deeply_nested() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_deeply_nested() { - let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_multiline() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_multiline() { - let input = "/* \n line 1 \n /* inner \n line */ \n */"; - let (tokens, errors) = lex(input); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_unclosed() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_unclosed() { - let input = "/* unclosed comment start"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); - assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); - let err = &errors[0]; - assert_eq!(err.span().start, 0); - assert_eq!(err.span().end, 2); - assert_eq!(err.to_string(), "Unclosed block comment"); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_partial_nesting_unclosed() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_partial_nesting_unclosed() { - let input = "/* outer /* inner */"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!(tokens, Some(vec![])); + } - assert_eq!(errors.len(), 1); - assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![Token::BlockComment])); - } + #[test] + fn test_block_comment_double_unclosed() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex(input); - #[test] - fn test_block_comment_double_unclosed() { - let input = "/* outer /* inner"; - let (tokens, errors) = lex(input); + assert_eq!(errors.len(), 2); - assert_eq!(errors.len(), 2); + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - assert_eq!(errors[0].span().start, 9); - assert_eq!(errors[0].to_string(), "Unclosed block comment"); + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert_eq!(errors[1].span().start, 0); - assert_eq!(errors[1].to_string(), "Unclosed block comment"); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn simc_is_reserved() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, Token::Simc)), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); + } + + #[test] + fn lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); + + let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); + let _ = tokens.unwrap(); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert!(lex_errs.is_empty()); + } } - #[test] - fn simc_is_reserved() { - // The prescan consumes the one legitimate leading directive before lexing, - // so `lex` reports any `simc` it sees and drops the sentinel token. - for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { - let (tokens, errors) = super::lex(0, src, 0); - assert!( - errors.iter().any(|e| e.to_string().contains("reserved")), - "expected a reserved-keyword error for {src:?}, got: {errors:?}" + #[cfg(feature = "fmt")] + mod fmt_lexer { + use super::*; + + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } + + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]), + "Should produce a single block comment token" ); - assert!( - tokens - .expect("recovery keeps the stream") - .iter() - .all(|(tok, _)| !matches!(tok, Token::Simc)), - "the sentinel must not reach the token stream for {src:?}" + } + + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| matches!(token, FmtToken::Trivia(TriviaKind::Newline))) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } + + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) ); } - // Identifiers merely starting with `simc` are ordinary identifiers. - let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); - assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); - } + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } + + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } + + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } - #[test] - fn lexer_test() { - use chumsky::prelude::*; + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); - // Check if the lexer parses the example file without errors. - let src = include_str!("../examples/last_will.simf"); + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } + + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); - let (tokens, lex_errs) = lexer().parse(src).into_output_errors(); - let _ = tokens.unwrap(); + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert!(lex_errs.is_empty()); + assert_eq!( + tokens, + Some(vec![FmtToken::Trivia(TriviaKind::BlockComment)]) + ); + } + + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Whitespace,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + FmtToken::Trivia(TriviaKind::Newline,), + ]) + ); + } + + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } + + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); + + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } } } diff --git a/src/parse.rs b/src/parse.rs index 61c20360..eea6f88f 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -21,7 +21,11 @@ use crate::driver::{CRATE_STR, MAIN_MODULE}; use crate::error::ErrorCollector; use crate::error::{Error, RichError, Span}; use crate::impl_eq_hash; +#[cfg(not(feature = "fmt"))] use crate::lexer::Token; +use crate::lexer::Tokens; +#[cfg(feature = "fmt")] +use crate::lexer::{FmtToken, FmtTokens, Token}; use crate::num::NonZeroPow2Usize; use crate::pattern::Pattern; use crate::source::SourceFile; @@ -30,7 +34,7 @@ use crate::str::{ SymbolName, WitnessName, }; use crate::types::{AliasedType, BuiltinAlias, TypeConstructible, UIntType}; -use crate::unstable::{impl_require_feature, UnstableFeature, UnstableFeatures}; +use crate::unstable::{impl_require_feature, RequireFeature, UnstableFeature, UnstableFeatures}; use crate::version::SimcDirective; /// A program is a sequence of items. @@ -40,6 +44,41 @@ pub struct Program { span: Span, } +/// Source-aware parse result used by formatters. +/// +/// The compiler keeps using [`Program`] directly. Formatters need the same +/// semantic tree plus the lossless trivia stream that the grammar intentionally +/// does not consume. +#[cfg(feature = "fmt")] +#[derive(Clone, Debug)] +pub struct ParsedSource<'src> { + simc_directive: String, + program: Program, + tokens: FmtTokens<'src>, + prefix: Span, +} + +#[cfg(feature = "fmt")] +impl<'src> ParsedSource<'src> { + /// Access the semantic program used for formatting. + pub fn program(&self) -> &Program { + &self.program + } + + /// Access the lossless token stream in source order. + pub fn tokens(&self) -> &FmtTokens<'src> { + &self.tokens + } + + /// Access the source prefix skipped by the version-directive prescan. + /// + /// A formatter should preserve this range verbatim until the directive gains + /// its own formatting grammar. + pub fn prefix(&self) -> Span { + self.prefix + } +} + impl Program { // Need for driver usage pub(crate) fn new(items: &[Item], span: Span) -> Self { @@ -53,6 +92,66 @@ impl Program { pub fn items(&self) -> &[Item] { &self.items } + + /// Parse source for formatting while retaining all comments and whitespace. + #[cfg(feature = "fmt")] + pub fn parse_for_formatting<'src>( + file_id: usize, + source: &'src SourceFile, + unstable_features: &UnstableFeatures, + ) -> Result, Vec> { + let input = source.as_ref(); + let mut errors = ErrorCollector::new(); + + let Some(start) = pipeline::directive_prescan(source, file_id, &mut errors) else { + return Err(errors.get().to_vec()); + }; + let simc_directive = source.content()[0..start].to_string(); + + let (tokens, lex_errors) = crate::lexer::lex_lossless(file_id, input, start); + let lex_ok = match pipeline::is_lex_ok(source, lex_errors, &mut errors) { + None => { + return Err(errors.get().to_vec()); + } + Some(x) => x, + }; + let Some(tokens) = tokens else { + return Err(errors.get().to_vec()); + }; + + let semantic_tokens = tokens + .iter() + .filter_map(|(token, span)| match token { + FmtToken::Token(token) => Some((token.clone(), *span)), + FmtToken::Trivia(_) => None, + }) + .collect::>(); + + let (program, parse_ok) = + pipeline::parse_ast(file_id, source, semantic_tokens, &mut errors); + + if parse_ok && lex_ok { + pipeline::post_check(source, unstable_features, &program, &mut errors); + } + + if errors.has_errors() { + Err(errors.get().to_vec()) + } else { + let Some(program) = program else { + return Err(vec![RichError::parsing_error( + "Empty AST without an error.", + ) + .with_source(source.clone())]); + }; + + Ok(ParsedSource { + simc_directive, + program, + tokens, + prefix: Span::new(file_id, 0..start), + }) + } + } } impl_eq_hash!(Program; items); @@ -78,6 +177,21 @@ pub enum Item { Ignored, } +impl Item { + /// Access the source span when this item was parsed successfully. + /// + /// Error-recovery placeholders have no source node to decorate. + pub fn span(&self) -> Option<&Span> { + match self { + Self::TypeAlias(alias) => Some(alias.span()), + Self::Function(function) => Some(function.span()), + Self::Use(use_decl) => Some(use_decl.span()), + Self::Module(module) => Some(module.span()), + Self::Ignored => None, + } + } +} + impl_require_feature!(Item { variants: TypeAlias(alias), @@ -274,11 +388,12 @@ impl_eq_hash!(Function; visibility, name, params, ret, body); impl_require_feature!(Function { recurse: params, ret, body; }); /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct FunctionParam { identifier: Identifier, ty: AliasedType, + span: Span, } impl FunctionParam { @@ -291,8 +406,15 @@ impl FunctionParam { pub fn ty(&self) -> &AliasedType { &self.ty } + + /// Access the source span of the complete parameter. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + impl_require_feature!(FunctionParam { recurse: ty; }); /// A statement is a component of a block expression. @@ -304,6 +426,19 @@ pub enum Statement { Expression(Expression), } +impl Statement { + /// Access the span of the statement contents. + /// + /// The terminating semicolon is deliberately not part of this span: it is a + /// separator owned by the containing block's layout. + pub fn span(&self) -> &Span { + match self { + Self::Assignment(assignment) => assignment.span(), + Self::Expression(expression) => expression.span(), + } + } +} + impl_require_feature!(Statement { variants: Assignment(assignment), @@ -650,10 +785,12 @@ impl_eq_hash!(Match; scrutinee, left, right); impl_require_feature!(Match {recurse: scrutinee, left, right; }); /// Arm of a match expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, + comma_span: Option, } impl MatchArm { @@ -666,8 +803,20 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } + + /// Access the span of the arm's trailing comma, when present. + pub fn comma_span(&self) -> Option<&Span> { + self.comma_span.as_ref() + } } +impl_eq_hash!(MatchArm; pattern, expression); + impl_require_feature!(MatchArm {recurse: pattern, expression; }); /// Pattern of a match arm. @@ -1174,6 +1323,10 @@ impl_parse_wrapped_string!(WitnessName, "witness name"); impl_parse_wrapped_string!(AliasName, "alias name"); impl_parse_wrapped_string!(ModuleName, "module name"); +pub trait AstNode: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + +impl AstNode for T where T: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + /// Copy of [`FromStr`] that internally uses the `chumsky` parser. pub trait ParseFromStr: Sized { /// Parse a value from the string `s`. @@ -1244,31 +1397,40 @@ impl ParseFromStr for A { } } -impl ParseFromStrWithErrors - for A -{ - fn parse_from_str_with_errors( +mod pipeline { + use super::*; + + /// Handle the `simc` directive before lexing: an incompatible or malformed + /// directive is reported as the only diagnostic (the rest is noise), and + /// lexing starts right after a valid one, so the lexer and grammar never + /// see it. + pub fn directive_prescan( + src_file: &SourceFile, file_id: usize, - source: impl Into, - unstable_features: &UnstableFeatures, - handler: &mut ErrorCollector, - ) -> Option { - let source: SourceFile = source.into(); - let content = source.content(); - - // Handle the `simc` directive before lexing: an incompatible or malformed - // directive is reported as the only diagnostic (the rest is noise), and - // lexing starts right after a valid one, so the lexer and grammar never - // see it. - let start = match SimcDirective::prescan(&content, file_id) { - Ok(start) => start, + err_collector: &mut ErrorCollector, + ) -> Option { + match SimcDirective::prescan(&src_file.content(), file_id) { + Ok(start) => Some(start), Err((err, span)) => { - handler.push(RichError::new(err, span).with_source(source)); - return None; + err_collector.push(RichError::new(err, span).with_source(src_file.clone())); + None } - }; + } + } - let (tokens, mut lex_errs) = crate::lexer::lex(file_id, &content, start); + pub fn lex( + file_id: usize, + src_file: &SourceFile, + start: usize, + ) -> (Option>, Vec) { + crate::lexer::lex(file_id, src_file.as_ref(), start) + } + + pub fn is_lex_ok( + src: &SourceFile, + mut lex_errs: Vec, + err_sink: &mut ErrorCollector, + ) -> Option { // A stray `simc` makes every other diagnostic noise — its `"";` remnant // does not lex — so the reserved-keyword errors are reported alone. if lex_errs @@ -1276,27 +1438,68 @@ impl ParseF .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) { lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - handler.extend(source, lex_errs); + err_sink.extend(src.clone(), lex_errs); return None; } + let lex_ok = lex_errs.is_empty(); - handler.extend(source.clone(), lex_errs); - let tokens = tokens?; + err_sink.extend(src.clone(), lex_errs); + Some(lex_ok) + } - let eoi = Span::eof(file_id, content.len()); - let (ast, parse_errs) = A::parser() + pub fn parse_ast( + file_id: usize, + src: &SourceFile, + tokens: Tokens<'_>, + err_sink: &mut ErrorCollector, + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.as_ref().len()); + let (ast, parse_errs) = T::parser() .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) .into_output_errors(); + let parse_ok = parse_errs.is_empty(); - handler.extend(source.clone(), parse_errs); + err_sink.extend(src.clone(), parse_errs); + (ast, parse_ok) + } - if let (Some(ast), true) = (&ast, lex_ok && parse_ok) { - unstable_features.check_program(ast, &source, handler); + pub fn post_check( + src_file: &SourceFile, + unstable_features: &UnstableFeatures, + program: &Option, + handler: &mut ErrorCollector, + ) { + if let Some(ast) = program { + unstable_features.check_program(ast, &src_file, handler); + } + } +} + +impl ParseFromStrWithErrors for A { + fn parse_from_str_with_errors( + file_id: usize, + source: impl Into, + unstable_features: &UnstableFeatures, + err_sink: &mut ErrorCollector, + ) -> Option { + let source: SourceFile = source.into(); + + let start = pipeline::directive_prescan(&source, file_id, err_sink).unwrap_or_default(); + + let (tokens, lex_errs) = pipeline::lex(file_id, &source, start); + + let lex_ok = pipeline::is_lex_ok(&source, lex_errs, err_sink)?; + let tokens = tokens?; + + let (ast, parse_status) = pipeline::parse_ast(file_id, &source, tokens, err_sink); + + if parse_status && lex_ok { + let _ = pipeline::post_check(&source, unstable_features, &ast, err_sink); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module // is not handling poisoned tree right now, we don't return parsed result - if handler.has_errors() { + if err_sink.has_errors() { None } else { ast @@ -1673,7 +1876,11 @@ impl ChumskyParse for FunctionParam { identifier .then_ignore(just(Token::Colon)) .then(ty) - .map(|(identifier, ty)| Self { identifier, ty }) + .map_with(|(identifier, ty), e| Self { + identifier, + ty, + span: e.span(), + }) } } @@ -2136,7 +2343,7 @@ impl MatchArm { MatchPattern::parser() .then_ignore(just(Token::FatArrow)) .then(expr.map(Arc::new)) - .then(just(Token::Comma).or_not()) + .then(just(Token::Comma).to_span().or_not()) .validate(|((pattern, expression), comma), e, emitter| { let is_block = matches!(expression.as_ref().inner, ExpressionInner::Block(_, _)); @@ -2150,10 +2357,13 @@ impl MatchArm { ); } - Self { - pattern, - expression, - } + (pattern, expression, comma) + }) + .map_with(|(pattern, expression, comma_span), e| Self { + pattern, + expression, + span: e.span(), + comma_span, }) } } @@ -2233,6 +2443,8 @@ impl Match { let match_arm_fallback = MatchArm { expression: Arc::new(Expression::empty(Span::DUMMY)), pattern: MatchPattern::False, + span: Span::DUMMY, + comma_span: None, }; let (left, right) = ( @@ -2308,6 +2520,12 @@ impl AsRef for Function { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Assignment { fn as_ref(&self) -> &Span { &self.span @@ -2344,6 +2562,12 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for UseDecl { fn as_ref(&self) -> &Span { &self.span @@ -2647,10 +2871,14 @@ impl crate::ArbitraryRec for Match { left: MatchArm { pattern: pat_l, expression: expr_l, + span: Span::DUMMY, + comma_span: None, }, right: MatchArm { pattern: pat_r, expression: expr_r, + span: Span::DUMMY, + comma_span: None, }, span: Span::DUMMY, }) @@ -2675,98 +2903,101 @@ mod test { } } - #[test] - fn test_reject_redefined_builtin_type() { - let ty = TypeAlias::parse_from_str("type Ctx8 = u32") - .expect_err("Redefining built-in alias should be rejected"); + mod regular_parsing { + use super::*; - assert!(ty - .error() - .to_string() - .contains("Type alias `Ctx8` is already exists as built-in alias")); - } + #[test] + fn test_reject_redefined_builtin_type() { + let ty = TypeAlias::parse_from_str("type Ctx8 = u32") + .expect_err("Redefining built-in alias should be rejected"); - #[test] - fn test_fragment_rejects_version_directive() { - let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") - .expect_err("a version directive must not be accepted in a fragment"); + assert!(ty + .error() + .to_string() + .contains("Type alias `Ctx8` is already exists as built-in alias")); + } - assert!(matches!(err.error(), Error::ReservedSimcKeyword)); - } + #[test] + fn test_fragment_rejects_version_directive() { + let err = TypeAlias::parse_from_str("simc \"*\"; type Ctx8 = u32") + .expect_err("a version directive must not be accepted in a fragment"); - #[test] - fn test_double_colon() { - let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; - let source = SourceFile::anonymous(Arc::from(input)); - let mut error_handler = ErrorCollector::new(); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - source, - &UnstableFeatures::all(), - &mut error_handler, - ); + assert!(matches!(err.error(), Error::ReservedSimcKeyword)); + } - assert!(parsed_program.is_none()); - assert!(ErrorCollector::to_string(&error_handler).contains("Expected '::', found ':'")); - } + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + let source = SourceFile::anonymous(Arc::from(input)); + let mut error_handler = ErrorCollector::new(); + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + source, + &UnstableFeatures::all(), + &mut error_handler, + ); - #[test] - fn test_double_double_colon() { - let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; - let source = SourceFile::anonymous(Arc::from(input)); - let mut error_handler = ErrorCollector::new(); - let parsed_program = Program::parse_from_str_with_errors( - MAIN_MODULE, - source, - &UnstableFeatures::all(), - &mut error_handler, - ); + assert!(parsed_program.is_none()); + assert!(ErrorCollector::to_string(&error_handler).contains("Expected '::', found ':'")); + } - assert!(parsed_program.is_none()); - assert!(ErrorCollector::to_string(&error_handler).contains("Expected ';', found '::'")); - } + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + let source = SourceFile::anonymous(Arc::from(input)); + let mut error_handler = ErrorCollector::new(); + let parsed_program = Program::parse_from_str_with_errors( + MAIN_MODULE, + source, + &UnstableFeatures::all(), + &mut error_handler, + ); - /// Parse `input` and return whether it was rejected and the collected error text. - fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { - let source = SourceFile::anonymous(Arc::from(input)); - let mut handler = ErrorCollector::new(); - let program = - Program::parse_from_str_with_errors(MAIN_MODULE, source, features, &mut handler); - (program.is_none(), ErrorCollector::to_string(&handler)) - } + assert!(parsed_program.is_none()); + assert!(ErrorCollector::to_string(&error_handler).contains("Expected ';', found '::'")); + } - #[test] - fn test_gated_syntax_is_rejected_without_features() { - // Real `use`/`mod` syntax: rejected under none() (naming the feature + a - // -Z hint), accepted under all(). Delete when `imports` stabilizes. - for input in [ - "use crate::foo::bar;\nfn main() { }", - "mod inner { }\nfn main() { }", - ] { - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - rejected, - "gated syntax must be rejected without features (if this feature was \ + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let source = SourceFile::anonymous(Arc::from(input)); + let mut handler = ErrorCollector::new(); + let program = + Program::parse_from_str_with_errors(MAIN_MODULE, source, features, &mut handler); + (program.is_none(), ErrorCollector::to_string(&handler)) + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ just stabilized, delete this test):\n{input}" - ); - assert!( - error.contains("imports") && error.contains("-Z"), - "rejection should name the feature and suggest -Z, got:\n{error}" - ); + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); - let (rejected, error) = parse_with(input, &UnstableFeatures::all()); - assert!( - !rejected, - "the same syntax must parse with all features enabled:\n{error}" - ); + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } } - } - #[test] - fn test_type_heavy_program_needs_no_features() { - // Types are traversed by `RequireFeature` but not gated, so a type-heavy, - // import-free program must parse with no features (no false-positive gates). - let input = r#"type Alias = u32; + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; fn pick(e: Either) -> u32 { match e { Left(a: u32) => a, @@ -2779,51 +3010,129 @@ fn main() { assert!(jet::eq_32(chosen, chosen)); } "#; - // `parse_from_str_with_errors` returns `Some` only when no errors were - // collected, so a non-rejection already proves there were no gate errors. - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!( - !rejected, - "type-heavy but import-free program should parse with no features:\n{error}" - ); - } + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } - #[test] - fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { - // The gate check skips error-recovered ASTs, so a program with - // both a syntax error and gated syntax reports only the syntax error. - let input = "use crate::foo;\nfn main( {"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "broken program must be rejected"); - assert!( - !error.contains("-Z"), - "syntax error should not produce a spurious feature-gate hint, got:\n{error}" - ); - } + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } - #[test] - fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { - // Lexer-side companion to the case above: the stray `@` lexes with an - // error but recovers a cleanly-parsing `use` AST, and a program that - // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. - let input = "use crate@::foo;\nfn main() { }"; - let (rejected, error) = parse_with(input, &UnstableFeatures::none()); - assert!(rejected, "program with a lexer error must be rejected"); - assert!( - !error.contains("-Z"), - "lexer error should not produce a spurious feature-gate hint, got:\n{error}" - ); + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let program = parse::Program::parse_from_str(input).expect("program parses"); + assert_eq!(program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn parsed_nodes_keep_source_spans() { + let input = "fn choose(x: u8) -> u8 { match x { false => 0, true => 1, } }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + assert_eq!( + program.items()[0].span().unwrap().to_slice(input), + Some(input) + ); + assert_eq!(function.params()[0].span().to_slice(input), Some("x: u8")); + + let ExpressionInner::Block(_, Some(final_expr)) = function.body().inner() else { + panic!("expected a block with a final expression"); + }; + let ExpressionInner::Single(single) = final_expr.inner() else { + panic!("expected a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a match expression"); + }; + + assert_eq!(match_.left().span().to_slice(input), Some("false => 0,")); + assert_eq!( + match_.left().comma_span().unwrap().to_slice(input), + Some(",") + ); + assert_eq!(match_.right().span().to_slice(input), Some("true => 1,")); + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } } - #[test] - fn test_use_decl_display_round_trips_with_aliases() { - for input in [ - "use lib::A::foo;", - "use lib::A::foo as bar;", - "use lib::A::{foo, bar as baz};", - ] { - let program = parse::Program::parse_from_str(input).expect("parsing works"); - assert_eq!(program.to_string(), format!("{input}\n")); + #[cfg(feature = "fmt")] + mod fmt_parsing { + use super::*; + + #[test] + fn formatting_parse_keeps_trivia_and_directive_prefix() { + let input = "// header\nsimc \"*\";\n// body\nfn main() {}"; + let source = SourceFile::anonymous(Arc::from(input)); + let parsed = + Program::parse_for_formatting(0, &source, &UnstableFeatures::none()).unwrap(); + // .expect("formatting parse succeeds"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\nsimc \"*\";") + ); + assert!(parsed.tokens().iter().any(|(token, span)| { + matches!( + token, + FmtToken::Trivia(crate::lexer::TriviaKind::LineComment) + ) && span.to_slice(input) == Some("// body") + })); + assert_eq!(parsed.program().items().len(), 1); } } } diff --git a/src/source.rs b/src/source.rs index 49a70bc7..4826b4c6 100644 --- a/src/source.rs +++ b/src/source.rs @@ -22,6 +22,12 @@ impl From for SourceFile { } } +impl AsRef for SourceFile{ + fn as_ref(&self) -> &str { + self.content.as_ref() + } +} + impl SourceFile { /// Creates a standard `SourceFile` from a file path and its content. pub fn new(name: &Path, content: Arc) -> Self { diff --git a/src/version.rs b/src/version.rs index 7dd7bdf6..8ae2178c 100644 --- a/src/version.rs +++ b/src/version.rs @@ -41,6 +41,7 @@ enum DirectiveScan<'a> { } /// Operations on the `simc "";` compiler-version directive. +#[derive(Clone, Debug)] pub struct SimcDirective; impl SimcDirective {