From 0779650a7fc009ee5e63c8c370eb8758e1b01317 Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Tue, 13 Oct 2026 22:40:56 -0700 Subject: [PATCH 1/2] Test token locations and fix stale lexer doc comment The comment on Lexer::loc said the column was computed on demand, but it has been set eagerly for every token. Add a unit test pinning the Endl location and the deferred line advance introduced in #73. Co-Authored-By: Claude Fable 5.1 --- src/lexer.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/lexer.rs b/src/lexer.rs index fbfe225..990e7a5 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -104,7 +104,7 @@ pub struct Lexer { /// Current token. pub tok: Token, - /// Current source code location (file and line, col computed on demand). + /// Location of the current token. pub loc: Loc, } @@ -504,6 +504,61 @@ mod tests { Token::Id(Name::new(s.into())) } + /// Lexes `code` and returns each token with its (line, col). + fn located(code: &str) -> Vec<(Token, u32, u32)> { + let mut lex = Lexer::new(code, "test_lexer"); + let mut out = vec![]; + lex.next(); + while lex.tok != Token::End { + out.push((lex.tok.clone(), lex.loc.line, lex.loc.col)); + lex.next(); + } + out + } + + #[test] + fn test_token_locations() { + use Token::*; + + // Endl sits at the newline on the line it ends; the line advances + // for the following token. + assert_eq!( + located("foo(\n x"), + vec![ + (id("foo"), 1, 1), + (Lparen, 1, 4), + (Endl, 1, 5), + (id("x"), 2, 3) + ] + ); + + // Consecutive newlines, including blank lines with whitespace. + assert_eq!( + located("a\n\n \nb"), + vec![ + (id("a"), 1, 1), + (Endl, 1, 2), + (Endl, 2, 1), + (Endl, 3, 3), + (id("b"), 4, 1) + ] + ); + + // A comment ends with its newline on the comment's line. + assert_eq!( + located("x // note\ny"), + vec![(id("x"), 1, 1), (Endl, 1, 10), (id("y"), 2, 1)] + ); + + // End right after a newline is on the new line. + let mut lex = Lexer::new("x\n", "test_lexer"); + lex.next(); + lex.next(); + assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (Endl, 1, 2)); + lex.next(); + assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (End, 2, 1)); + } + #[test] fn test_lexer() { use Token::*; From 94969610521adc21ad89384efb33bd342aa2bfeb Mon Sep 17 00:00:00 2001 From: Taylor Holliday Date: Tue, 13 Oct 2026 22:42:53 -0700 Subject: [PATCH 2/2] Count raw newlines inside string literals The lexer only advanced the line counter for whitespace and comment newlines, so every token after a multi-line string literal reported the wrong line and a column measured from an earlier line. Replace the "previous token was Endl" check with a pending_lines counter that both the newline handler and the string scanner bump. The line advances when the next token is lexed, so the string token keeps the line of its opening quote. The parser's struct-literal backtrack now saves and restores the counter along with the rest of the lexer state. Co-Authored-By: Claude Fable 5.1 --- src/lexer.rs | 31 ++++++++++++++++++++++++++++--- src/parser.rs | 2 ++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 990e7a5..077e84c 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -104,6 +104,12 @@ pub struct Lexer { /// Current token. pub tok: Token, + /// Newlines consumed while lexing the current token (the newline of an + /// `Endl`, or raw newlines inside a string literal). Applied to `loc.line` + /// when the next token is lexed, so the current token keeps the line it + /// started on. + pub pending_lines: u32, + /// Location of the current token. pub loc: Loc, } @@ -119,6 +125,7 @@ impl Lexer { i: 0, line_start: 0, tok: Token::Error, + pending_lines: 0, loc: Loc { file: Name::new(file.into()), line: 1, @@ -135,16 +142,18 @@ impl Lexer { self.loc.col = (self.i - self.line_start + 1) as u32; self.i += 1; self.line_start = self.i; + self.pending_lines = 1; } fn _next(&mut self) -> Token { let bytes = self.code.as_bytes(); let n = bytes.len(); - // The previous token was a newline: move on to the next line. - if self.tok == Token::Endl { - self.loc.line += 1; + // The previous token consumed newlines: move on to the next line. + if self.pending_lines > 0 { + self.loc.line += self.pending_lines; self.loc.col = 1; + self.pending_lines = 0; } // Skip whitespace. @@ -309,6 +318,10 @@ impl Lexer { _ => return Token::Error, } } else { + if bytes[self.i] == b'\n' { + self.pending_lines += 1; + self.line_start = self.i + 1; + } s.push(bytes[self.i] as char); } self.i += 1; @@ -550,6 +563,18 @@ mod tests { vec![(id("x"), 1, 1), (Endl, 1, 10), (id("y"), 2, 1)] ); + // Raw newlines inside a string literal advance the line for the + // tokens that follow; the string itself keeps its opening line. + assert_eq!( + located("\"a\nbc\" x\n y"), + vec![ + (String(Name::str("a\nbc")), 1, 1), + (id("x"), 2, 5), + (Endl, 2, 6), + (id("y"), 3, 2), + ] + ); + // End right after a newline is on the new line. let mut lex = Lexer::new("x\n", "test_lexer"); lex.next(); diff --git a/src/parser.rs b/src/parser.rs index e038c10..c5b5124 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -474,6 +474,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext let saved_line_start = cx.lex.line_start; let saved_tok = cx.lex.tok.clone(); let saved_loc = cx.lex.loc; + let saved_pending_lines = cx.lex.pending_lines; cx.next(); if cx.lex.tok == Token::Colon { // Struct literal: Name(field: value, ...) @@ -504,6 +505,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext cx.lex.line_start = saved_line_start; cx.lex.tok = saved_tok; cx.lex.loc = saved_loc; + cx.lex.pending_lines = saved_pending_lines; let args = parse_exprlist(arena, typevars, cx); expect(Token::Rparen, cx); e = arena.add(Expr::Call(e, args), call_loc);