diff --git a/src/lexer.rs b/src/lexer.rs index fbfe225..077e84c 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -104,7 +104,13 @@ pub struct Lexer { /// Current token. pub tok: Token, - /// Current source code location (file and line, col computed on demand). + /// Newlines consumed while lexing the current token (the newline of an + /// `Endl`, or raw newlines inside a string literal). Applied to `loc.line` + /// when the next token is lexed, so the current token keeps the line it + /// started on. + pub pending_lines: u32, + + /// Location of the current token. pub loc: Loc, } @@ -119,6 +125,7 @@ impl Lexer { i: 0, line_start: 0, tok: Token::Error, + pending_lines: 0, loc: Loc { file: Name::new(file.into()), line: 1, @@ -135,16 +142,18 @@ impl Lexer { self.loc.col = (self.i - self.line_start + 1) as u32; self.i += 1; self.line_start = self.i; + self.pending_lines = 1; } fn _next(&mut self) -> Token { let bytes = self.code.as_bytes(); let n = bytes.len(); - // The previous token was a newline: move on to the next line. - if self.tok == Token::Endl { - self.loc.line += 1; + // The previous token consumed newlines: move on to the next line. + if self.pending_lines > 0 { + self.loc.line += self.pending_lines; self.loc.col = 1; + self.pending_lines = 0; } // Skip whitespace. @@ -309,6 +318,10 @@ impl Lexer { _ => return Token::Error, } } else { + if bytes[self.i] == b'\n' { + self.pending_lines += 1; + self.line_start = self.i + 1; + } s.push(bytes[self.i] as char); } self.i += 1; @@ -504,6 +517,73 @@ mod tests { Token::Id(Name::new(s.into())) } + /// Lexes `code` and returns each token with its (line, col). + fn located(code: &str) -> Vec<(Token, u32, u32)> { + let mut lex = Lexer::new(code, "test_lexer"); + let mut out = vec![]; + lex.next(); + while lex.tok != Token::End { + out.push((lex.tok.clone(), lex.loc.line, lex.loc.col)); + lex.next(); + } + out + } + + #[test] + fn test_token_locations() { + use Token::*; + + // Endl sits at the newline on the line it ends; the line advances + // for the following token. + assert_eq!( + located("foo(\n x"), + vec![ + (id("foo"), 1, 1), + (Lparen, 1, 4), + (Endl, 1, 5), + (id("x"), 2, 3) + ] + ); + + // Consecutive newlines, including blank lines with whitespace. + assert_eq!( + located("a\n\n \nb"), + vec![ + (id("a"), 1, 1), + (Endl, 1, 2), + (Endl, 2, 1), + (Endl, 3, 3), + (id("b"), 4, 1) + ] + ); + + // A comment ends with its newline on the comment's line. + assert_eq!( + located("x // note\ny"), + vec![(id("x"), 1, 1), (Endl, 1, 10), (id("y"), 2, 1)] + ); + + // Raw newlines inside a string literal advance the line for the + // tokens that follow; the string itself keeps its opening line. + assert_eq!( + located("\"a\nbc\" x\n y"), + vec![ + (String(Name::str("a\nbc")), 1, 1), + (id("x"), 2, 5), + (Endl, 2, 6), + (id("y"), 3, 2), + ] + ); + + // End right after a newline is on the new line. + let mut lex = Lexer::new("x\n", "test_lexer"); + lex.next(); + lex.next(); + assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (Endl, 1, 2)); + lex.next(); + assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (End, 2, 1)); + } + #[test] fn test_lexer() { use Token::*; diff --git a/src/parser.rs b/src/parser.rs index e038c10..c5b5124 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -474,6 +474,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext let saved_line_start = cx.lex.line_start; let saved_tok = cx.lex.tok.clone(); let saved_loc = cx.lex.loc; + let saved_pending_lines = cx.lex.pending_lines; cx.next(); if cx.lex.tok == Token::Colon { // Struct literal: Name(field: value, ...) @@ -504,6 +505,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext cx.lex.line_start = saved_line_start; cx.lex.tok = saved_tok; cx.lex.loc = saved_loc; + cx.lex.pending_lines = saved_pending_lines; let args = parse_exprlist(arena, typevars, cx); expect(Token::Rparen, cx); e = arena.add(Expr::Call(e, args), call_loc);