Lexer location fixes and tests

src/lexer.rs

@@ -104,7 +104,13 @@ pub struct Lexer {
     /// Current token.
     pub tok: Token,
 
-    /// Current source code location (file and line, col computed on demand).
+    /// Newlines consumed while lexing the current token (the newline of an
+    /// `Endl`, or raw newlines inside a string literal). Applied to `loc.line`
+    /// when the next token is lexed, so the current token keeps the line it
+    /// started on.
+    pub pending_lines: u32,
+
+    /// Location of the current token.
     pub loc: Loc,
 }
 
@@ -119,6 +125,7 @@ impl Lexer {
             i: 0,
             line_start: 0,
             tok: Token::Error,
+            pending_lines: 0,
             loc: Loc {
                 file: Name::new(file.into()),
                 line: 1,
@@ -135,16 +142,18 @@ impl Lexer {
         self.loc.col = (self.i - self.line_start + 1) as u32;
         self.i += 1;
         self.line_start = self.i;
+        self.pending_lines = 1;
     }
 
     fn _next(&mut self) -> Token {
         let bytes = self.code.as_bytes();
         let n = bytes.len();
 
-        // The previous token was a newline: move on to the next line.
-        if self.tok == Token::Endl {
-            self.loc.line += 1;
+        // The previous token consumed newlines: move on to the next line.
+        if self.pending_lines > 0 {
+            self.loc.line += self.pending_lines;
             self.loc.col = 1;
+            self.pending_lines = 0;
         }
 
         // Skip whitespace.
@@ -309,6 +318,10 @@ impl Lexer {
                         _ => return Token::Error,
                     }
                 } else {
+                    if bytes[self.i] == b'\n' {
+                        self.pending_lines += 1;
+                        self.line_start = self.i + 1;
+                    }
                     s.push(bytes[self.i] as char);
                 }
                 self.i += 1;
@@ -504,6 +517,73 @@ mod tests {
         Token::Id(Name::new(s.into()))
     }
 
+    /// Lexes `code` and returns each token with its (line, col).
+    fn located(code: &str) -> Vec<(Token, u32, u32)> {
+        let mut lex = Lexer::new(code, "test_lexer");
+        let mut out = vec![];
+        lex.next();
+        while lex.tok != Token::End {
+            out.push((lex.tok.clone(), lex.loc.line, lex.loc.col));
+            lex.next();
+        }
+        out
+    }
+
+    #[test]
+    fn test_token_locations() {
+        use Token::*;
+
+        // Endl sits at the newline on the line it ends; the line advances
+        // for the following token.
+        assert_eq!(
+            located("foo(\n  x"),
+            vec![
+                (id("foo"), 1, 1),
+                (Lparen, 1, 4),
+                (Endl, 1, 5),
+                (id("x"), 2, 3)
+            ]
+        );
+
+        // Consecutive newlines, including blank lines with whitespace.
+        assert_eq!(
+            located("a\n\n  \nb"),
+            vec![
+                (id("a"), 1, 1),
+                (Endl, 1, 2),
+                (Endl, 2, 1),
+                (Endl, 3, 3),
+                (id("b"), 4, 1)
+            ]
+        );
+
+        // A comment ends with its newline on the comment's line.
+        assert_eq!(
+            located("x // note\ny"),
+            vec![(id("x"), 1, 1), (Endl, 1, 10), (id("y"), 2, 1)]
+        );
+
+        // Raw newlines inside a string literal advance the line for the
+        // tokens that follow; the string itself keeps its opening line.
+        assert_eq!(
+            located("\"a\nbc\" x\n y"),
+            vec![
+                (String(Name::str("a\nbc")), 1, 1),
+                (id("x"), 2, 5),
+                (Endl, 2, 6),
+                (id("y"), 3, 2),
+            ]
+        );
+
+        // End right after a newline is on the new line.
+        let mut lex = Lexer::new("x\n", "test_lexer");
+        lex.next();
+        lex.next();
+        assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (Endl, 1, 2));
+        lex.next();
+        assert_eq!((lex.tok, lex.loc.line, lex.loc.col), (End, 2, 1));
+    }
+
     #[test]
     fn test_lexer() {
         use Token::*;

src/parser.rs

@@ -474,6 +474,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext
                     let saved_line_start = cx.lex.line_start;
                     let saved_tok = cx.lex.tok.clone();
                     let saved_loc = cx.lex.loc;
+                    let saved_pending_lines = cx.lex.pending_lines;
                     cx.next();
                     if cx.lex.tok == Token::Colon {
                         // Struct literal: Name(field: value, ...)
@@ -504,6 +505,7 @@ fn parse_postfix(arena: &mut ExprArena, typevars: &[Name], cx: &mut ParseContext
                         cx.lex.line_start = saved_line_start;
                         cx.lex.tok = saved_tok;
                         cx.lex.loc = saved_loc;
+                        cx.lex.pending_lines = saved_pending_lines;
                         let args = parse_exprlist(arena, typevars, cx);
                         expect(Token::Rparen, cx);
                         e = arena.add(Expr::Call(e, args), call_loc);