gh-153569: report tokenizer diagnostics without rewinding the scanner

Author: pablogsal

Lib/test/test_codeop.py

@@ -113,6 +113,17 @@ def test_valid(self, compiler):
         av("def f():\n pass\n#foo\n")
         av("@a.b.c\ndef f():\n pass\n")
 
+    @subTests('symbol', ('single', 'exec'))
+    @subTests('prefix', ('', 'f', 't'))
+    def test_incomplete_string_diagnostics(self, symbol, prefix):
+        opening = f'    á = {prefix}"""first\n'
+        source = 'if True:\n' + opening + 'second'
+        with self.assertRaises(_IncompleteInputError) as cm:
+            Compile()(source, '<input>', symbol)
+        text = opening + 'second' + ('\n' if symbol == 'exec' else '')
+        self.assertEqual(cm.exception.args, (
+            'incomplete input', ('<input>', 2, 9, text, 2, -1)))
+
     @subTests('compiler', COMPILERS)
     def test_incomplete(self, compiler):
         ai = functools.partial(self.assertIncomplete, compiler=compiler)

Lib/test/test_source_encoding.py

@@ -3,8 +3,8 @@
 import unittest
 from test import support
 from test.support import script_helper
-from test.support.os_helper import TESTFN, unlink, rmtree
-from test.support.import_helper import unload
+from test.support.os_helper import TESTFN, TESTFN_ASCII, unlink, rmtree
+from test.support.import_helper import import_module, unload
 import importlib
 import os
 import sys
@@ -83,12 +83,30 @@ def test_truncated_utf8_at_eof(self):
                 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
 
     def test_invalid_utf8_offset_after_non_ascii(self):
+        for name in ('é', 'éé', '𝒜'):
+            with self.subTest(name=name):
+                source = ('x = ' + name).encode() + b'\xff\n'
+                with self.assertRaises(SyntaxError) as caught:
+                    compile(source, '<test>', 'exec')
+                error = caught.exception
+                self.assertEqual(
+                    (error.lineno, error.offset, error.end_lineno, error.end_offset),
+                    (1, 5 + len(name), 1, 5 + len(name)),
+                )
+
+    @support.cpython_only
+    def test_invalid_utf8_file_offset_after_non_ascii(self):
+        _testcapi = import_module('_testcapi')
+        self.addCleanup(unlink, TESTFN_ASCII)
+        with open(TESTFN_ASCII, 'wb') as f:
+            f.write(b'\nx = \xc3\xa9\xc3\xa9\xff\n')
         with self.assertRaises(SyntaxError) as caught:
-            compile(b"x = \xc3\xa9\xff\n", "<test>", "exec")
+            _testcapi.run_file(
+                os.fsencode(TESTFN_ASCII), _testcapi.Py_file_input, {})
         error = caught.exception
         self.assertEqual(
             (error.lineno, error.offset, error.end_lineno, error.end_offset),
-            (1, 6, 1, 6),
+            (2, 7, 2, 7),
         )
 
     def test_long_bom_conflict_message_is_not_truncated(self):

Lib/test/test_tstring.py

@@ -207,6 +207,8 @@ def test_nested_templates(self):
 
     def test_syntax_errors(self):
         for case, err in (
+            ('t"""{(\n1\n)}\ntail', "unterminated triple-quoted t-string literal"),
+            ('f"""{(\n1\n)}\ntail', "unterminated triple-quoted f-string literal"),
             ("t'", "unterminated t-string literal"),
             ("t'''", "unterminated triple-quoted t-string literal"),
             ("t''''", "unterminated triple-quoted t-string literal"),

Parser/lexer/lexer.c

@@ -99,6 +99,7 @@ verify_identifier(struct tok_state *tok)
     assert(PyUnicode_GET_LENGTH(s) > 0);
     if (invalid < PyUnicode_GET_LENGTH(s)) {
         Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
+        const char *error_cursor = tok->cur;
         if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
             /* Determine the offset in UTF-8 encoded input */
             Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
@@ -109,14 +110,20 @@ verify_identifier(struct tok_state *tok)
                 tok->done = E_ERROR;
                 return 0;
             }
-            tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+            error_cursor = tok->start + PyBytes_GET_SIZE(s);
         }
         Py_DECREF(s);
         if (Py_UNICODE_ISPRINTABLE(ch)) {
-            _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid character '%c' (U+%04X)", ch, ch);
         }
         else {
-            _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
+            _PyTokenizer_syntaxerror_at(
+                tok, tok->line_start,
+                error_cursor - tok->line_start, tok->lineno, -1, -1,
+                "invalid non-printable character U+%04X", ch);
         }
         return 0;
     }

Parser/lexer/state.h

@@ -77,6 +77,14 @@ typedef struct {
     indentation_level stack[MAXINDENT];
 } lexer_layout_state;
 
+/* Supplemental source context for a terminal error. location is the reporting
+   cursor, independent of the scanner cursor; lineno == 0 means absent.
+   The text span may cover multiple physical lines. */
+typedef struct {
+    _PyTok_Loc location;
+    _PyTok_Span text_span;
+} _PyTokenizer_Diagnostic;
+
 /* Tokenizer state */
 struct tok_state {
     /* Input state; buf <= cur <= inp */
@@ -92,6 +100,7 @@ struct tok_state {
     lexer_layout_state layout;
     int lineno;         /* Current line number */
     _PyTok_Loc start_loc;
+    _PyTokenizer_Diagnostic diagnostic;
     int level;          /* () [] {} Parentheses nesting level */
             /* Used to allow free continuations inside them */
     char parenstack[MAXLEVEL];

Parser/lexer/string.c

@@ -7,13 +7,18 @@
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
-static void
-rewind_to_string_start(struct tok_state *tok, const char *start,
-                       _PyTok_Loc location)
+static int
+string_error_token(struct tok_state *tok, struct token *token,
+                   const char *start, _PyTok_Loc location)
 {
-    tok->cur = (char *)start + 1;
-    tok->line_start = start - location.byte_col;
-    tok->lineno = location.lineno;
+    tok->diagnostic = (_PyTokenizer_Diagnostic){
+        .location = {location.lineno, location.byte_col + 1},
+        .text_span = _PyLexer_BufferSpan(tok, start - location.byte_col, tok->inp),
+    };
+    int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, NULL, NULL);
+    token->start_loc = location;
+    token->end_loc = (_PyTok_Loc){location.lineno, -1};
+    return type;
 }
 
 int
@@ -351,44 +356,51 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
             int end_lineno = tok->lineno;
-            rewind_to_string_start(tok, tok->start, tok->start_loc);
+            _PyTok_Loc location = tok->start_loc;
+            const char *line = tok->start - location.byte_col;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             const ftstring_state *state = _PyLexer_CurrentFTString(tok);
             if (state != NULL) {
                 /* A matching quote belongs to the surrounding formatted
                  * string, so the expression is missing its closing brace. */
                 if (state->quote == quote && state->quote_size == quote_size) {
-                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
                         "%c-string: expecting '}'",
-                        _PyLexer_StringPrefix(state->kind)));
+                        _PyLexer_StringPrefix(state->kind));
+                    return string_error_token(tok, token, tok->start, location);
                 }
             }
 
             if (quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
-                                 " (detected at line %d)", end_lineno);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted string literal"
+                    " (detected at line %d)", end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
             else {
                 if (has_escaped_quote) {
-                    _PyTokenizer_syntaxerror(
-                        tok,
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
                         "unterminated string literal (detected at line %d); "
                         "perhaps you escaped the end quote?",
                         end_lineno
                     );
                 } else {
-                    _PyTokenizer_syntaxerror(
-                        tok, "unterminated string literal (detected at line %d)", end_lineno
+                    _PyTokenizer_syntaxerror_at(
+                        tok, line, cursor_offset, location.lineno, -1, -1,
+                        "unterminated string literal (detected at line %d)", end_lineno
                     );
                 }
                 if (c != '\n') {
                     tok->done = E_EOLS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token, tok->start, location);
             }
         }
         if (c == quote) {
@@ -451,25 +463,29 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
             }
 
             int end_lineno = tok->lineno;
-            rewind_to_string_start(tok,
-                _PyLexer_BufferPointer(tok, current->start),
-                current->start_loc);
+            _PyTok_Loc location = current->start_loc;
+            const char *line = _PyLexer_BufferPointer(tok, current->start) - location.byte_col;
+            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
 
             if (quote_size == 3) {
-                _PyTokenizer_syntaxerror(tok,
-                                    "unterminated triple-quoted %c-string literal"
-                                    " (detected at line %d)",
-                                    _PyLexer_StringPrefix(current->kind), end_lineno);
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated triple-quoted %c-string literal"
+                    " (detected at line %d)",
+                    _PyLexer_StringPrefix(current->kind), end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
-                return MAKE_TOKEN(ERRORTOKEN);
+                return string_error_token(tok, token,
+                    _PyLexer_BufferPointer(tok, current->start), location);
             }
             else {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                                    "unterminated %c-string literal (detected at"
-                                    " line %d)",
-                                    _PyLexer_StringPrefix(current->kind), end_lineno));
+                _PyTokenizer_syntaxerror_at(
+                    tok, line, cursor_offset, location.lineno, -1, -1,
+                    "unterminated %c-string literal (detected at line %d)",
+                    _PyLexer_StringPrefix(current->kind), end_lineno);
+                return string_error_token(tok, token,
+                    _PyLexer_BufferPointer(tok, current->start), location);
             }
         }
 

Parser/pegen.c

@@ -218,11 +218,11 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
 
     parser_token->level = new_token->level;
     parser_token->lineno = new_token->start_loc.lineno;
-    parser_token->col_offset = p->tok->lineno == p->starting_lineno
+    parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->start_loc.byte_col
         : new_token->start_loc.byte_col;
     parser_token->end_lineno = new_token->end_loc.lineno;
-    parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+    parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->end_loc.byte_col
         : new_token->end_loc.byte_col;
 

Parser/pegen_errors.c

@@ -202,7 +202,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
     Py_ssize_t col_offset;
     Py_ssize_t end_col_offset = -1;
     if (t->col_offset == -1) {
-        if (p->tok->cur == p->tok->buf) {
+        _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
+        if (diagnostic.location.lineno != 0) {
+            col_offset = diagnostic.location.byte_col;
+        } else if (p->tok->cur == p->tok->buf) {
             col_offset = 0;
         } else {
             const char* start = p->tok->buf  ? p->tok->line_start : p->tok->buf;
@@ -251,21 +254,27 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     PyObject *error_line = NULL;
     PyObject *tmp = NULL;
     p->error_indicator = 1;
+    _PyTokenizer_Diagnostic diagnostic = p->tok->diagnostic;
+    _PyTok_Loc location = diagnostic.location.lineno != 0
+        ? diagnostic.location : (_PyTok_Loc){p->tok->lineno,
+            p->tok->line_start == NULL ? -1 : _PyLexer_ByteColumn(p->tok)};
+    _PyTok_Span text_span = diagnostic.location.lineno != 0
+        ? diagnostic.text_span : _PyLexer_BufferSpan(
+            p->tok, p->tok->line_start, p->tok->inp);
 
     if (end_lineno == CURRENT_POS) {
-        end_lineno = p->tok->lineno;
+        end_lineno = location.lineno;
     }
     if (end_col_offset == CURRENT_POS) {
-        end_col_offset = p->tok->cur - p->tok->line_start;
+        end_col_offset = location.byte_col;
     }
 
     errstr = PyUnicode_FromFormatV(errmsg, va);
     if (!errstr) {
         goto error;
     }
 
-    if (_PyTok_ReaderIsInteractive(p->tok) &&
-            _PyTokenizer_RetainedSource(p->tok) != NULL) {
+    if (_PyTok_ReaderIsInteractive(p->tok) && _PyTokenizer_RetainedSource(p->tok) != NULL) {
         error_line = get_error_line_from_source(p, lineno);
     }
     else if (p->start_rule == Py_file_input) {
@@ -281,13 +290,16 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
            we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
            `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
            does not physically exist */
-        assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
-
-        if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
-            Py_ssize_t size = p->tok->inp - p->tok->line_start;
-            error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
+        assert((p->tok->fp == NULL || p->tok->fp == stdin) || p->tok->done == E_EOF);
+
+        if (location.lineno <= lineno &&
+                p->tok->inp > p->tok->buf) {
+            Py_ssize_t size;
+            const char *line = _PyLexer_BufferSpanView(
+                p->tok, text_span, &size);
+            error_line = PyUnicode_DecodeUTF8(line, size, "replace");
         }
-        else if (p->tok->fp == NULL || p->tok->fp == stdin) {
+        else if ((p->tok->fp == NULL || p->tok->fp == stdin)) {
             error_line = get_error_line_from_source(p, lineno);
         }
         else {

Parser/tokenizer/decoder.c

@@ -240,22 +240,14 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
         const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
         const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
         Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
-        const char *saved_line_start = tok->line_start;
-        char *saved_cur = tok->cur;
-        int saved_lineno = tok->lineno;
-        tok->line_start = line_data;
-        tok->cur = (char *)line_data;
-        tok->lineno = cookie_line;
         int end_col = (int)Py_MIN(line_len, INT_MAX);
         if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
                             line_data[end_col - 1] == '\r')) {
             end_col--;
         }
-        _PyTokenizer_syntaxerror_known_range(
-            tok, 0, end_col, "encoding problem: %s with BOM", cookie);
-        tok->line_start = saved_line_start;
-        tok->cur = saved_cur;
-        tok->lineno = saved_lineno;
+        _PyTokenizer_syntaxerror_at(
+            tok, line_data, 0, cookie_line, 0, end_col,
+            "encoding problem: %s with BOM", cookie);
         PyMem_Free(cookie);
         return _PYTOK_ENCODING_ERROR;
     }