scanner: don't allow an unknown mark to run into a string or comment

[ocean] / csrc / scanner.mdc

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index 6dda84832c8931a0f9e7f33d5118680380d95fdf..7e33d0cbc1a89126706d2f72ce1f2351e0d395ee 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -318,6 +318,12 @@ in a known mark, it will return that first known mark.
 
 If no known mark is found we will test against strings and comments
 below before giving up and assuming an unknown mark.
+
+If an unknown mark contains a quote character or a comment marker, and
+that token is not being ignored, then we terminate the unknown mark
+before that quote or comment.  This ensure that an unknown mark
+immediately before a string is handled correctly.
+
 If `TK_mark` is ignored, then unknown marks as returned as an error.
 
 ###### token types
@@ -329,6 +335,7 @@ Known marks are included in the same list as the list of known words.
        tk.num = TK_error;
        while (is_mark(ch, state->conf)) {
                int n;
+               wchar_t prev;
                close_token(state, &tk);
                n = find_known(state->conf, tk.txt);
                if (n >= 0)
@@ -339,7 +346,22 @@ Known marks are included in the same list as the list of known words.
                        close_token(state, &tk);
                        return tk;
                }
+               prev = ch;
+               if (prev == '/')
+                       save_unget_state(state);
                ch = get_char(state);
+               if (!(ignored && (1<<TK_string)) && is_quote(ch))
+                       break;
+               if (!(ignored && (1<<TK_line_comment)) &&
+                   prev == '/' && ch == '/') {
+                       restore_unget_state(state);
+                       break;
+               }
+               if (!(ignored && (1<<TK_block_comment)) &&
+                   prev == '/' && ch == '*') {
+                       restore_unget_state(state);
+                       break;
+               }
        }
        unget_char(state);
        if (tk.num != TK_error)