Scanner: parsing of comments and strings must recognise end-of-node

If the file does not end in a newline, then a node might not also.

This requires a little more care in parsing strings and comments.

Signed-off-by: NeilBrown <neil@brown.name>

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index ecc202b3bfd4ad6b852678f89e6dc4881e24375d..37b336f2233c152b3cf398f385748cf647849c70 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -450,7 +450,8 @@ followed by the start of a new string.
                         * unget so the newline is seen,
                         * but return rest of string as an error.
                         */
-                       unget_char(state);
+                       if (is_newline(ch))
+                               unget_char(state);
                        close_token(state, &tk);
                        tk.num = TK_error;
                        return tk;
@@ -484,14 +485,18 @@ If `TK_string` is ignored, then quote characters will appear as `TK_mark`s.
            !(ignored & (1<<TK_string))) {
                wchar_t first = tk.txt.txt[0];
                reset_token(state, &tk);
-               get_char(state);
-               do
+               ch = get_char(state);
+               tk.num = TK_error;
+               while (!at_eon(state) && !is_newline(ch)) {
                        ch = get_char(state);
-               while (ch != first && !is_newline(ch));
-               tk.num = TK_string;
-               if (is_newline(ch)) {
-                       unget_char(state);
-                       tk.num = TK_error;
+                       if (ch == first) {
+                               tk.num = TK_string;
+                               break;
+                       }
+                       if (is_newline(ch)) {
+                               unget_char(state);
+                               break;
+                       }
                }
                close_token(state, &tk);
                return tk;
@@ -537,14 +542,16 @@ still parsed, but is discarded.
 
 #### Single line comments
 
-A single-line comment continues up to, but not including the newline.
+A single-line comment continues up to, but not including the newline
+or end of node.
 
 ###### parse comment
 
        if (is_line_comment(tk.txt)) {
-               while (!is_newline(ch))
+               while (!is_newline(ch) && !at_eon(state))
                        ch = get_char(state);
-               unget_char(state);
+               if (is_newline(ch))
+                       unget_char(state);
                close_token(state, &tk);
                tk.num = TK_line_comment;
                if (ignored & (1 << TK_line_comment))