parsergen: make it easier to test the simple 'calc' code.

[ocean] / csrc / scanner.mdc

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index a5eeb1f128b5163b1715b263a033a3a64265c73a..15306dd17494245986dbd014ebd2a2c8693c977d 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -520,6 +520,10 @@ If `TK_string` is ignored, then quote characters will appear as `TK_mark`s.
                                break;
                        }
                }
+               while (!at_eon(state) && (ch = get_char(state)) &&
+                                         iswalpha(ch))
+                       ;
+               unget_char(state);
                close_token(state, &tk);
                return tk;
        }
@@ -1064,8 +1068,11 @@ parsed too much already.  For that there is `reset_token`.
        static void close_token(struct token_state *state,
                                struct token *tk)
        {
-               tk->txt.len = (state->node->code.txt + state->offset)
-                             - tk->txt.txt;
+               if (state->node != tk->node)
+                       tk->txt.len = tk->node->code.len - (tk->txt.txt - tk->node->code.txt);
+               else
+                       tk->txt.len = (state->node->code.txt + state->offset)
+                                     - tk->txt.txt;
        }
 
        static void reset_token(struct token_state *state, struct token *tok)