scanner: handle missing newline at EOF

[ocean] / csrc / scanner.mdc

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index 0d7a62d05127697cfdfed7ff207dd923de60c69c..15306dd17494245986dbd014ebd2a2c8693c977d 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -1068,8 +1068,11 @@ parsed too much already.  For that there is `reset_token`.
        static void close_token(struct token_state *state,
                                struct token *tk)
        {
-               tk->txt.len = (state->node->code.txt + state->offset)
-                             - tk->txt.txt;
+               if (state->node != tk->node)
+                       tk->txt.len = tk->node->code.len - (tk->txt.txt - tk->node->code.txt);
+               else
+                       tk->txt.len = (state->node->code.txt + state->offset)
+                                     - tk->txt.txt;
        }
 
        static void reset_token(struct token_state *state, struct token *tok)