scanner: improve transition from node to node.

[ocean] / csrc / scanner.mdc

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index 9a040e1c39e97e1205dcccb5208919eb6b4203ad..777688482566796691d97f6df6cd3e8e5cf9e863 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -807,9 +807,11 @@ information and return one token.
        }
 
 ###### white space
+       if (is_newline(ch))
+               state_check_node(state);
        if (is_newline(ch) || (at_son(state) && ch <= ' ')) {
                int newlines = 0;
-               int was_son = at_son(state);
+               int was_nl = is_newline(ch);
                if (ignored & (1<<TK_in)) {
                        if (!is_newline(ch))
                                continue;
@@ -820,17 +822,17 @@ information and return one token.
                        return tk;
                }
                // Indents are needed, so check all white space.
-               while (ch <= ' ' && !at_eon(state)) {
+               while (ch <= ' ' && ch != WEOF) {
                        if (is_newline(ch))
                                newlines += 1;
                        ch = get_char(state);
+                       if (is_newline(ch))
+                               state_check_node(state);
                }
-               if (at_eon(state)) {
-                       newlines += 1;
-               } else
+               if (ch != WEOF)
                        unget_char(state);
                state->delayed_lines = newlines;
-               state->out_next = was_son;
+               state->out_next = !was_nl;
                state->check_indent = 1;
                continue;
        }
@@ -962,26 +964,37 @@ a flag that tells us whether or not we need to strip.
                }
        }
 
+       static void state_check_node(struct token_state *state)
+       {
+               if (!state->node)
+                       return;
+               if (state->node->code.len > state->offset)
+                       return;
+
+               do
+                       state->node = state->node->next;
+               while (state->node && state->node->code.txt == NULL);
+               state->offset = 0;
+               state->prev_offset = 0;
+               state->strip_offset = 0;
+               state->col = 0;
+               if (state->node == NULL)
+                       return;
+               state->line = state->node->line_no;
+               do_strip(state);
+               state->col = state->node->needs_strip;
+               state->strip_offset = state->offset;
+       }
+
        static wint_t get_char(struct token_state *state)
        {
                wchar_t next;
                size_t n;
                mbstate_t mbstate;
 
+               state_check_node(state);
                if (state->node == NULL)
                        return WEOF;
-               if (state->node->code.len <= state->offset) {
-                       do
-                               state->node = state->node->next;
-                       while (state->node && state->node->code.txt == NULL);
-                       state->offset = 0;
-                       if (state->node == NULL)
-                               return WEOF;
-                       state->line = state->node->line_no;
-                       do_strip(state);
-                       state->col = state->node->needs_strip;
-                       state->strip_offset = state->offset;
-               }
 
                ## before get_char