parsergen: move EOL handling out of shift()

shift() is called in several places, and in only one of those is there a
need to make NEWLINE to EOL.

Move the code out of shift() and instead make a second call to shift()
if shifting NEWLINE failed.  I think this make the code clearer.

Signed-off-by: NeilBrown <neil@brown.name>

diff --git a/csrc/parsergen.mdc b/csrc/parsergen.mdc
index 709a9acb962e8be130030d7d7d183c2fb3222ed2..d9467442eeab1dfec4720f7a2b4f5c6666d43867 100644 (file)
--- a/csrc/parsergen.mdc
+++ b/csrc/parsergen.mdc
@@ -2688,25 +2688,6 @@ stack is empty, it always chooses zero as the next state.
 So `shift` finds the next state.  If that succeeds it extends the
 allocations if needed and pushes all the information onto the stacks.
 
-An extra complication is added to `shift` by the `EOL` token.  This
-token must be generated when a `NEWLINE` is seen, but an `EOL` is
-expected.  When this happens, the `NEWLINE` is NOT consumed, so multiple
-EOL can appear before a NEWLINE.  To indicate that the token was shifted
-by not consumed, `shift` can return the special value `2`.  The token
-number for `EOL` cannot be statically declared, so when the parser
-starts we need to look through the array of non-terminals to find the
-EOL.
-
-###### parser state
-       int tk_eol;
-
-###### find eol
-       p.tk_eol = 0;
-       while (strcmp(non_term[p.tk_eol], "EOL") != 0)
-               p.tk_eol += 1;
-       p.tk_eol += TK_reserved + config->known_count;
-
-
 ###### parser functions
 
        static int shift(struct parser *p,
@@ -2714,27 +2695,12 @@ EOL.
                         const struct state states[])
        {
                struct frame next = {0};
-               int ret;
                int newstate = p->tos
                        ? search(&states[p->stack[p->tos-1].state],
                                 sym)
                        : 0;
-               if (newstate >= 0)
-                       ret = 1;
-               else if (sym != TK_newline)
+               if (newstate < 0)
                        return 0;
-               else {
-                       // have a NEWLINE, might need an EOL
-                       sym = p->tk_eol;
-                       newstate = p->tos
-                               ? search(&states[p->stack[p->tos-1].state],
-                                        sym)
-                               : 0;
-                       if (newstate < 0)
-                               return 0;
-                       ret = 2;
-                       asn = tok_copy(*(struct token*)asn);
-               }
 
                if (p->tos >= p->stack_size) {
                        p->stack_size += 10;
@@ -2749,7 +2715,7 @@ EOL.
                p->stack[p->tos] = next;
                p->asn_stack[p->tos] = asn;
                p->tos++;
-               return ret;
+               return 1;
        }
 
 `pop` primarily moves the top of stack (`tos`) back down the required
@@ -2807,9 +2773,27 @@ of indentation.
 ###### parser state
        unsigned long ignored_indents;
 
-NEWLINE/EOL is ignored when in an indented section of text which was not
+NEWLINE is ignored when in an indented section of text which was not
 explicitly expected by the grammar.  So if the most recent indent is
-ignored, so is any EOL token.
+ignored, so is any NEWLINE token.
+
+If a NEWLINE is seen but it cannot be shifted, we try to shift an EOL
+token instead.  If that succeeds, we make a new copy of the NEWLINE
+token and continue.  This allows a NEWLINE to appear to be preceded by
+an indefinite number of EOL tokens.
+
+The token number for `EOL` cannot be statically declared, so when the
+parser starts we need to look through the array of non-terminals to find
+the EOL.
+
+###### parser state
+       int tk_eol;
+
+###### find eol
+       p.tk_eol = 0;
+       while (strcmp(non_term[p.tk_eol], "EOL") != 0)
+               p.tk_eol += 1;
+       p.tk_eol += TK_reserved + config->known_count;
 
 For other tokens, we shift the next token if that is possible, otherwise
 we try to reduce a production.
@@ -2827,19 +2811,21 @@ we try to reduce a production.
                continue;
        }
 
-       switch (shift(&p, tk->num, tk, states)) {
-       case 1:
+       if (shift(&p, tk->num, tk, states)) {
                if (tk->num == TK_out)
                        p.ignored_indents >>= 1;
                if (tk->num == TK_in)
                        p.ignored_indents <<= 1;
 
+               parser_trace_action(trace, "Shift");
                tk = NULL;
-               /* fallthrough */
-       case 2:
-               parser_trace_action(trace, tk ? "ShiftEOL" : "Shift");
                ## did shift
                continue;
+       } else if (tk->num == TK_newline &&
+                  shift(&p, p.tk_eol, tk, states)) {
+               tk = tok_copy(*tk);
+               parser_trace_action(trace, "ShiftEOL");
+               continue;
        }
 
        if (tk->num == TK_in && states[p.stack[p.tos-1].state].go_to_cnt > 0) {