parsergen: track when newline is permitted, and discard if not.

A newline is only permitted (as a recognised symbol) if we are
parsing a non-indented line-like segment.
If we have seen an internal indent since the last line-like start,
newline tokens should be ignored.

Signed-off-by: NeilBrown <neilb@suse.de>

diff --git a/csrc/parsergen.mdc b/csrc/parsergen.mdc
index 2cd4f5c84534903f114d696939cb698d125e1cd3..88c652a83d1de2df34350628a0594fd18091c3a0 100644 (file)
--- a/csrc/parsergen.mdc
+++ b/csrc/parsergen.mdc
@@ -2347,6 +2347,7 @@ pushing it onto the stack.
                        short sym;
                        short starts_indented;
                        short indents;
+                       short newline_permitted;
                } *stack, next;
                void **asn_stack;
                int stack_size;
@@ -2394,6 +2395,9 @@ if needed and pushed all the information onto the stacks.
                p->next.state = newstate;
                p->next.indents = 0;
                p->next.starts_indented = 0;
+               // if new state doesn't start a line, we inherit newline_permitted status
+               if (states[newstate].starts_line)
+                       p->next.newline_permitted = 1;
                return 1;
        }
 
@@ -2417,6 +2421,9 @@ reduce a production, just before we `shift` the nonterminal in.
                if (num) {
                        p->next.state = p->stack[p->tos].state;
                        p->next.starts_indented = p->stack[p->tos].starts_indented;
+                       p->next.newline_permitted = p->stack[p->tos].newline_permitted;
+                       if (p->next.indents > p->next.starts_indented)
+                               p->next.newline_permitted = 0;
                }
        }
 
@@ -2507,6 +2514,13 @@ at the start of a production.
                                    (p.stack[p.tos-1].indents == 1 &&
                                     states[p.next.state].reduce_size > 1)) {
                                        p.stack[p.tos-1].indents -= 1;
+                                       if (p.stack[p.tos-1].indents == p.stack[p.tos-1].starts_indented) {
+                                               // no internal indent any more, reassess 'newline_permitted'
+                                               if (states[p.stack[p.tos-1].state].starts_line)
+                                                       p.stack[p.tos-1].newline_permitted = 1;
+                                               else if (p.tos > 1)
+                                                       p.stack[p.tos-1].newline_permitted = p.stack[p.tos-2].newline_permitted;
+                                       }
                                        free(tk);
                                        tk = NULL;
                                        continue;
@@ -2514,6 +2528,13 @@ at the start of a production.
                                // fall through and force a REDUCE (as 'shift'
                                // will fail).
                        }
+                       if (p.next.sym == TK_newline) {
+                               if (! p.stack[p.tos-1].newline_permitted) {
+                                       free(tk);
+                                       tk = NULL;
+                                       continue;
+                               }
+                       }
                        if (shift(&p, tk, states)) {
                                tk = NULL;
                                continue;