scanner: change the meaning of ignoring comment tokens.

[ocean] / csrc / parsergen.mdc

diff --git a/csrc/parsergen.mdc b/csrc/parsergen.mdc
index 76fec3a70a779cefe4589a9d79e861a61acd726e..3f36df9a6873c3c94f2f127d8a0ca6419e50603c 100644 (file)
--- a/csrc/parsergen.mdc
+++ b/csrc/parsergen.mdc
@@ -636,6 +636,11 @@ to produce errors that the parser is better positioned to handle.
                        } else if (tk.num == TK_mark
                                   && text_is(tk.txt, "$*")) {
                                err = dollar_line(state, g, 1);
+                       } else if (tk.num == TK_mark
+                                  && text_is(tk.txt, "//")) {
+                               while (tk.num != TK_newline &&
+                                      tk.num != TK_eof)
+                                       tk = token_next(state);
                        } else {
                                err = "Unrecognised token at start of line.";
                        }
@@ -1334,11 +1339,11 @@ into the go to set, so the item is ineffective.
                        continue;
                if (symset_find(&done, s->num) < 0) {
                        symset_add(&done, s->num, 0);
-                       if (s->line_like)
-                               is->starts_line = 1;
                }
                if (s->type != Nonterminal)
                        continue;
+               if (s->line_like)
+                       is->starts_line = 1;
                again = 1;
                if (type >= LALR) {
                        // Need the LA set.
@@ -1911,7 +1916,6 @@ pieces of code provided in the grammar file, so they are generated first.
                fprintf(f, "\tstruct token_state *tokens;\n");
                fprintf(f, "\tconfig->words_marks = known;\n");
                fprintf(f, "\tconfig->known_count = sizeof(known)/sizeof(known[0]);\n");
-               fprintf(f, "\tconfig->ignored |= (1 << TK_line_comment) | (1 << TK_block_comment);\n");
                fprintf(f, "\ttokens = token_open(code, config);\n");
                fprintf(f, "\tvoid *rv = parser_run(tokens, states, do_reduce, do_free, trace, non_term, config);\n");
                fprintf(f, "\ttoken_close(tokens);\n");
@@ -2871,9 +2875,13 @@ checks if a given token is in any of these look-ahead sets.
                force_reduce:
                        if (states[tos->state].reduce_prod >= 0 &&
                            states[tos->state].newline_only &&
-                           tk->num != TK_newline && tk->num != TK_eof && tk->num != TK_out) {
-                               /* Anything other than newline in an error as this
-                                * production must end at EOL
+                           !(tk->num == TK_newline ||
+                             tk->num == TK_eof ||
+                             tk->num == TK_out ||
+                             (tos->indents == 0 && tos->since_newline == 0))) {
+                               /* Anything other than newline or out or eof
+                                * in an error unless we are already at start
+                                * of line, as this production must end at EOL.
                                 */
                        } else if (states[tos->state].reduce_prod >= 0) {
                                void **body;
@@ -3025,7 +3033,7 @@ end inside square brackets.
                        fputs(reserved_words[tk->num], trace);
                else
                        text_dump(trace, tk->txt, 20);
-               fputs("]", trace);
+               fprintf(trace, ":%d:%d]", tk->line, tk->col);
        }
 
        void parser_trace_action(FILE *trace, char *action)
@@ -3056,7 +3064,7 @@ an error.
 
 # calc: header
 
-       #include "number.h"
+       #include "parse_number.h"
        // what do we use for a demo-grammar?  A calculator of course.
        struct number {
                mpq_t val;
@@ -3076,7 +3084,6 @@ an error.
        #include <string.h>
        #include "mdcode.h"
        #include "scanner.h"
-       #include "number.h"
        #include "parser.h"
 
        #include "calc.h"
@@ -3102,7 +3109,6 @@ an error.
                struct section *s;
                struct token_config config = {
                        .ignored = (1 << TK_line_comment)
-                                | (1 << TK_block_comment)
                                 | (1 << TK_in)
                                 | (1 << TK_out),
                        .number_chars = ".,_+-",
@@ -3124,7 +3130,7 @@ an error.
 # calc: grammar
 
        $LEFT + -
-       $LEFT * /
+       $LEFT * / //
 
        Session -> Session Line
                | Line
@@ -3152,6 +3158,16 @@ an error.
                | Expression - Expression ${ mpq_init($0.val); mpq_sub($0.val, $1.val, $3.val); }$
                | Expression * Expression ${ mpq_init($0.val); mpq_mul($0.val, $1.val, $3.val); }$
                | Expression / Expression ${ mpq_init($0.val); mpq_div($0.val, $1.val, $3.val); }$
+               | Expression // Expression ${ {
+                       mpz_t z0, z1, z2;
+                       mpq_init($0.val);
+                       mpz_init(z0); mpz_init(z1); mpz_init(z2);
+                       mpz_tdiv_q(z1, mpq_numref($1.val), mpq_denref($1.val));
+                       mpz_tdiv_q(z2, mpq_numref($3.val), mpq_denref($3.val));
+                       mpz_tdiv_q(z0, z1, z2);
+                       mpq_set_z($0.val, z0);
+                       mpz_clear(z0); mpz_clear(z1); mpz_clear(z2);
+               } }$
                | NUMBER ${ if (number_parse($0.val, $0.tail, $1.txt) == 0) mpq_init($0.val); }$
                | ( Expression ) ${ mpq_init($0.val); mpq_set($0.val, $2.val); }$
 
@@ -3164,4 +3180,6 @@ an error.
        10 * 9 / 2
        1 * 1000 + 2 * 100 + 3 * 10 + 4 * 1
 
+       355//113
+
        error