scanner: fix handling of indents in sub-nodes

I seem to have confused ->indent_sizes[] and ->col

->col is used for the reported location of a token so
must be the actual column in the file, with no adjustment.

->indent_sizes[] is indents, which must include any inherited from
parent nodes.  So this is a completely different value.

So change mdcode to store the local node indent in ->needs_strip -
this is the number of text columns that are stripped off.
This, subtracted from ->indent is the text offset of the physical
start-of-line.  Adding the measured ->col then gives us
the indent in the composed file, the indent that must be used
for detecting TK_in and TK_out.

Introduce a new function state_indent() which determines that indent,
and use it instead of ->col.

Signed-off-by: NeilBrown <neil@brown.name>

diff --git a/csrc/mdcode.mdc b/csrc/mdcode.mdc
index 2789671a6bf0ba9780dfeec773965d4b624fbabe..cf6887dea25c316bab6bc6679ce2f8e2d622a046 100644 (file)
--- a/csrc/mdcode.mdc
+++ b/csrc/mdcode.mdc
@@ -161,8 +161,11 @@ the root.
 
 Finally we need to know if the `code_node` was recognised by being
 indented or not.  If it was, the client of this data will want to
 
 Finally we need to know if the `code_node` was recognised by being
 indented or not.  If it was, the client of this data will want to

-strip of the leading tab or 4 spaces.  Hence a `needs_strip` flag is
-needed.
+strip off the leading tab or 4 spaces.  Hence a `needs_strip` flag is
+needed.  This will be set to 8 if a tab is found and 4 if four spaces are found.
+This means the relative indent of text in the node
+is `node->indent - node->needs_strip`.
+The relative indent is needed for detecting indents in the overall file.

 
 ##### exported types
 
 
 ##### exported types
 


@@ -287,7 +290,13 @@ in a new node.
                n->code = txt;
                n->indent = 0;
                n->line_no = line_no;
                n->code = txt;
                n->indent = 0;
                n->line_no = line_no;

-               n->needs_strip = needs_strip;
+               if (needs_strip) {
+                       if (txt.txt[0] == '\t')
+                               n->needs_strip = 8;
+                       else
+                               n->needs_strip = 4;
+               } else
+                       n->needs_strip = 0;

                n->next = NULL;
                n->child = NULL;
                if (where->last)
                n->next = NULL;
                n->child = NULL;
                if (where->last)

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index fa27a89524a85d4eeb9a16019558e61a993afb5f..21ec828c1d36dea273cabe05c3a3dce3844c29a2 100644 (file)
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -716,6 +716,8 @@ ignored.
        int     indent_level;
        int     indent_sizes[20];
 
        int     indent_level;
        int     indent_sizes[20];
 

+`indent_sizes[0]` will always be zero - this simplifies some code.
+

 #### Newlines
 
 Newlines can optionally be reported.  Newlines within a block comment
 #### Newlines
 
 Newlines can optionally be reported.  Newlines within a block comment


@@ -796,6 +798,14 @@ Separately we need, on each call to `token_next`, to check if
 there are some delayed tokens and if so we need to advance the state
 information and return one token.
 
 there are some delayed tokens and if so we need to advance the state
 information and return one token.
 

+###### internal functions
+       static int state_indent(struct token_state *state)
+       {
+               if (state->node == NULL)
+                       return state->col;
+               return state->node->indent - state->node->needs_strip + state->col;
+       }
+

 ###### white space
        if (is_newline(ch) || (at_son(state) && ch <= ' ')) {
                int newlines = 0;
 ###### white space
        if (is_newline(ch) || (at_son(state) && ch <= ' ')) {
                int newlines = 0;


@@ -817,11 +827,6 @@ information and return one token.
                }
                if (at_eon(state)) {
                        newlines += 1;
                }
                if (at_eon(state)) {
                        newlines += 1;

-                       if (state->node->next &&
-                           state->node->next->indent > state->node->indent)
-                               state->col = state->node->next->indent;
-                       else
-                               state->col = state->node->indent;

                } else
                        unget_char(state);
                state->delayed_lines = newlines;
                } else
                        unget_char(state);
                state->delayed_lines = newlines;


@@ -833,7 +838,7 @@ information and return one token.
 ###### delayed tokens
 
        if (state->check_indent || state->delayed_lines) {
 ###### delayed tokens
 
        if (state->check_indent || state->delayed_lines) {

-               if (state->col < state->indent_sizes[state->indent_level]) {
+               if (state_indent(state) < state->indent_sizes[state->indent_level]) {

                        if (!state->out_next &&
                            !(ignored & (1<<TK_newline))) {
                                state->out_next = 1;
                        if (!state->out_next &&
                            !(ignored & (1<<TK_newline))) {
                                state->out_next = 1;


@@ -845,10 +850,10 @@ information and return one token.
                        tk.num = TK_out;
                        return tk;
                }
                        tk.num = TK_out;
                        return tk;
                }

-               if (state->col > state->indent_sizes[state->indent_level] &&
+               if (state_indent(state) > state->indent_sizes[state->indent_level] &&

                    state->indent_level < sizeof(state->indent_sizes)-1) {
                        state->indent_level += 1;
                    state->indent_level < sizeof(state->indent_sizes)-1) {
                        state->indent_level += 1;

-                       state->indent_sizes[state->indent_level] = state->col;
+                       state->indent_sizes[state->indent_level] = state_indent(state);

                        state->delayed_lines -= 1;
                        tk.num = TK_in;
                        return tk;
                        state->delayed_lines -= 1;
                        tk.num = TK_in;
                        return tk;


@@ -938,7 +943,7 @@ a flag that tells us whether or not we need to strip.
 
 ###### internal functions
 
 
 ###### internal functions
 

-       static int do_strip(struct token_state *state)
+       static void do_strip(struct token_state *state)

        {
                int indent = 0;
                if (state->node->needs_strip) {
        {
                int indent = 0;
                if (state->node->needs_strip) {


@@ -954,7 +959,6 @@ a flag that tells us whether or not we need to strip.
                                n -= 4;
                        }
                }
                                n -= 4;
                        }
                }

-               return indent;

        }
 
        static wint_t get_char(struct token_state *state)
        }
 
        static wint_t get_char(struct token_state *state)


@@ -973,7 +977,8 @@ a flag that tells us whether or not we need to strip.
                        if (state->node == NULL)
                                return WEOF;
                        state->line = state->node->line_no;
                        if (state->node == NULL)
                                return WEOF;
                        state->line = state->node->line_no;

-                       state->col = do_strip(state);
+                       do_strip(state);
+                       state->col = state->node->needs_strip;

                        state->strip_offset = state->offset;
                }
 
                        state->strip_offset = state->offset;
                }
 


@@ -999,7 +1004,8 @@ a flag that tells us whether or not we need to strip.
                        state->col += 1;
                } else if (is_newline(next)) {
                        state->line += 1;
                        state->col += 1;
                } else if (is_newline(next)) {
                        state->line += 1;

-                       state->col = do_strip(state);
+                       do_strip(state);
+                       state->col = state->node->needs_strip;

                } else if (next == '\t') {
                        state->col = indent_tab(state->col);
                }
                } else if (next == '\t') {
                        state->col = indent_tab(state->col);
                }


@@ -1211,7 +1217,8 @@ As well as getting tokens, we need to be able to create the
                memset(state, 0, sizeof(*state));
                state->node = code;
                state->line = code->line_no;
                memset(state, 0, sizeof(*state));
                state->node = code;
                state->line = code->line_no;

-               state->col = do_strip(state);
+               do_strip(state);
+               state->col = state->node->needs_strip;

                state->strip_offset = state->offset;
                state->conf = conf;
                return state;
                state->strip_offset = state->offset;
                state->conf = conf;
                return state;