ASCII subset. If the text provided does not conform to UTF-8 an error
will be reported and some number of bytes will be skipped.
-###### includes
+###### public types
#include <wchar.h>
#include <wctype.h>
#include <unicode/uchar.h>
int sign_ok = 0;
switch(expect_p) {
case 0:
- if (ch == 'e')
+ if (ch == 'e' || ch == 'E')
sign_ok = 1;
break;
case 1:
- if (ch == 'p')
+ if (ch == 'p' || ch == 'P')
sign_ok = 1;
break;
}
If no known mark is found we will test against strings and comments
below before giving up and assuming an unknown mark.
+
+If an unknown mark contains a quote character or a comment marker, and
+that token is not being ignored, then we terminate the unknown mark
+before that quote or comment. This ensure that an unknown mark
+immediately before a string is handled correctly.
+
If `TK_mark` is ignored, then unknown marks as returned as an error.
###### token types
tk.num = TK_error;
while (is_mark(ch, state->conf)) {
int n;
+ wchar_t prev;
close_token(state, &tk);
n = find_known(state->conf, tk.txt);
if (n >= 0)
close_token(state, &tk);
return tk;
}
+ prev = ch;
+ save_unget_state(state);
ch = get_char(state);
+ if (!(ignored && (1<<TK_string)) && is_quote(ch))
+ break;
+ if (prev == '#')
+ break;
+ if (prev == '/' && ch == '/' && tk.txt.len > 1) {
+ restore_unget_state(state);
+ break;
+ }
+ if (prev == '/' && ch == '*' && tk.txt.len > 1) {
+ restore_unget_state(state);
+ break;
+ }
}
unget_char(state);
- if (tk.num != TK_error)
+ if (tk.num != TK_error) {
+ close_token(state, &tk);
return tk;
+ }
+
+If we don't find a known mark, we will check for strings and comments
+before assuming that we have an unknown mark
+
+###### parse mark
+ ## parse string
+ ## parse comment
+ ## unknown mark
###### unknown mark
if (tk.txt.len) {
* unget so the newline is seen,
* but return rest of string as an error.
*/
- unget_char(state);
+ if (is_newline(ch))
+ unget_char(state);
close_token(state, &tk);
tk.num = TK_error;
return tk;
!(ignored & (1<<TK_string))) {
wchar_t first = tk.txt.txt[0];
reset_token(state, &tk);
- get_char(state);
- do
+ ch = get_char(state);
+ tk.num = TK_error;
+ while (!at_eon(state) && !is_newline(ch)) {
ch = get_char(state);
- while (ch != first && !is_newline(ch));
- tk.num = TK_string;
- if (is_newline(ch)) {
- unget_char(state);
- tk.num = TK_error;
+ if (ch == first) {
+ tk.num = TK_string;
+ break;
+ }
+ if (is_newline(ch)) {
+ unget_char(state);
+ break;
+ }
}
close_token(state, &tk);
return tk;
These two comment types are reported as two separate token types, and
consequently can be ignored separately. When ignored a comment is
-parsed and discarded.
+still parsed, but is discarded.
###### token types
TK_line_comment,
#### Single line comments
-A single-line comment continues up to, but not including the newline.
+A single-line comment continues up to, but not including the newline
+or end of node.
###### parse comment
if (is_line_comment(tk.txt)) {
- while (!is_newline(ch))
+ while (!is_newline(ch) && !at_eon(state))
ch = get_char(state);
- unget_char(state);
+ if (is_newline(ch))
+ unget_char(state);
close_token(state, &tk);
tk.num = TK_line_comment;
if (ignored & (1 << TK_line_comment))
If a line starts with more white-space than the previous non-blank
line - or if the first non-blank line in the document starts with any
-white-space - then an Indent is reported at the start of the line.
+white-space - then an "IN" is reported at the start of the line.
Before the next non-blank line which starts with less white space, or
-at the latest at the end of the document, a matching Undent token
-is reported. There will always be an exact match between Indent and
-Undent tokens.
+at the latest at the end of the document, a matching "OUT" token
+is reported. There will always be an exact match between "IN" and
+"OUT" tokens.
-It is possible for Undent to be followed (almost) immediately by an
-Indent. This happens if, for example, the indent of three consecutive
+It is possible for "OUT" to be followed (almost) immediately by an
+"IN". This happens if, for example, the indent of three consecutive
lines are 0, 8, 4 spaces. Before the second line we report an
-Indent. Before the third line we must report an Undent, as 4 is less
+"IN". Before the third line we must report an "OUT", as 4 is less
than 8, then also an Ident as 4 is greater than 0.
###### token types
- TK_indent,
- TK_undent,
+ TK_in,
+ TK_out,
For the purpose of measuring the length of white space, a tab adds at
least one space, and rounds up to a multiple of 8.
must be followed immediately by a newline so these constructs cannot
hide the fact that a newline was present.
-When Indents are being reported, the Newline which would normally be
-reported immediately before the Indent is delayed until after the
-matching undent. This makes an indented section act like a
+When indents are being reported, the Newline which would normally be
+reported immediately before the "IN" is delayed until after the
+matching "OUT". This makes an indented section act like a
continuation of the previous line to some extent.
A blank line would normally be reported simply as two consecutive Newline
delayed - but how many Newlines?
The approach we will take is to report the extra Newlines immediately after
-the Indent token, so the blank line is treated as though it were an indented
+the IN token, so the blank line is treated as though it were an indented
blank line.
###### token types
If we find a newline or white space at the start of a block, we keep
collecting spaces, tabs, and newlines until we find some real text.
Then depending on the indent we generate some number of tokens. These
-will be a sequence of "Newline Undent" pairs representing a decrease
-in indent, then either a Newline or an Indent depending on whether the
+will be a sequence of "Newline OUT" pairs representing a decrease
+in indent, then either a Newline or an IN depending on whether the
next line is indented, then zero or more Newlines representing all the
blank lines that have been skipped.
When a Newline leads to the next block of code there is a question of
-whether the various Newline and Undent/Indent tokens should appear to
+whether the various Newline and OUT/IN tokens should appear to
pbelong to the earlier or later block. This is addressed by processing
the tokens in two stages based on the relative indent levels of the
two blocks (each block has a base indent to which the actual indents
are added).
-Any "Newline Undent" pairs needed to reduce the current indent to the
+Any "Newline OUT" pairs needed to reduce the current indent to the
maximum of the base indents of the old and new blocks are generated
against the old block. Then if the next block does not have an
increased indent, one more "Newline" is generated.
-If further "Newline Undent" pairs are needed to get to the indent
+If further "Newline OUT" pairs are needed to get to the indent
level of the 'next' block, they are generated against that block,
though the first Newline is suppressed (it having already been
generated).
-Finally the Newline or Indent for the first line of the new block is
+Finally the Newline or IN for the first line of the new block is
generated, unless the Newline needs to be suppressed because it
appeared at the end of the previous block.
-This means that a block may start with an Undent or an Indent, but
+This means that a block may start with an OUT or an IN, but
will only start with a Newline if it actually starts with a blank
line.
We will need to represent in the `token_state` where in this sequence
of delayed tokens we are. As `state.col` records the target indent we
-don't need to record how many undents or indents are needed. We do
+don't need to record how many OUTs or INs are needed. We do
need to record the number of blank lines, and which of Newline and
-Undent is needed next in the initial sequence of pairs.
+OUT is needed next in the initial sequence of pairs.
For this we store one more than the number of blank lines as
-`delayed_lines` and a flag for `undent_next`.
+`delayed_lines` and a flag for `out_next`.
###### state fields
int check_indent;
int delayed_lines;
- int undent_next;
+ int out_next;
Generating these tokens involve two separate pieces of code.
if (is_newline(ch) || (at_son(state) && ch <= ' ')) {
int newlines = 0;
int was_son = at_son(state);
- if (ignored & (1<<TK_indent)) {
+ if (ignored & (1<<TK_in)) {
if (!is_newline(ch))
continue;
if (ignored & (1<<TK_newline))
continue;
tk.num = TK_newline;
+ close_token(state, &tk);
return tk;
}
// Indents are needed, so check all white space.
} else
unget_char(state);
state->delayed_lines = newlines;
- state->undent_next = was_son;
+ state->out_next = was_son;
state->check_indent = 1;
continue;
}
if (state->check_indent || state->delayed_lines) {
if (state->col < state->indent_sizes[state->indent_level]) {
- if (!state->undent_next &&
+ if (!state->out_next &&
!(ignored & (1<<TK_newline))) {
- state->undent_next = 1;
+ state->out_next = 1;
tk.num = TK_newline;
return tk;
}
state->indent_level -= 1;
- state->undent_next = 0;
- tk.num = TK_undent;
+ state->out_next = 0;
+ tk.num = TK_out;
return tk;
}
if (state->col > state->indent_sizes[state->indent_level] &&
state->indent_level += 1;
state->indent_sizes[state->indent_level] = state->col;
state->delayed_lines -= 1;
- tk.num = TK_indent;
+ tk.num = TK_in;
return tk;
}
state->check_indent = 0;
state->offset += 1;
n -= 1;
}
- while (n == 4 && state->node->code.txt[0] == '\t') {
+ while (n == 4 && state->node->code.txt[state->offset] == '\t') {
state->offset += 1;
n -= 4;
}
Now we have all the bits there is just one section missing: combining
all the token parsing code into one block.
-The handling of delayed tokens (newlines, indents, undents) must come
+The handling of delayed tokens (Newlines, INs, OUTs) must come
first before we try getting another character.
Then we parse all the test, making sure that we check for known marks
## parse number
## parse word
## parse mark
- ## parse string
- ## parse comment
- ## unknown mark
### Start and stop
memset(state, 0, sizeof(*state));
state->node = code;
state->line = code->line_no;
+ state->col = code->indent;
state->conf = conf;
+ do_strip(state);
return state;
}
void token_close(struct token_state *state)
[TK_multi_string] = "mstring",
[TK_line_comment] = "lcomment",
[TK_block_comment] = "bcomment",
- [TK_indent] = "indent",
- [TK_undent] = "undent",
+ [TK_in] = "in",
+ [TK_out] = "out",
[TK_newline] = "newline",
[TK_eof] = "eof",
[TK_error] = "ERROR",
default: /* known word or mark */
fprintf(f, "%.*s", tok.txt.len, tok.txt.txt);
break;
- case TK_indent:
- case TK_undent:
+ case TK_in:
+ case TK_out:
case TK_newline:
case TK_eof:
/* No token text included */
mpq_set_ui(tens, 10, 1);
while (1) {
if (lexp & 1) {
- if (esign > 1)
+ if (esign > 0)
mpq_mul(num, num, tens);
else
mpq_div(num, num, tens);
code / ocean / blobdiff