}
###### white space
+ if (is_newline(ch))
+ state_check_node(state);
if (is_newline(ch) || (at_son(state) && ch <= ' ')) {
int newlines = 0;
- int was_son = at_son(state);
+ int was_nl = is_newline(ch);
if (ignored & (1<<TK_in)) {
if (!is_newline(ch))
continue;
return tk;
}
// Indents are needed, so check all white space.
- while (ch <= ' ' && !at_eon(state)) {
+ while (ch <= ' ' && ch != WEOF) {
if (is_newline(ch))
newlines += 1;
ch = get_char(state);
+ if (is_newline(ch))
+ state_check_node(state);
}
- if (at_eon(state)) {
- newlines += 1;
- } else
+ if (ch != WEOF)
unget_char(state);
state->delayed_lines = newlines;
- state->out_next = was_son;
+ state->out_next = !was_nl;
state->check_indent = 1;
continue;
}
state->indent_level < sizeof(state->indent_sizes)-1) {
state->indent_level += 1;
state->indent_sizes[state->indent_level] = state_indent(state);
- state->delayed_lines -= 1;
+ if (state->delayed_lines)
+ state->delayed_lines -= 1;
tk.num = TK_in;
return tk;
}
###### white space
if (ch == WEOF) {
- if (state->col) {
- state->col = 0;
- state->check_indent = 1;
- continue;
- }
tk.num = TK_eof;
return tk;
}
}
}
+ static void state_check_node(struct token_state *state)
+ {
+ if (!state->node)
+ return;
+ if (state->node->code.len > state->offset)
+ return;
+
+ do
+ state->node = state->node->next;
+ while (state->node && state->node->code.txt == NULL);
+ state->offset = 0;
+ state->prev_offset = 0;
+ state->strip_offset = 0;
+ state->col = 0;
+ if (state->node == NULL)
+ return;
+ state->line = state->node->line_no;
+ do_strip(state);
+ state->col = state->node->needs_strip;
+ state->strip_offset = state->offset;
+ }
+
static wint_t get_char(struct token_state *state)
{
wchar_t next;
size_t n;
mbstate_t mbstate;
+ state_check_node(state);
if (state->node == NULL)
return WEOF;
- if (state->node->code.len <= state->offset) {
- do
- state->node = state->node->next;
- while (state->node && state->node->code.txt == NULL);
- state->offset = 0;
- if (state->node == NULL)
- return WEOF;
- state->line = state->node->line_no;
- do_strip(state);
- state->col = state->node->needs_strip;
- state->strip_offset = state->offset;
- }
## before get_char
&mbstate);
if (n == -2 || n == 0) {
/* Not enough bytes - not really possible */
- next = '\n';
- state->offset = state->node->code.len;
+ next = '\n'; // NOTEST
+ state->offset = state->node->code.len; // NOTEST
} else if (n == -1) {
/* error */
- state->offset += 1;
- next = 0x7f; // an illegal character
+ state->offset += 1; // NOTEST
+ next = 0x7f; // an illegal character // NOTEST
} else
state->offset += n;
## number includes
## number functions
-###### File: number.h
+###### File: parse_number.h
int number_parse(mpq_t num, char tail[3], struct text tok);
###### File: scanner.mk
## string functions
## string main
-###### File: string.h
+###### File: parse_string.h
int string_parse(struct token *tok, char escape,
struct text *str, char tail[3]);
#include <getopt.h>
#include "mdcode.h"
#include "scanner.h"
- #include "number.h"
- #include "string.h"
+ #include "parse_number.h"
+ #include "parse_string.h"
static int errs;
static void pr_err(char *msg)
{ "ignore-block-comment", 0, NULL, 'C'},
{ "ignore-indent", 0, NULL, 'i'},
{ "file", 1, NULL, 'f'},
+ { "section", 1, NULL, 's'},
{ NULL, 0, NULL, 0},
};
- static const char options[] = "W:w:n:NIMSzclCif:";
+ static const char options[] = "W:w:n:NIMSzclCif:s:";
struct section *table, *s, *prev;
int opt;
+ char *section_name = NULL;
+ int section_found = 0;
setlocale(LC_ALL,"");
while ((opt = getopt_long(argc, argv, options, long_options, NULL))
case 'l': conf.ignored |= 1 << TK_newline; break;
case 'i': conf.ignored |= 1 << TK_in; break;
case 'f': filename = optarg; break;
+ case 's': section_name = optarg; break;
default: fprintf(stderr, "scanner: unknown option '%c'.\n",
opt);
exit(1);
for (s = table; s;
(code_free(s->code), prev = s, s = s->next, free(prev))) {
+ if (section_name &&
+ (s->section.len != strlen(section_name) ||
+ strncmp(s->section.txt, section_name, s->section.len) != 0))
+ continue;
+ if (section_name)
+ section_found = 1;
printf("Tokenizing: %.*s\n", s->section.len,
s->section.txt);
state = token_open(s->code, &conf);
}
if (conf.words_marks != known)
free(conf.words_marks);
+ if (section_name && !section_found) {
+ fprintf(stderr, "scanner: section %s not found\n", section_name);
+ errs = 1;
+ }
exit(!!errs);
}
###### File: scanner.mk