From b5bfd47c99b0014e3b5328ba41c914287d8517eb Mon Sep 17 00:00:00 2001
From: NeilBrown <neil@brown.name>
Date: Wed, 29 May 2019 18:25:50 +1000
Subject: [PATCH] scanner: handle completely unrecognized characters.

If we are ignoring numbers, then a digit looks like
nothing at all, not even an unknown mark.
Make sure to handle that properly.

Signed-off-by: NeilBrown <neil@brown.name>
---
 csrc/scanner.mdc | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index 6eee089..705e02e 100644
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -867,12 +867,23 @@ If the token we have is not empty and `TK_mark` is allowed,
 we have an unknown mark, otherwise this must be an error.
 
 ###### unknown mark
-	/* one unknown character */
+
+	/* one unknown mark character */
+	if (tk.txt.len) {
+		close_token(state, &tk);
+		if (ignored & (1<<TK_mark))
+			tk.num = TK_error;
+		else
+			tk.num = TK_mark;
+		return tk;
+	}
+	/* Completely unrecognised character is next, possibly
+	 * a digit and we are ignoring numbers.
+	 * What ever it is, make it an error.
+	 */
+	get_char(state);
 	close_token(state, &tk);
-	if (ignored & (1<<TK_mark))
-		tk.num = TK_error;
-	else
-		tk.num = TK_mark;
+	tk.num = TK_error;
 	return tk;
 
 ## Tools For The Task
-- 
2.43.0