From: NeilBrown <neil@brown.name>
Date: Tue, 3 Jun 2014 11:15:45 +0000 (+1000)
Subject: Scanner: parsing of comments and strings must recognise end-of-node
X-Git-Tag: workingparser~50
X-Git-Url: https://ocean-lang.org/code/?p=ocean;a=commitdiff_plain;h=f2bea0fc0f861d28485ff4ef59974461808599a9

Scanner: parsing of comments and strings must recognise end-of-node

If the file does not end in a newline, then a node might not also.

This requires a little more care in parsing strings and comments.

Signed-off-by: NeilBrown <neil@brown.name>
---

diff --git a/csrc/scanner.mdc b/csrc/scanner.mdc
index ecc202b..37b336f 100644
--- a/csrc/scanner.mdc
+++ b/csrc/scanner.mdc
@@ -450,7 +450,8 @@ followed by the start of a new string.
 			 * unget so the newline is seen,
 			 * but return rest of string as an error.
 			 */
-			unget_char(state);
+			if (is_newline(ch))
+				unget_char(state);
 			close_token(state, &tk);
 			tk.num = TK_error;
 			return tk;
@@ -484,14 +485,18 @@ If `TK_string` is ignored, then quote characters will appear as `TK_mark`s.
 	    !(ignored & (1<<TK_string))) {
 		wchar_t first = tk.txt.txt[0];
 		reset_token(state, &tk);
-		get_char(state);
-		do
+		ch = get_char(state);
+		tk.num = TK_error;
+		while (!at_eon(state) && !is_newline(ch)) {
 			ch = get_char(state);
-		while (ch != first && !is_newline(ch));
-		tk.num = TK_string;
-		if (is_newline(ch)) {
-			unget_char(state);
-			tk.num = TK_error;
+			if (ch == first) {
+				tk.num = TK_string;
+				break;
+			}
+			if (is_newline(ch)) {
+				unget_char(state);
+				break;
+			}
 		}
 		close_token(state, &tk);
 		return tk;
@@ -537,14 +542,16 @@ still parsed, but is discarded.
 
 #### Single line comments
 
-A single-line comment continues up to, but not including the newline.
+A single-line comment continues up to, but not including the newline
+or end of node.
 
 ###### parse comment
 
 	if (is_line_comment(tk.txt)) {
-		while (!is_newline(ch))
+		while (!is_newline(ch) && !at_eon(state))
 			ch = get_char(state);
-		unget_char(state);
+		if (is_newline(ch))
+			unget_char(state);
 		close_token(state, &tk);
 		tk.num = TK_line_comment;
 		if (ignored & (1 << TK_line_comment))