tests:: oceani_test_suite
oceani_test_suite: oceani coverage_oceani
+ @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
+ { echo "Grammar container conflicts, please review" ; exit 1; }
@rm -rf coverage; mkdir -p coverage
@cp *.gcno coverage
@for T in $(oceani_tests); do \
if ! cmp -s .tmp.want .tmp.have; then \
echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
echo -n "passed ... "; \
- if ! valgrind ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
- > /dev/null 2> .tmp.valg; then \
+ if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
+ > /dev/null 2>&1 ; then \
echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
+ if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
+ echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
+ if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
+ echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
echo -n "valgrind passed ... "; \
echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
@gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
@mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
- @ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
+ @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
- if (ran < (ran + skip) *0.75) exit(1) }' \
+ if (ran < (ran + skip) *0.93) exit(1) }' \
coverage/oceani.mdc.gcov
@rm -f .tmp*
###### test: valvar
program:
- a := 23; b:=12
+ a := 23; b:=12 ; b1 := -b
print a, b, a+b, a-b, a*b, a/b, a%b
print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
+ print +a, +b, +b1, -a, -b, -b1
+ x := True; y := False
+ print x and y, x or y, x and x, y or y, x and not x, x < y
c ::= "This is a string"
d ::= " field theory"
print c, d, c++d
+ aconst :: string = "unchanging"
+
###### output: valvar
23 12 35 11 276 1.91667 11
False False True True False False True
+ 23 12 12 -23 -12 12
+ False True True False False False
This is a string field theory This is a string field theory
Next we change the value of variables
count = count + 1
print "error is ", target - guess * guess
+ for j:=0; then j = j+3 ; while j < 10:
+ if j != 0 and then 20 / j > 3:
+ print "20 /", j," =", 20 / j
+ else:
+ print "I won't calculate 20 /", j
+ pi ::= 3.1415926535897
+ if 355/113 == pi or else +(pi - 355/113) < 0.001:
+ print "Close enough"
+ print "lower" if 355/113 < pi else "higher"
+
###### output: cond_loop
Success
1 2 4 8 16 32 64
2 1.41422
3 1.41421
error is -4.51095e-12
+ I won't calculate 20 / 0
+ 20 / 3 = 6.66667
+ 20 / 6 = 3.33333
+ I won't calculate 20 / 9
+ Close enough
+ higher
## Say Hello
list[ 53 ]= 9040
list[ 54 ]= 9768
+## Arrays
+
+We already have some array tests, but this is where we put other
+ad-hoc things array related.
+
+###### test list
+ oceani_tests += arrays
+
+###### test: arrays
+
+ program:
+ bools:[5]Boolean
+ strings:[4]string
+
+ bools[3] = strings[1] == "Hello"
+ bools[1] = strings[2] <= "there"
+
+ for i:=0; then i=i+1; while i<5:
+ print '', bools[i],
+ print
+
+###### output: arrays
+ False True False False False
+
+## Test code with syntax errors
+
+Syntax errors aren't handled well yet - the result is almost always a
+single message about the first error. So this section will be fairly
+thin until we add proper parsing recovery in the face of common errors.
+
+A special case of syntax errors is token errors, when a token is only
+accepted because the parser doesn't know quite enough to reject it.
+There are handled better as they are quite local, so a single test
+program can trigger most of the possible errors.
+
+To handle erronous code, we need a different set of tests, as we need to
+capture `stderr`. The same test code will be used for type errors too.
+As error messages contain the line number, and we don't want changes to
+this file to change the reported numbers, we copy the code into a
+separate file first, then run from there.
+
+###### test code
+ @for t in $(oceani_failing_tests); do \
+ echo -n "Test $$t ... "; \
+ ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
+ echo '``````' > .tmp.code; \
+ ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
+ ./oceani .tmp.code > .tmp.have 2>&1; \
+ if ! cmp -s .tmp.want .tmp.have; then \
+ echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
+ echo "passed"; \
+ ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
+ done || true
+
+###### test list
+ oceani_failing_tests := syn1
+ oceani_failing_tests += tokerr
+
+###### test: syn1
+
+ program:
+ if then else while do
+
+###### output: syn1
+ .tmp.code:3:11: error: unhandled parse error: then
+
+###### test: tokerr
+ program:
+ a := 1i // imaginary numbers aren't understood
+ b:[2i]number // array sizes are handled separately
+ c:[3.14159]Boolean // array sizes must be integers
+ d:[1_000_000_000_000]number // they mustn't be huge
+ patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
+
+ multi := """
+ This is a multiline string
+ With an unsupportable suffix
+ """Aa
+
+ xx:unknown = 24
+ yy:[unknowable]number
+ zzsize := 4
+ zz:[zzsize]string // size must be constant, use ::=
+
+ // These numbers should be bad in all contexts: FIXME
+ aa:[00123]number
+
+###### output: tokerr
+ .tmp.code:3:13: error: unsupported number suffix: 1i
+ .tmp.code:4:11: error: unsupported number suffix: 2i
+ .tmp.code:5:11: error: array size must be an integer: 3.14159
+ .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
+ .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
+ .tmp.code:9:17: error: unsupported string suffix: """
+ This is a multiline string
+ With an unsupportable suffix
+ """Aa
+ .tmp.code:14:11: error: undefined type: unknown
+ .tmp.code:15:12: error: name undeclared: unknowable
+ .tmp.code:17:12: error: array size must be a constant: zzsize
+ .tmp.code:20:12: error: unrecognised number: 00123
+
+## Tests for type errors
+
+Type error don't cause parsing to abort, so we can fit many in the
+one test program. Some type errors are found during the parse, others
+during type analysis which doesn't run if parsing failed. So we cannot
+fit everything in one.
+
+These programs were generated by looking for the
+various places that `type_err()` are called.
+
+###### test list
+ oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
+
+###### test: type_err1
+
+ program:
+ print "hello" ++ 5, 5 ++ "hello"
+
+ b ::= 3
+ b = b + 1
+
+ if 3 * 4 and not True: print "Weird"
+
+###### output: type_err1
+ .tmp.code:3:25: error: expected string found number
+ .tmp.code:3:28: error: expected string found number
+ .tmp.code:6:8: error: Cannot assign to a constant: b
+ .tmp.code:5:8: info: name was defined as a constant here
+ .tmp.code:6:8: error: Cannot assign to a constant: b
+ .tmp.code:5:8: info: name was defined as a constant here
+ .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
+ oceani: type error in program - not running.
+
+###### test: type_err2
+
+ program:
+ a := 1
+ a := 2
+ a ::= 3
+ a:number = 4
+ a ::number = 5
+ c:
+
+###### output: type_err2
+ .tmp.code:4:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:5:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:6:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:7:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:8:8: Variable declared with no type or value: c
+
+###### test: type_err3
+
+ program:
+ c := "hello"
+ c = c + 1
+ c = "hello" ++ (True and False)
+ c = 4 < 5
+ print 45 + ( "Hello" ++ "there")
+ c[5] = 1
+
+ while:
+ use 1
+ use True
+ use "Hello"
+ do:
+ print
+ case 1: print "one"
+ case "Hello": print "Hello"
+
+ a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
+ a1 = a2
+ a1 = a3
+ a1 = a4
+ a1[2] = "hello"
+ a4[1] = True
+ c = a2[3]
+
+###### output: type_err3
+ .tmp.code:4:12: error: expected number but variable 'c' is string
+ .tmp.code:3:8: info: this is where 'c' was set to string
+ .tmp.code:4:12: error: Arithmetic returns number but string expected
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ .tmp.code:5:24: error: Boolean operation found where string expected
+ .tmp.code:6:12: error: Comparison returns Boolean but string expected
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ .tmp.code:7:21: error: Concat returns string but number expected
+ .tmp.code:8:8: error: string cannot be indexed
+ .tmp.code:8:8: error: string cannot be indexed
+ .tmp.code:17:13: error: expected number found string
+ .tmp.code:13:16: error: expected number, found string
+ .tmp.code:20:8: error: cannot assign value of type [5]number
+ .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
+ .tmp.code:19:36: info: this is where 'a3' was set to [10]number
+ .tmp.code:21:8: error: cannot assign value of type [5]number
+ .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
+ .tmp.code:19:51: info: this is where 'a4' was set to [5]string
+ .tmp.code:22:8: error: cannot assign value of type [5]number
+ .tmp.code:23:16: error: expected number found string
+ .tmp.code:24:16: error: expected string found Boolean
+ .tmp.code:25:12: error: have number but need string
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ oceani: type error in program - not running.
+
+###### test: type_err4
+ program:
+ a:=1; b=2; c::=3
+ print a, b, c
+
+###### output: type_err4
+ .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
+ .tmp.code:3:14: info: this is where 'b' was set to label
+ .tmp.code:3:16: error: expected label found number
+ .tmp.code:3:14: info: variable 'b' was set as label here.
+ .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
+ .tmp.code:3:14: info: this is where 'b' was set to label
+ oceani: type error in program - not running.
+
## Test erroneous command line args
To improve coverage, we want to test correct handling of strange command
code / ocean / blobdiff