###### File: oceani-tests.mk
oceani_tests :=
+ ## test list
tests:: oceani_test_suite
oceani_test_suite: oceani coverage_oceani
./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
done
+
+ ## test code
+
@gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
@mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
@ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
- if (ran < (ran + skip) *0.75) exit(1) }' \
+ if (ran < (ran + skip) *0.90) exit(1) }' \
coverage/oceani.mdc.gcov
@rm -f .tmp*
coverage_oceani: oceani.c
$(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
- ## test list
-
## Values and variables
The first test stores values in variables and performs various
###### test: valvar
program:
- a := 23; b:=12
+ a := 23; b:=12 ; b1 := -b
print a, b, a+b, a-b, a*b, a/b, a%b
print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
+ print +a, +b, +b1, -a, -b, -b1
+ x := True; y := False
+ print x and y, x or y, x and x, y or y, x and not x, x < y
c ::= "This is a string"
d ::= " field theory"
print c, d, c++d
+ aconst :: string = "unchanging"
+
###### output: valvar
23 12 35 11 276 1.91667 11
False False True True False False True
+ 23 12 12 -23 -12 12
+ False True True False False False
This is a string field theory This is a string field theory
Next we change the value of variables
list[ 53 ]= 9040
list[ 54 ]= 9768
+## Arrays
+
+We already have some array tests, but this is where we put other
+ad-hoc things array related.
+
+###### test list
+ oceani_tests += arrays
+
+###### test: arrays
+
+ program:
+ bools:[5]Boolean
+ strings:[4]string
+
+ bools[3] = strings[1] == "Hello"
+ bools[1] = strings[2] <= "there"
+
+ for i:=0; then i=i+1; while i<5:
+ print '', bools[i],
+ print
+
+###### output: arrays
+ False True False False False
+
+## Test code with syntax errors
+
+Syntax errors aren't handled well yet - the result is almost always a
+single message about the first error. So this section will be fairly
+thin until we add proper parsing recovery in the face of common errors.
+
+A special case of syntax errors is token errors, when a token is only
+accepted because the parser doesn't know quite enough to reject it.
+There are handled better as they are quite local, so a single test
+program can trigger most of the possible errors.
+
+To handle erronous code, we need a different set of tests, as we need to
+capture `stderr`. The same test code will be used for type errors too.
+As error messages contain the line number, and we don't want changes to
+this file to change the reported numbers, we copy the code into a
+separate file first, then run from there.
+
+###### test code
+ @for t in $(oceani_failing_tests); do \
+ echo -n "Test $$t ... "; \
+ ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
+ echo '``````' > .tmp.code; \
+ ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
+ ./oceani .tmp.code > .tmp.have 2>&1; \
+ if ! cmp -s .tmp.want .tmp.have; then \
+ echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
+ echo "passed"; \
+ ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
+ done || true
+
+###### test list
+ oceani_failing_tests := syn1
+ oceani_failing_tests += tokerr
+
+###### test: syn1
+
+ program:
+ if then else while do
+
+###### output: syn1
+ .tmp.code:3:11: error: unhandled parse error: then
+
+###### test: tokerr
+ program:
+ a := 1i // imaginary numbers aren't understood
+ b:[2i]number // array sizes are handled separately
+ c:[3.14159]Boolean // array sizes must be integers
+ d:[1_000_000_000_000]number // they mustn't be huge
+ patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
+
+ multi := """
+ This is a multiline string
+ With an unsupportable suffix
+ """Aa
+
+ xx:unknown = 24
+ yy:[unknowable]number
+ zzsize := 4
+ zz:[zzsize]string // size must be constant, use ::=
+
+ // These numbers should be bad in all contexts: FIXME
+ aa:[00123]number
+
+###### output: tokerr
+ .tmp.code:3:13: error: unsupported number suffix: 1i
+ .tmp.code:4:11: error: unsupported number suffix: 2i
+ .tmp.code:5:11: error: array size must be an integer: 3.14159
+ .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
+ .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
+ .tmp.code:9:17: error: unsupported string suffix: """
+ This is a multiline string
+ With an unsupportable suffix
+ """Aa
+ .tmp.code:14:11: error: undefined type: unknown
+ .tmp.code:15:12: error: name undeclared: unknowable
+ .tmp.code:17:12: error: array size must be a constant: zzsize
+ .tmp.code:20:12: error: unrecognised number: 00123
+
+## Tests for type errors
+
+Type error don't cause parsing to abort, so we can fit many in the
+one test program. Some type errors are found during the parse, others
+during type analysis which doesn't run if parsing failed. So we cannot
+fit everything in one.
+
+These programs were generated by looking for the
+various places that `type_err()` are called.
+
+###### test list
+ oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
+
+###### test: type_err1
+
+ program:
+ print "hello" ++ 5, 5 ++ "hello"
+
+ b ::= 3
+ b = b + 1
+
+ if 3 * 4 and not True: print "Weird"
+
+###### output: type_err1
+ .tmp.code:3:25: error: expected string found number
+ .tmp.code:3:28: error: expected string found number
+ .tmp.code:6:8: error: Cannot assign to a constant: b
+ .tmp.code:5:8: info: name was defined as a constant here
+ .tmp.code:6:8: error: Cannot assign to a constant: b
+ .tmp.code:5:8: info: name was defined as a constant here
+ .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
+ oceani: type error in program - not running.
+
+###### test: type_err2
+
+ program:
+ a := 1
+ a := 2
+ a ::= 3
+ a:number = 4
+ a ::number = 5
+ c:
+
+###### output: type_err2
+ .tmp.code:4:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:5:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:6:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:7:8: error: variable 'a' redeclared
+ .tmp.code:3:8: info: this is where 'a' was first declared
+ .tmp.code:8:8: Variable declared with no type or value: c
+
+###### test: type_err3
+
+ program:
+ c := "hello"
+ c = c + 1
+ c = "hello" ++ (True and False)
+ c = 4 < 5
+ print 45 + ( "Hello" ++ "there")
+ c[5] = 1
+
+ while:
+ use 1
+ use True
+ use "Hello"
+ do:
+ print
+ case 1: print "one"
+ case "Hello": print "Hello"
+
+ a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
+ a1 = a2
+ a1 = a3
+ a1 = a4
+ a1[2] = "hello"
+ a4[1] = True
+ c = a2[3]
+
+###### output: type_err3
+ .tmp.code:4:12: error: expected number but variable 'c' is string
+ .tmp.code:3:8: info: this is where 'c' was set to string
+ .tmp.code:4:12: error: Arithmetic returns number but string expected
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ .tmp.code:5:24: error: Boolean operation found where string expected
+ .tmp.code:6:12: error: Comparison returns Boolean but string expected
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ .tmp.code:7:21: error: Concat returns string but number expected
+ .tmp.code:8:8: error: string cannot be indexed
+ .tmp.code:8:8: error: string cannot be indexed
+ .tmp.code:17:13: error: expected number found string
+ .tmp.code:13:16: error: expected number, found string
+ .tmp.code:20:8: error: cannot assign value of type [5]number
+ .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
+ .tmp.code:19:36: info: this is where 'a3' was set to [10]number
+ .tmp.code:21:8: error: cannot assign value of type [5]number
+ .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
+ .tmp.code:19:51: info: this is where 'a4' was set to [5]string
+ .tmp.code:22:8: error: cannot assign value of type [5]number
+ .tmp.code:23:16: error: expected number found string
+ .tmp.code:24:16: error: expected string found Boolean
+ .tmp.code:25:12: error: have number but need string
+ .tmp.code:3:8: info: variable 'c' was set as string here.
+ oceani: type error in program - not running.
+
+###### test: type_err4
+ program:
+ a:=1; b=2; c::=3
+ print a, b, c
+
+###### output: type_err4
+ .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
+ .tmp.code:3:14: info: this is where 'b' was set to label
+ .tmp.code:3:16: error: expected label found number
+ .tmp.code:3:14: info: variable 'b' was set as label here.
+ .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
+ .tmp.code:3:14: info: this is where 'b' was set to label
+ oceani: type error in program - not running.
+
+## Test erroneous command line args
+
+To improve coverage, we want to test correct handling of strange command
+line arguments. These tests won't use code, so the exiting test types
+won't work. So we need to be able to explicitly give the command line,
+and the expected output, and have that tested and the coverage assessed.
+Rather than having to spell out the whole command name, just give "cmd",
+and discard that. Requiring but discarding the command make an empty
+command list possible.
+
+###### test code
+ @for t in $(oceani_special_tests); do \
+ echo -n "Test $$t ... ";\
+ i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
+ ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
+ ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
+ if ! cmp -s .tmp.want .tmp.have; then \
+ echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
+ echo "passed"; \
+ ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
+ done || true
+
+
+###### test list
+ oceani_special_tests += "cmd"
+ oceani_special_tests += "cmd,-zyx"
+ oceani_special_tests += "cmd,nofile"
+ oceani_special_tests += "cmd,/dev/null"
+ oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
+
+###### output: cmd
+ oceani: no input file given
+
+###### output: cmd,-zyx
+ ./oceani: invalid option -- 'z'
+ Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
+
+###### output: cmd,nofile
+ oceani: cannot open nofile
+
+###### output: cmd,/dev/null
+ oceani: could not find any code in /dev/null
+
+###### output: cmd,--section,toast:nothing,oceani-tests.mdc
+ oceani: cannot find section toast:nothing
+
code / ocean / blobdiff