1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
42 { echo "Grammar container conflicts, please review" ; exit 1; }
43 @rm -rf coverage; mkdir -p coverage
45 @for T in $(oceani_tests); do \
46 echo -n "Test $$T ... "; \
47 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
48 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
49 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
50 if ! cmp -s .tmp.want .tmp.have; then \
51 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
52 echo -n "passed ... "; \
53 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
54 > /dev/null 2>&1 ; then \
55 echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
56 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
57 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
58 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
59 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
60 echo -n "valgrind passed ... "; \
61 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
62 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
63 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
64 if ! cmp -s .tmp.code1 .tmp.code2; then \
65 echo "Printing Failed"; diff -u .tmp.code1 .tmp.code2; exit1 ; fi ; \
66 echo "Printing passed"; \
67 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
68 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
73 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
74 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
75 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
76 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
77 if (ran < (ran + skip) *0.93) exit(1) }' \
78 coverage/oceani.mdc.gcov
81 coverage_oceani: oceani.c
82 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
84 ## Values and variables
86 The first test stores values in variables and performs various
90 oceani_tests += "valvar"
96 a := 23; b:=12 ; b1 := -b
97 print a, b, a+b, a-b, a*b, a/b, a%b
98 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
99 print +a, +b, +b1, -a, -b, -b1
100 x := True; y := False
101 print x and y, x or y, x and x, y or y, x and not x, x < y
103 c ::= "This is a string"
104 d ::= " field theory"
107 aconst :: string = "unchanging"
109 ###### output: valvar
111 23 12 35 11 276 1.91667 11
112 False False True True False False True
114 False True True False False False
115 This is a string field theory This is a string field theory
117 Next we change the value of variables
120 oceani_tests += "setvar"
127 a = (a + a) * (a + a)
131 ###### output: setvar
134 ## Conditions and Loops
136 Now we need to test if/else and some different loops
139 oceani_tests += cond_loop
141 ###### test: cond_loop
149 for b:=1; then b=b+b; while b < 100:
152 // Newtons method for square root of 2
158 current := guess * guess
159 use +(current - target) > 0.000000001
161 guess = (guess + (target / guess) ) / 2
164 print "error is ", target - guess * guess
166 for j:=0; then j = j+3 ; while j < 10:
167 if j != 0 and then 20 / j > 3:
168 print "20 /", j," =", 20 / j
170 print "I won't calculate 20 /", j
171 pi ::= 3.1415926535897
172 if 355/113 == pi or else +(pi - 355/113) < 0.001:
174 print "lower" if 355/113 < pi else "higher"
176 ###### output: cond_loop
183 error is -4.51095e-12
184 I won't calculate 20 / 0
187 I won't calculate 20 / 9
193 The demonstration code presented in the interpreted is suitable for the test suite.
194 Here I break it into two parts, keeping the array code separate.
197 oceani_tests += "sayhello,55,33"
198 oceani_tests += "sayhello,12,60"
200 ###### test: sayhello
203 print "Hello World, what lovely oceans you have!"
204 /* When a variable is defined in both branches of an 'if',
205 * and used afterwards, the variables are merged.
211 print "Is", A, "bigger than", B,"? ", bigger
212 /* If a variable is not used after the 'if', no
213 * merge happens, so types can be different
216 double:string = "yes"
217 print A, "is more than twice", B, "?", double
220 print "double", B, "is", double
231 print "GCD of", A, "and", B,"is", a
233 print a, "is not positive, cannot calculate GCD"
235 print b, "is not positive, cannot calculate GCD"
240 print "Fibonacci:", f1,f2,
249 /* Binary search... */
266 print "Yay, I found", target
268 print "Closest I found was", mid
270 ###### output: sayhello,55,33
271 Hello World, what lovely oceans you have!
272 Is 55 bigger than 33 ? yes
274 GCD of 55 and 33 is 11
275 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
276 Closest I found was 77.3438
278 ###### output: sayhello,12,60
279 Hello World, what lovely oceans you have!
280 Is 12 bigger than 60 ? no
282 GCD of 12 and 60 is 12
283 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
284 Closest I found was 77.3438
287 oceani_tests += "insert_sort"
288 ###### test: insert_sort
293 for i:=1; then i = i + 1; while i < size:
294 n := list[i-1] * list[i-1]
295 list[i] = (n / 100) % 10000
298 for i:=0; then i = i + 1; while i < size:
299 print "list[",i,"]=",list[i]
301 for i := 1; then i=i+1; while i < size:
302 for j:=i-1; then j=j-1; while j >= 0:
303 if list[j] > list[j+1]:
308 for i:=0; then i = i + 1; while i < size:
309 print "list[",i,"]=",list[i]
311 ###### output: insert_sort
427 We already have some array tests, but this is where we put other
428 ad-hoc things array related.
431 oceani_tests += arrays
439 bools[3] = strings[1] == "Hello"
440 bools[1] = strings[2] <= "there"
442 for i:=0; then i=i+1; while i<5:
446 ###### output: arrays
447 False True False False False
449 ## Test code with syntax errors
451 Syntax errors aren't handled well yet - the result is almost always a
452 single message about the first error. So this section will be fairly
453 thin until we add proper parsing recovery in the face of common errors.
455 A special case of syntax errors is token errors, when a token is only
456 accepted because the parser doesn't know quite enough to reject it.
457 There are handled better as they are quite local, so a single test
458 program can trigger most of the possible errors.
460 To handle erronous code, we need a different set of tests, as we need to
461 capture `stderr`. The same test code will be used for type errors too.
462 As error messages contain the line number, and we don't want changes to
463 this file to change the reported numbers, we copy the code into a
464 separate file first, then run from there.
467 @for t in $(oceani_failing_tests); do \
468 echo -n "Test $$t ... "; \
469 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
470 echo '``````' > .tmp.code; \
471 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
472 ./oceani .tmp.code > .tmp.have 2>&1; \
473 if ! cmp -s .tmp.want .tmp.have; then \
474 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
476 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
480 oceani_failing_tests := syn1
481 oceani_failing_tests += tokerr
486 if then else while do
489 .tmp.code:3:11: error: unhandled parse error: then
490 oceani: no program found.
494 a := 1i // imaginary numbers aren't understood
495 b:[2i]number // array sizes are handled separately
496 c:[3.14159]Boolean // array sizes must be integers
497 d:[1_000_000_000_000]number // they mustn't be huge
498 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
501 This is a multiline string
502 With an unsupportable suffix
506 yy:[unknowable]number
508 zz:[zzsize]string // size must be constant, use ::=
510 // These numbers should be bad in all contexts: FIXME
513 ###### output: tokerr
514 .tmp.code:3:13: error: unsupported number suffix: 1i
515 .tmp.code:4:11: error: unsupported number suffix: 2i
516 .tmp.code:5:11: error: array size must be an integer: 3.14159
517 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
518 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
519 .tmp.code:9:17: error: unsupported string suffix: """
520 This is a multiline string
521 With an unsupportable suffix
523 .tmp.code:14:11: error: undefined type: unknown
524 .tmp.code:15:12: error: name undeclared: unknowable
525 .tmp.code:17:12: error: array size must be a constant: zzsize
526 .tmp.code:20:12: error: unrecognised number: 00123
528 ## Tests for type errors
530 Type error don't cause parsing to abort, so we can fit many in the
531 one test program. Some type errors are found during the parse, others
532 during type analysis which doesn't run if parsing failed. So we cannot
533 fit everything in one.
535 These programs were generated by looking for the
536 various places that `type_err()` are called.
539 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
541 ###### test: type_err1
544 print "hello" ++ 5, 5 ++ "hello"
549 if 3 * 4 and not True: print "Weird"
551 ###### output: type_err1
552 .tmp.code:3:25: error: expected string found number
553 .tmp.code:3:28: error: expected string found number
554 .tmp.code:6:8: error: Cannot assign to a constant: b
555 .tmp.code:5:8: info: name was defined as a constant here
556 .tmp.code:6:8: error: Cannot assign to a constant: b
557 .tmp.code:5:8: info: name was defined as a constant here
558 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
559 oceani: type error in program - not running.
561 ###### test: type_err2
571 ###### output: type_err2
572 .tmp.code:4:8: error: variable 'a' redeclared
573 .tmp.code:3:8: info: this is where 'a' was first declared
574 .tmp.code:5:8: error: variable 'a' redeclared
575 .tmp.code:3:8: info: this is where 'a' was first declared
576 .tmp.code:6:8: error: variable 'a' redeclared
577 .tmp.code:3:8: info: this is where 'a' was first declared
578 .tmp.code:7:8: error: variable 'a' redeclared
579 .tmp.code:3:8: info: this is where 'a' was first declared
580 .tmp.code:8:8: Variable declared with no type or value: c
582 ###### test: type_err3
587 c = "hello" ++ (True and False)
589 print 45 + ( "Hello" ++ "there")
599 case "Hello": print "Hello"
601 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
609 ###### output: type_err3
610 .tmp.code:4:12: error: expected number but variable 'c' is string
611 .tmp.code:3:8: info: this is where 'c' was set to string
612 .tmp.code:4:12: error: Arithmetic returns number but string expected
613 .tmp.code:3:8: info: variable 'c' was set as string here.
614 .tmp.code:5:24: error: Boolean operation found where string expected
615 .tmp.code:6:12: error: Comparison returns Boolean but string expected
616 .tmp.code:3:8: info: variable 'c' was set as string here.
617 .tmp.code:7:21: error: Concat returns string but number expected
618 .tmp.code:8:8: error: string cannot be indexed
619 .tmp.code:8:8: error: string cannot be indexed
620 .tmp.code:17:13: error: expected number found string
621 .tmp.code:13:16: error: expected number, found string
622 .tmp.code:20:8: error: cannot assign value of type [5]number
623 .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
624 .tmp.code:19:36: info: this is where 'a3' was set to [10]number
625 .tmp.code:21:8: error: cannot assign value of type [5]number
626 .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
627 .tmp.code:19:51: info: this is where 'a4' was set to [5]string
628 .tmp.code:22:8: error: cannot assign value of type [5]number
629 .tmp.code:23:16: error: expected number found string
630 .tmp.code:24:16: error: expected string found Boolean
631 .tmp.code:25:12: error: have number but need string
632 .tmp.code:3:8: info: variable 'c' was set as string here.
633 oceani: type error in program - not running.
635 ###### test: type_err4
640 ###### output: type_err4
641 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
642 .tmp.code:3:14: info: this is where 'b' was set to label
643 .tmp.code:3:16: error: expected label found number
644 .tmp.code:3:14: info: variable 'b' was set as label here.
645 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
646 .tmp.code:3:14: info: this is where 'b' was set to label
647 oceani: type error in program - not running.
649 ## Test erroneous command line args
651 To improve coverage, we want to test correct handling of strange command
652 line arguments. These tests won't use code, so the exiting test types
653 won't work. So we need to be able to explicitly give the command line,
654 and the expected output, and have that tested and the coverage assessed.
655 Rather than having to spell out the whole command name, just give "cmd",
656 and discard that. Requiring but discarding the command make an empty
657 command list possible.
660 @for t in $(oceani_special_tests); do \
661 echo -n "Test $$t ... ";\
662 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
663 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
664 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
665 if ! cmp -s .tmp.want .tmp.have; then \
666 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
668 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
673 oceani_special_tests += "cmd"
674 oceani_special_tests += "cmd,-zyx"
675 oceani_special_tests += "cmd,nofile"
676 oceani_special_tests += "cmd,/dev/null"
677 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
680 oceani: no input file given
682 ###### output: cmd,-zyx
683 ./oceani: invalid option -- 'z'
684 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
686 ###### output: cmd,nofile
687 oceani: cannot open nofile
689 ###### output: cmd,/dev/null
690 oceani: could not find any code in /dev/null
692 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
693 oceani: cannot find section toast:nothing