1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.94) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
201 ###### output: varmerge
202 zero , one , two , three , many ,
204 ## Conditions and Loops
206 Now we need to test if/else and some different loops
209 oceani_tests += cond_loop
211 ###### test: cond_loop
219 for b:=1; then b=b+b; while b < 100:
222 // Newtons method for square root of 2
228 current := guess * guess
229 use +(current - target) > 0.000000001
231 guess = (guess + (target / guess) ) / 2
234 print "error is ", target - guess * guess
236 for j:=0; then j = j+3 ; while j < 10:
237 if j != 0 and then 20 / j > 3:
238 print "20 /", j," =", 20 / j
240 print "I won't calculate 20 /", j
241 pi ::= 3.1415926535897
242 if 355/113 == pi or else +(pi - 355/113) < 0.001:
244 print "lower" if 355/113 < pi else "higher"
246 if pi > 3 then print "pi exceeds three"; else print "need more pie"
247 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
248 for { i := 0; sum := 0 }
254 print "sum 1..10 is", sum
256 ###### output: cond_loop
263 error is -4.51095e-12
264 I won't calculate 20 / 0
267 I won't calculate 20 / 9
276 The demonstration code presented in the interpreted is suitable for the test suite.
277 Here I break it into two parts, keeping the array code separate.
280 oceani_tests += "sayhello,55,33,hello,True"
281 oceani_tests += "sayhello,12,60,there,False"
283 ###### test: sayhello
286 A := $av[1]; B := $av[2]
288 bbool := av[ac-1] == "True"
289 print "Hello World, what lovely oceans you have!"
290 /* When a variable is defined in both branches of an 'if',
291 * and used afterwards, the variables are merged.
297 print "Is", A, "bigger than", B,"? ", bigger
298 /* If a variable is not used after the 'if', no
299 * merge happens, so types can be different
302 double:string = "yes"
303 print A, "is more than twice", B, "?", double
306 print "double", B, "is", double
317 print "GCD of", A, "and", B,"is", a
319 print a, "is not positive, cannot calculate GCD"
321 print b, "is not positive, cannot calculate GCD"
326 print "Fibonacci:", f1,f2,
336 print astr ++ " was the str"
338 print "I found the str over " ++ astr
340 /* Binary search... */
357 print "Yay, I found", target
359 print "Closest I found was", mid
361 ###### output: sayhello,55,33,hello,True
362 Hello World, what lovely oceans you have!
363 Is 55 bigger than 33 ? yes
365 GCD of 55 and 33 is 11
366 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
368 Closest I found was 77.3438
370 ###### output: sayhello,12,60,there,False
371 Hello World, what lovely oceans you have!
372 Is 12 bigger than 60 ? no
374 GCD of 12 and 60 is 12
375 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
376 I found the str over there
377 Closest I found was 77.3438
380 oceani_tests += "insert_sort"
381 ###### test: insert_sort
386 for i:=1; then i = i + 1; while i < size:
387 n := list[i-1] * list[i-1]
388 list[i] = (n / 100) % 10000
391 for i:=0; then i = i + 1; while i < size:
392 print "list[",i,"]=",list[i]
394 for i := 1; then i=i+1; while i < size:
395 for j:=i-1; then j=j-1; while j >= 0:
396 if list[j] > list[j+1]:
401 for i:=0; then i = i + 1; while i < size:
402 print "list[",i,"]=",list[i]
404 ###### output: insert_sort
520 We already have some array tests, but this is where we put other
521 ad-hoc things array related.
524 oceani_tests += arrays
532 bools[3] = strings[1] == "Hello"
533 bools[1] = strings[2] <= "there"
535 for i:=0; then i=i+1; while i<5:
539 ra[6] = 42 // mustn't crash
540 print '', bools[i], ra[j-1],
543 ###### output: arrays
544 False 0 True 1 False 4 False 9 False 16
548 Time to test if structure declarations and accesses work correctly.
551 oceani_tests += structs
559 active:Boolean = True
561 struct baz { a:number; b:Boolean; }
566 for i:=0; then i=i+1; while i < 4:
574 info[i].size[0] = i*i
576 info[i].active = False
578 for i:=0; then i=i+1; while i < 4:
579 print info[i].name, info[i].active, info[i].size[0]
581 ###### output: structs
588 ## Test code with syntax errors
590 Syntax errors aren't handled well yet - the result is almost always a
591 single message about the first error. So this section will be fairly
592 thin until we add proper parsing recovery in the face of common errors.
594 A special case of syntax errors is token errors, when a token is only
595 accepted because the parser doesn't know quite enough to reject it.
596 There are handled better as they are quite local, so a single test
597 program can trigger most of the possible errors.
599 To handle erronous code, we need a different set of tests, as we need to
600 capture `stderr`. The same test code will be used for type errors too.
601 As error messages contain the line number, and we don't want changes to
602 this file to change the reported numbers, we copy the code into a
603 separate file first, then run from there.
606 @for t in $(oceani_failing_tests); do \
607 echo -n "Test $$t ... "; \
608 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
609 echo '``````' > .tmp.code; \
610 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
611 ./oceani .tmp.code > .tmp.have 2>&1; \
612 if ! cmp -s .tmp.want .tmp.have; then \
613 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
615 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
619 oceani_failing_tests := syn1
620 oceani_failing_tests += tokerr
625 if then else while do
628 .tmp.code:3:11: Syntax error in statement: then
632 a := 1i // imaginary numbers aren't understood
633 b:[2i]number // array sizes are handled separately
634 c:[3.14159]Boolean // array sizes must be integers
635 d:[1_000_000_000_000]number // they mustn't be huge
636 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
639 This is a multiline string
640 With an unsupportable suffix
644 yy:[unknowable]number
646 zz:[zzsize]string // size must be constant, use ::=
648 // These numbers should be bad in all contexts: FIXME
651 ###### output: tokerr
652 .tmp.code:3:13: error: unsupported number suffix: 1i
653 .tmp.code:4:11: error: unsupported number suffix: 2i
654 .tmp.code:5:11: error: array size must be an integer: 3.14159
655 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
656 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
657 .tmp.code:9:17: error: unsupported string suffix: """
658 This is a multiline string
659 With an unsupportable suffix
661 .tmp.code:14:11: error: undefined type: unknown
662 .tmp.code:15:12: error: name undeclared: unknowable
663 .tmp.code:17:12: error: array size must be a constant: zzsize
664 .tmp.code:20:12: error: unrecognised number: 00123
666 ## Tests for type errors
668 Type error don't cause parsing to abort, so we can fit many in the
669 one test program. Some type errors are found during the parse, others
670 during type analysis which doesn't run if parsing failed. So we cannot
671 fit everything in one.
673 These programs were generated by looking for the
674 various places that `type_err()` are called.
677 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
679 ###### test: type_err1
682 print "hello" ++ 5, 5 ++ "hello"
687 if 3 * 4 and not True: print "Weird"
689 ###### output: type_err1
690 .tmp.code:3:25: error: expected string found number
691 .tmp.code:3:28: error: expected string found number
692 .tmp.code:6:8: error: Cannot assign to a constant: b
693 .tmp.code:5:8: info: name was defined as a constant here
694 .tmp.code:6:8: error: Cannot assign to a constant: b
695 .tmp.code:5:8: info: name was defined as a constant here
696 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
697 oceani: type error in program - not running.
699 ###### test: type_err2
709 ###### output: type_err2
710 .tmp.code:4:8: error: variable 'a' redeclared
711 .tmp.code:3:8: info: this is where 'a' was first declared
712 .tmp.code:5:8: error: variable 'a' redeclared
713 .tmp.code:3:8: info: this is where 'a' was first declared
714 .tmp.code:6:8: error: variable 'a' redeclared
715 .tmp.code:3:8: info: this is where 'a' was first declared
716 .tmp.code:7:8: error: variable 'a' redeclared
717 .tmp.code:3:8: info: this is where 'a' was first declared
718 .tmp.code:8:8: Variable declared with no type or value: c
720 ###### test: type_err3
729 c = "hello" ++ (True and False)
731 print 45 + ( "Hello" ++ "there")
741 case "Hello": print "Hello"
743 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
755 ###### output: type_err3
756 .tmp.code:8:12: error: expected number but variable 'c' is string
757 .tmp.code:7:8: info: this is where 'c' was set to string
758 .tmp.code:8:12: error: Arithmetic returns number but string expected
759 .tmp.code:7:8: info: variable 'c' was set as string here.
760 .tmp.code:9:24: error: Boolean operation found where string expected
761 .tmp.code:10:12: error: Comparison returns Boolean but string expected
762 .tmp.code:7:8: info: variable 'c' was set as string here.
763 .tmp.code:11:21: error: Concat returns string but number expected
764 .tmp.code:12:8: error: string cannot be indexed
765 .tmp.code:12:8: error: string cannot be indexed
766 .tmp.code:21:13: error: expected number found string
767 .tmp.code:17:16: error: expected number, found string
768 .tmp.code:24:8: error: cannot assign value of type [5]number
769 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
770 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
771 .tmp.code:25:8: error: cannot assign value of type [5]number
772 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
773 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
774 .tmp.code:26:8: error: cannot assign value of type [5]number
775 .tmp.code:27:16: error: expected number found string
776 .tmp.code:28:16: error: expected string found Boolean
777 .tmp.code:29:12: error: have number but need string
778 .tmp.code:7:8: info: variable 'c' was set as string here.
779 .tmp.code:32:8: error: variable used but not declared: foo
780 .tmp.code:32:8: error: field reference attempted on none, not a struct
781 .tmp.code:32:16: error: expected none found number
782 .tmp.code:33:14: error: field reference attempted on string, not a struct
783 oceani: type error in program - not running.
785 ###### test: type_err4
790 ###### output: type_err4
791 .tmp.code:3:14: error: variable used but not declared: b
792 .tmp.code:3:16: error: expected none found number
793 .tmp.code:3:14: info: variable 'b' was set as none here.
794 oceani: type error in program - not running.
797 oceani_failing_tests += type_err_const type_err_const1
799 ###### test: type_err_const
802 bar ::= "string" + 56
809 ###### output: type_err_const
810 .tmp.code:4:16: error: expected number found string
811 .tmp.code:6:8: error: name already declared: bar
812 .tmp.code:4:8: info: this is where 'bar' was first declared
813 .tmp.code:8:8: error: variable 'foo' redeclared
814 .tmp.code:3:8: info: this is where 'foo' was first declared
816 ###### test: type_err_const1
824 ###### output: type_err_const1
825 .tmp.code:3:12: Syntax error in constant: :
826 .tmp.code:4:12: Syntax error in constant: :
828 ## Test erroneous command line args
830 To improve coverage, we want to test correct handling of strange command
831 line arguments. These tests won't use code, so the exiting test types
832 won't work. So we need to be able to explicitly give the command line,
833 and the expected output, and have that tested and the coverage assessed.
834 Rather than having to spell out the whole command name, just give "cmd",
835 and discard that. Requiring but discarding the command make an empty
836 command list possible.
839 @for t in $(oceani_special_tests); do \
840 echo -n "Test $$t ... ";\
841 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
842 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
843 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
844 if ! cmp -s .tmp.want .tmp.have; then \
845 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
847 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
851 oceani_special_tests += "cmd"
852 oceani_special_tests += "cmd,-zyx"
853 oceani_special_tests += "cmd,nofile"
854 oceani_special_tests += "cmd,/dev/null"
855 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
858 oceani: no input file given
860 ###### output: cmd,-zyx
861 ./oceani: invalid option -- 'z'
862 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
864 ###### output: cmd,nofile
865 oceani: cannot open nofile
867 ###### output: cmd,/dev/null
868 oceani: could not find any code in /dev/null
870 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
871 oceani: cannot find section toast:nothing