1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.95) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
116 a := 23; b:=12 ; b1 := -b
117 print a, b, a+b, a-b, a*b, a/b, a%b
118 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
119 print +a, +b, +b1, -a, -b, -b1
120 x := True; y := False
121 print x and y, x or y, x and x, y or y, x and not x, x < y
123 c ::= "This is a string"
124 d ::= " field theory"
127 aconst :: string = "unchanging"
136 ###### output: valvar
138 23 12 35 11 276 1.91667 11
139 False False True True False False True
141 False True True False False False
142 This is a string field theory This is a string field theory
145 Next we change the value of variables
148 oceani_tests += "setvar"
155 a = (a + a) * (a + a)
159 ###### output: setvar
165 oceani_tests += "consts"
170 four ::= 2 + 2 ; five ::= 10/2
171 const pie ::= "I like Pie";
172 cake ::= "The cake is"
176 print "Hello World, what lovely oceans you have!"
177 print "are there", five, "?"
178 print pi, pie, "but", cake
180 ###### output: consts
181 Hello World, what lovely oceans you have!
183 3.14159 I like Pie but The cake is a lie
185 Test merging of variables from multiple cases
188 oceani_tests += varmerge
190 ###### test: varmerge
193 for i:=0; then i=i+1; while i < 5:
203 for i:=0; then i=i+1; while i < 5:
210 // re-declare a CondScope variable
215 ###### output: varmerge
216 zero , one , two , three , many ,
219 ## Conditions and Loops
221 Now we need to test if/else and some different loops
224 oceani_tests += cond_loop
226 ###### test: cond_loop
234 for b:=1; then b=b+b; while b < 100:
237 // Newtons method for square root of 2
243 current := guess * guess
244 use +(current - target) > 0.000000001
246 guess = (guess + (target / guess) ) / 2
249 print "error is ", target - guess * guess
251 for j:=0; then j = j+3 ; while j < 10:
252 if j != 0 and then 20 / j > 3:
253 print "20 /", j," =", 20 / j
255 print "I won't calculate 20 /", j
256 pi ::= 3.1415926535897
257 if 355/113 == pi or else +(pi - 355/113) < 0.001:
259 print "lower" if 355/113 < pi else "higher"
261 if pi > 3 then print "pi exceeds three"; else print "need more pie"
262 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
263 for { i := 0; sum := 0 }
269 print "sum 1..10 is", sum
271 ###### output: cond_loop
278 error is -4.51095e-12
279 I won't calculate 20 / 0
282 I won't calculate 20 / 9
291 The demonstration code presented in the interpreted is suitable for the test suite.
292 Here I break it into two parts, keeping the array code separate.
295 oceani_tests += "sayhello,55,33,hello,True"
296 oceani_tests += "sayhello,12,60,there,False"
298 ###### test: sayhello
300 func main(av:[ac::number]string)
301 A := $av[1]; B := $av[2]
303 bbool := av[ac-1] == "True"
304 print "Hello World, what lovely oceans you have!"
305 /* When a variable is defined in both branches of an 'if',
306 * and used afterwards, the variables are merged.
312 print "Is", A, "bigger than", B,"? ", bigger
313 /* If a variable is not used after the 'if', no
314 * merge happens, so types can be different
317 double:string = "yes"
318 print A, "is more than twice", B, "?", double
321 print "double", B, "is", double
332 print "GCD of", A, "and", B,"is", a
334 print a, "is not positive, cannot calculate GCD"
336 print b, "is not positive, cannot calculate GCD"
341 print "Fibonacci:", f1,f2,
351 print astr ++ " was the str"
353 print "I found the str over " ++ astr
355 /* Binary search... */
372 print "Yay, I found", target
374 print "Closest I found was", mid
376 ###### output: sayhello,55,33,hello,True
377 Hello World, what lovely oceans you have!
378 Is 55 bigger than 33 ? yes
380 GCD of 55 and 33 is 11
381 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
383 Closest I found was 77.3438
385 ###### output: sayhello,12,60,there,False
386 Hello World, what lovely oceans you have!
387 Is 12 bigger than 60 ? no
389 GCD of 12 and 60 is 12
390 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
391 I found the str over there
392 Closest I found was 77.3438
395 oceani_tests += "insert_sort"
396 ###### test: insert_sort
401 for i:=1; then i = i + 1; while i < size:
402 n := list[i-1] * list[i-1]
403 list[i] = (n / 100) % 10000
406 for i:=0; then i = i + 1; while i < size:
407 print "list[",i,"]=",list[i]
409 for i := 1; then i=i+1; while i < size:
410 for j:=i-1; then j=j-1; while j >= 0:
411 if list[j] > list[j+1]:
416 for i:=0; then i = i + 1; while i < size:
417 print "list[",i,"]=",list[i]
419 ###### output: insert_sort
535 We already have some array tests, but this is where we put other
536 ad-hoc things array related.
539 oceani_tests += arrays
547 bools[3] = strings[1] == "Hello"
548 bools[1] = strings[2] <= "there"
550 for i:=0; then i=i+1; while i<5:
554 ra[6] = 42 // mustn't crash
555 print '', bools[i], ra[j-1],
558 ###### output: arrays
559 False 0 True 1 False 4 False 9 False 16
563 Time to test if structure declarations and accesses work correctly.
566 oceani_tests += structs
574 active:Boolean = True
576 struct baz { a:number; b:Boolean; }
582 for i:=0; then i=i+1; while i < 4:
590 info[i].size[0] = i*i
592 info[i].active = False
594 for i:=0; then i=i+1; while i < 4:
595 print info[i].name, info[i].active, info[i].size[0]
597 ###### output: structs
604 ## Test code with syntax errors
606 Syntax errors aren't handled well yet - the result is almost always a
607 single message about the first error. So this section will be fairly
608 thin until we add proper parsing recovery in the face of common errors.
610 A special case of syntax errors is token errors, when a token is only
611 accepted because the parser doesn't know quite enough to reject it.
612 There are handled better as they are quite local, so a single test
613 program can trigger most of the possible errors.
615 To handle erronous code, we need a different set of tests, as we need to
616 capture `stderr`. The same test code will be used for type errors too.
617 As error messages contain the line number, and we don't want changes to
618 this file to change the reported numbers, we copy the code into a
619 separate file first, then run from there.
622 @for t in $(oceani_failing_tests); do \
623 echo -n "Test $$t ... "; \
624 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
625 echo '``````' > .tmp.code; \
626 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
627 ./oceani .tmp.code > .tmp.have 2>&1; \
628 if ! cmp -s .tmp.want .tmp.have; then \
629 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
631 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
635 oceani_failing_tests := syn1
636 oceani_failing_tests += tokerr
641 if then else while do
644 .tmp.code:3:11: Syntax error in statement: then
648 a := 1i // imaginary numbers aren't understood
649 b:[2i]number // array sizes are handled separately
650 c:[3.14159]Boolean // array sizes must be integers
651 d:[1_000_000_000_000]number // they mustn't be huge
652 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
655 This is a multiline string
656 With an unsupportable suffix
660 yy:[unknowable]number
662 zz:[zzsize]string // size must be constant, use ::=
664 // These numbers should be bad in all contexts: FIXME
667 ###### output: tokerr
668 .tmp.code:3:13: error: unsupported number suffix: 1i
669 .tmp.code:4:11: error: unsupported number suffix: 2i
670 .tmp.code:5:11: error: array size must be an integer: 3.14159
671 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
672 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
673 .tmp.code:9:17: error: unsupported string suffix: """
674 This is a multiline string
675 With an unsupportable suffix
677 .tmp.code:14:11: error: undefined type: unknown
678 .tmp.code:15:12: error: name undeclared: unknowable
679 .tmp.code:17:12: error: array size must be a constant: zzsize
680 .tmp.code:20:12: error: unrecognised number: 00123
682 ## Tests for type errors
684 Type error don't cause parsing to abort, so we can fit many in the
685 one test program. Some type errors are found during the parse, others
686 during type analysis which doesn't run if parsing failed. So we cannot
687 fit everything in one.
689 These programs were generated by looking for the
690 various places that `type_err()` are called.
693 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
695 ###### test: type_err1
698 print "hello" ++ 5, 5 ++ "hello"
703 if 3 * 4 and not True: print "Weird"
705 ###### output: type_err1
706 .tmp.code:3:25: error: expected string found number
707 .tmp.code:3:28: error: expected string found number
708 .tmp.code:6:8: error: Cannot assign to a constant: b
709 .tmp.code:5:8: info: name was defined as a constant here
710 .tmp.code:6:8: error: Cannot assign to a constant: b
711 .tmp.code:5:8: info: name was defined as a constant here
712 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
713 oceani: type error in program - not running.
715 ###### test: type_err2
725 ###### output: type_err2
726 .tmp.code:4:8: error: variable 'a' redeclared
727 .tmp.code:3:8: info: this is where 'a' was first declared
728 .tmp.code:5:8: error: variable 'a' redeclared
729 .tmp.code:3:8: info: this is where 'a' was first declared
730 .tmp.code:6:8: error: variable 'a' redeclared
731 .tmp.code:3:8: info: this is where 'a' was first declared
732 .tmp.code:7:8: error: variable 'a' redeclared
733 .tmp.code:3:8: info: this is where 'a' was first declared
734 .tmp.code:8:8: Variable declared with no type or value: c
736 ###### test: type_err3
745 c = "hello" ++ (True and False)
747 print 45 + ( "Hello" ++ "there")
757 case "Hello": print "Hello"
759 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
774 // trigger 'labels not permitted' error message
775 while 1 if True else False:
779 ###### output: type_err3
780 .tmp.code:8:12: error: expected number but variable 'c' is string
781 .tmp.code:7:8: info: this is where 'c' was set to string
782 .tmp.code:8:12: error: Arithmetic returns number but string expected
783 .tmp.code:7:8: info: variable 'c' was set as string here.
784 .tmp.code:9:24: error: Boolean operation found where string expected
785 .tmp.code:10:12: error: Comparison returns Boolean but string expected
786 .tmp.code:7:8: info: variable 'c' was set as string here.
787 .tmp.code:11:21: error: Concat returns string but number expected
788 .tmp.code:12:8: error: string cannot be indexed
789 .tmp.code:12:8: error: string cannot be indexed
790 .tmp.code:21:13: error: expected number found string
791 .tmp.code:17:16: error: expected number, found string
792 .tmp.code:24:8: error: cannot assign value of type [5]number
793 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
794 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
795 .tmp.code:25:8: error: cannot assign value of type [5]number
796 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
797 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
798 .tmp.code:26:8: error: cannot assign value of type [5]number
799 .tmp.code:27:16: error: expected number found string
800 .tmp.code:28:16: error: expected string found Boolean
801 .tmp.code:29:12: error: have number but need string
802 .tmp.code:7:8: info: variable 'c' was set as string here.
803 .tmp.code:32:8: error: variable used but not declared: foo
804 .tmp.code:32:8: error: field reference attempted on none, not a struct
805 .tmp.code:32:16: error: expected none found number
806 .tmp.code:33:14: error: field reference attempted on string, not a struct
807 .tmp.code:34:14: error: cannot find requested field in foo
808 .tmp.code:35:17: error: have string but need number
809 .tmp.code:39:29: error: expected number (labels not permitted) found Boolean
810 oceani: type error in program - not running.
812 ###### test: type_err4
817 ###### output: type_err4
818 .tmp.code:3:14: error: variable used but not declared: b
819 .tmp.code:3:16: error: expected none found number
820 .tmp.code:3:14: info: variable 'b' was set as none here.
821 oceani: type error in program - not running.
824 oceani_failing_tests += type_err_const type_err_const1 missing_program
826 ###### test: type_err_const
829 bar ::= "string" + 56
836 // trigger duplicate-main error
841 ###### output: type_err_const
842 .tmp.code:4:16: error: expected number found string
843 .tmp.code:6:8: error: name already declared: bar
844 .tmp.code:4:8: info: this is where 'bar' was first declared
845 .tmp.code:8:8: error: variable 'foo' redeclared
846 .tmp.code:3:8: info: this is where 'foo' was first declared
847 .tmp.code:13:8: error: variable 'foo' redeclared
848 .tmp.code:3:8: info: this is where 'foo' was first declared
849 .tmp.code:13:8: "main" defined a second time
851 ###### test: type_err_const1
859 ###### output: type_err_const1
860 .tmp.code:3:12: Syntax error in constant: :
861 .tmp.code:4:12: Syntax error in constant: :
863 ###### test: missing_program
867 ###### output: missing_program
868 oceani: no main function found.
870 ## Test erroneous command line args
872 To improve coverage, we want to test correct handling of strange command
873 line arguments. These tests won't use code, so the exiting test types
874 won't work. So we need to be able to explicitly give the command line,
875 and the expected output, and have that tested and the coverage assessed.
876 Rather than having to spell out the whole command name, just give "cmd",
877 and discard that. Requiring but discarding the command make an empty
878 command list possible.
881 @for t in $(oceani_special_tests); do \
882 echo -n "Test $$t ... ";\
883 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
884 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
885 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
886 if ! cmp -s .tmp.want .tmp.have; then \
887 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
889 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
893 oceani_special_tests += "cmd"
894 oceani_special_tests += "cmd,-zyx"
895 oceani_special_tests += "cmd,nofile"
896 oceani_special_tests += "cmd,/dev/null"
897 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
900 oceani: no input file given
902 ###### output: cmd,-zyx
903 ./oceani: invalid option -- 'z'
904 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
906 ###### output: cmd,nofile
907 oceani: cannot open nofile
909 ###### output: cmd,/dev/null
910 oceani: could not find any code in /dev/null
912 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
913 oceani: cannot find section toast:nothing