1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.9495) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
201 for i:=0; then i=i+1; while i < 5:
208 // re-declare a CondScope variable
213 ###### output: varmerge
214 zero , one , two , three , many ,
217 ## Conditions and Loops
219 Now we need to test if/else and some different loops
222 oceani_tests += cond_loop
224 ###### test: cond_loop
232 for b:=1; then b=b+b; while b < 100:
235 // Newtons method for square root of 2
241 current := guess * guess
242 use +(current - target) > 0.000000001
244 guess = (guess + (target / guess) ) / 2
247 print "error is ", target - guess * guess
249 for j:=0; then j = j+3 ; while j < 10:
250 if j != 0 and then 20 / j > 3:
251 print "20 /", j," =", 20 / j
253 print "I won't calculate 20 /", j
254 pi ::= 3.1415926535897
255 if 355/113 == pi or else +(pi - 355/113) < 0.001:
257 print "lower" if 355/113 < pi else "higher"
259 if pi > 3 then print "pi exceeds three"; else print "need more pie"
260 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
261 for { i := 0; sum := 0 }
267 print "sum 1..10 is", sum
269 ###### output: cond_loop
276 error is -4.51095e-12
277 I won't calculate 20 / 0
280 I won't calculate 20 / 9
289 The demonstration code presented in the interpreted is suitable for the test suite.
290 Here I break it into two parts, keeping the array code separate.
293 oceani_tests += "sayhello,55,33,hello,True"
294 oceani_tests += "sayhello,12,60,there,False"
296 ###### test: sayhello
299 A := $av[1]; B := $av[2]
301 bbool := av[ac-1] == "True"
302 print "Hello World, what lovely oceans you have!"
303 /* When a variable is defined in both branches of an 'if',
304 * and used afterwards, the variables are merged.
310 print "Is", A, "bigger than", B,"? ", bigger
311 /* If a variable is not used after the 'if', no
312 * merge happens, so types can be different
315 double:string = "yes"
316 print A, "is more than twice", B, "?", double
319 print "double", B, "is", double
330 print "GCD of", A, "and", B,"is", a
332 print a, "is not positive, cannot calculate GCD"
334 print b, "is not positive, cannot calculate GCD"
339 print "Fibonacci:", f1,f2,
349 print astr ++ " was the str"
351 print "I found the str over " ++ astr
353 /* Binary search... */
370 print "Yay, I found", target
372 print "Closest I found was", mid
374 ###### output: sayhello,55,33,hello,True
375 Hello World, what lovely oceans you have!
376 Is 55 bigger than 33 ? yes
378 GCD of 55 and 33 is 11
379 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
381 Closest I found was 77.3438
383 ###### output: sayhello,12,60,there,False
384 Hello World, what lovely oceans you have!
385 Is 12 bigger than 60 ? no
387 GCD of 12 and 60 is 12
388 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
389 I found the str over there
390 Closest I found was 77.3438
393 oceani_tests += "insert_sort"
394 ###### test: insert_sort
399 for i:=1; then i = i + 1; while i < size:
400 n := list[i-1] * list[i-1]
401 list[i] = (n / 100) % 10000
404 for i:=0; then i = i + 1; while i < size:
405 print "list[",i,"]=",list[i]
407 for i := 1; then i=i+1; while i < size:
408 for j:=i-1; then j=j-1; while j >= 0:
409 if list[j] > list[j+1]:
414 for i:=0; then i = i + 1; while i < size:
415 print "list[",i,"]=",list[i]
417 ###### output: insert_sort
533 We already have some array tests, but this is where we put other
534 ad-hoc things array related.
537 oceani_tests += arrays
545 bools[3] = strings[1] == "Hello"
546 bools[1] = strings[2] <= "there"
548 for i:=0; then i=i+1; while i<5:
552 ra[6] = 42 // mustn't crash
553 print '', bools[i], ra[j-1],
556 ###### output: arrays
557 False 0 True 1 False 4 False 9 False 16
561 Time to test if structure declarations and accesses work correctly.
564 oceani_tests += structs
572 active:Boolean = True
574 struct baz { a:number; b:Boolean; }
579 for i:=0; then i=i+1; while i < 4:
587 info[i].size[0] = i*i
589 info[i].active = False
591 for i:=0; then i=i+1; while i < 4:
592 print info[i].name, info[i].active, info[i].size[0]
594 ###### output: structs
601 ## Test code with syntax errors
603 Syntax errors aren't handled well yet - the result is almost always a
604 single message about the first error. So this section will be fairly
605 thin until we add proper parsing recovery in the face of common errors.
607 A special case of syntax errors is token errors, when a token is only
608 accepted because the parser doesn't know quite enough to reject it.
609 There are handled better as they are quite local, so a single test
610 program can trigger most of the possible errors.
612 To handle erronous code, we need a different set of tests, as we need to
613 capture `stderr`. The same test code will be used for type errors too.
614 As error messages contain the line number, and we don't want changes to
615 this file to change the reported numbers, we copy the code into a
616 separate file first, then run from there.
619 @for t in $(oceani_failing_tests); do \
620 echo -n "Test $$t ... "; \
621 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
622 echo '``````' > .tmp.code; \
623 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
624 ./oceani .tmp.code > .tmp.have 2>&1; \
625 if ! cmp -s .tmp.want .tmp.have; then \
626 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
628 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
632 oceani_failing_tests := syn1
633 oceani_failing_tests += tokerr
638 if then else while do
641 .tmp.code:3:11: Syntax error in statement: then
645 a := 1i // imaginary numbers aren't understood
646 b:[2i]number // array sizes are handled separately
647 c:[3.14159]Boolean // array sizes must be integers
648 d:[1_000_000_000_000]number // they mustn't be huge
649 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
652 This is a multiline string
653 With an unsupportable suffix
657 yy:[unknowable]number
659 zz:[zzsize]string // size must be constant, use ::=
661 // These numbers should be bad in all contexts: FIXME
664 ###### output: tokerr
665 .tmp.code:3:13: error: unsupported number suffix: 1i
666 .tmp.code:4:11: error: unsupported number suffix: 2i
667 .tmp.code:5:11: error: array size must be an integer: 3.14159
668 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
669 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
670 .tmp.code:9:17: error: unsupported string suffix: """
671 This is a multiline string
672 With an unsupportable suffix
674 .tmp.code:14:11: error: undefined type: unknown
675 .tmp.code:15:12: error: name undeclared: unknowable
676 .tmp.code:17:12: error: array size must be a constant: zzsize
677 .tmp.code:20:12: error: unrecognised number: 00123
679 ## Tests for type errors
681 Type error don't cause parsing to abort, so we can fit many in the
682 one test program. Some type errors are found during the parse, others
683 during type analysis which doesn't run if parsing failed. So we cannot
684 fit everything in one.
686 These programs were generated by looking for the
687 various places that `type_err()` are called.
690 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
692 ###### test: type_err1
695 print "hello" ++ 5, 5 ++ "hello"
700 if 3 * 4 and not True: print "Weird"
702 ###### output: type_err1
703 .tmp.code:3:25: error: expected string found number
704 .tmp.code:3:28: error: expected string found number
705 .tmp.code:6:8: error: Cannot assign to a constant: b
706 .tmp.code:5:8: info: name was defined as a constant here
707 .tmp.code:6:8: error: Cannot assign to a constant: b
708 .tmp.code:5:8: info: name was defined as a constant here
709 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
710 oceani: type error in program - not running.
712 ###### test: type_err2
722 ###### output: type_err2
723 .tmp.code:4:8: error: variable 'a' redeclared
724 .tmp.code:3:8: info: this is where 'a' was first declared
725 .tmp.code:5:8: error: variable 'a' redeclared
726 .tmp.code:3:8: info: this is where 'a' was first declared
727 .tmp.code:6:8: error: variable 'a' redeclared
728 .tmp.code:3:8: info: this is where 'a' was first declared
729 .tmp.code:7:8: error: variable 'a' redeclared
730 .tmp.code:3:8: info: this is where 'a' was first declared
731 .tmp.code:8:8: Variable declared with no type or value: c
733 ###### test: type_err3
742 c = "hello" ++ (True and False)
744 print 45 + ( "Hello" ++ "there")
754 case "Hello": print "Hello"
756 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
769 ###### output: type_err3
770 .tmp.code:8:12: error: expected number but variable 'c' is string
771 .tmp.code:7:8: info: this is where 'c' was set to string
772 .tmp.code:8:12: error: Arithmetic returns number but string expected
773 .tmp.code:7:8: info: variable 'c' was set as string here.
774 .tmp.code:9:24: error: Boolean operation found where string expected
775 .tmp.code:10:12: error: Comparison returns Boolean but string expected
776 .tmp.code:7:8: info: variable 'c' was set as string here.
777 .tmp.code:11:21: error: Concat returns string but number expected
778 .tmp.code:12:8: error: string cannot be indexed
779 .tmp.code:12:8: error: string cannot be indexed
780 .tmp.code:21:13: error: expected number found string
781 .tmp.code:17:16: error: expected number, found string
782 .tmp.code:24:8: error: cannot assign value of type [5]number
783 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
784 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
785 .tmp.code:25:8: error: cannot assign value of type [5]number
786 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
787 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
788 .tmp.code:26:8: error: cannot assign value of type [5]number
789 .tmp.code:27:16: error: expected number found string
790 .tmp.code:28:16: error: expected string found Boolean
791 .tmp.code:29:12: error: have number but need string
792 .tmp.code:7:8: info: variable 'c' was set as string here.
793 .tmp.code:32:8: error: variable used but not declared: foo
794 .tmp.code:32:8: error: field reference attempted on none, not a struct
795 .tmp.code:32:16: error: expected none found number
796 .tmp.code:33:14: error: field reference attempted on string, not a struct
797 .tmp.code:34:14: error: cannot find requested field in foo
798 oceani: type error in program - not running.
800 ###### test: type_err4
805 ###### output: type_err4
806 .tmp.code:3:14: error: variable used but not declared: b
807 .tmp.code:3:16: error: expected none found number
808 .tmp.code:3:14: info: variable 'b' was set as none here.
809 oceani: type error in program - not running.
812 oceani_failing_tests += type_err_const type_err_const1 missing_program
814 ###### test: type_err_const
817 bar ::= "string" + 56
824 ###### output: type_err_const
825 .tmp.code:4:16: error: expected number found string
826 .tmp.code:6:8: error: name already declared: bar
827 .tmp.code:4:8: info: this is where 'bar' was first declared
828 .tmp.code:8:8: error: variable 'foo' redeclared
829 .tmp.code:3:8: info: this is where 'foo' was first declared
831 ###### test: type_err_const1
839 ###### output: type_err_const1
840 .tmp.code:3:12: Syntax error in constant: :
841 .tmp.code:4:12: Syntax error in constant: :
843 ###### test: missing_program
847 ###### output: missing_program
848 oceani: no program found.
850 ## Test erroneous command line args
852 To improve coverage, we want to test correct handling of strange command
853 line arguments. These tests won't use code, so the exiting test types
854 won't work. So we need to be able to explicitly give the command line,
855 and the expected output, and have that tested and the coverage assessed.
856 Rather than having to spell out the whole command name, just give "cmd",
857 and discard that. Requiring but discarding the command make an empty
858 command list possible.
861 @for t in $(oceani_special_tests); do \
862 echo -n "Test $$t ... ";\
863 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
864 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
865 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
866 if ! cmp -s .tmp.want .tmp.have; then \
867 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
869 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
873 oceani_special_tests += "cmd"
874 oceani_special_tests += "cmd,-zyx"
875 oceani_special_tests += "cmd,nofile"
876 oceani_special_tests += "cmd,/dev/null"
877 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
880 oceani: no input file given
882 ###### output: cmd,-zyx
883 ./oceani: invalid option -- 'z'
884 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
886 ###### output: cmd,nofile
887 oceani: cannot open nofile
889 ###### output: cmd,/dev/null
890 oceani: could not find any code in /dev/null
892 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
893 oceani: cannot find section toast:nothing