1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.968) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
113 func main(argv:[argc::]string)
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
194 case 1: scratch:=42; num:="one"
201 for i:=0; then i=i+1; while i < 5:
208 // re-declare a CondScope variable
213 ###### output: varmerge
214 zero , one , two , three , many ,
217 ## Conditions and Loops
219 Now we need to test if/else and some different loops
222 oceani_tests += cond_loop
224 ###### test: cond_loop
232 for b:=1; then b=b+b; while b < 100:
235 // Newtons method for square root of 2
241 current := guess * guess
242 use +(current - target) > 0.000000001
244 guess = (guess + (target / guess) ) / 2
247 print "error is ", target - guess * guess
249 for j:=0; then j = j+3 ; while j < 10:
250 if j != 0 and then 20 / j > 3:
251 print "20 /", j," =", 20 / j
253 print "I won't calculate 20 /", j
254 pi ::= 3.1415926535897
255 if 355/113 == pi or else +(pi - 355/113) < 0.001:
257 print "lower" if 355/113 < pi else "higher"
259 if pi > 3 then print "pi exceeds three"; else print "need more pie"
260 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
261 for { i := 0; sum := 0 }
267 print "sum 1..10 is", sum
276 ###### output: cond_loop
283 error is -4.51095e-12
284 I won't calculate 20 / 0
287 I won't calculate 20 / 9
297 The demonstration code presented in the interpreted is suitable for the test suite.
298 Here I break it into two parts, keeping the array code separate.
301 oceani_tests += "sayhello,55,33,hello,True"
302 oceani_tests += "sayhello,12,60,there,False"
304 ###### test: sayhello
306 func main(av:[ac::number]string)
307 A := $av[1]; B := $av[2]
309 bbool := av[ac-1] == "True"
310 print "Hello World, what lovely oceans you have!"
311 /* When a variable is defined in both branches of an 'if',
312 * and used afterwards, the variables are merged.
318 print "Is", A, "bigger than", B,"? ", bigger
319 /* If a variable is not used after the 'if', no
320 * merge happens, so types can be different
323 double:string = "yes"
324 print A, "is more than twice", B, "?", double
327 print "double", B, "is", double
332 print "still", bigger // check for regression in scoping
339 print "GCD of", A, "and", B,"is", a
341 print a, "is not positive, cannot calculate GCD"
343 print b, "is not positive, cannot calculate GCD"
348 print "Fibonacci:", f1,f2,
358 print astr ++ " was the str"
360 print "I found the str over " ++ astr
362 /* Binary search... */
380 print "Yay, I found", target
382 print "Closest I found was", lo
384 ###### output: sayhello,55,33,hello,True
385 Hello World, what lovely oceans you have!
386 Is 55 bigger than 33 ? yes
389 GCD of 55 and 33 is 11
390 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
392 Closest I found was 77.3438
394 ###### output: sayhello,12,60,there,False
395 Hello World, what lovely oceans you have!
396 Is 12 bigger than 60 ? no
399 GCD of 12 and 60 is 12
400 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
401 I found the str over there
402 Closest I found was 77.3438
405 oceani_tests += "insert_sort"
406 ###### test: insert_sort
411 for i:=1; then i = i + 1; while i < size:
412 n := list[i-1] * list[i-1]
413 list[i] = (n / 100) % 10000
416 for i:=0; then i = i + 1; while i < size:
417 print "list[",i,"]=",list[i]
419 for i := 1; then i=i+1; while i < size:
420 for j:=i-1; then j=j-1; while j >= 0:
421 if list[j] > list[j+1]:
426 for i:=0; then i = i + 1; while i < size:
427 print "list[",i,"]=",list[i]
429 ###### output: insert_sort
545 We already have some array tests, but this is where we put other
546 ad-hoc things array related.
549 oceani_tests += arrays
557 bools[3] = strings[1] == "Hello"
558 bools[1] = strings[2] <= "there"
560 for i:=0; then i=i+1; while i<5:
564 ra[6] = 42 // mustn't crash
565 print '', bools[i], ra[j-1],
568 ###### output: arrays
569 False 0 True 1 False 4 False 9 False 16
573 Time to test if structure declarations and accesses work correctly.
576 oceani_tests += structs
584 active:Boolean = True
586 struct baz { a:number; b:Boolean; }
592 for i:=0; then i=i+1; while i < 4:
600 info[i].size[0] = i*i
602 info[i].active = False
604 for i:=0; then i=i+1; while i < 4:
605 print info[i].name, info[i].active, info[i].size[0]
607 ###### output: structs
616 Test functions. They don't return anything, so we need to get them to print
619 oceani_tests += functions
621 ###### test: functions
633 func test(n:number; s:string)
640 for i:=0; then i = i + 1; while i < 5:
643 ###### output: functions
648 4 3 . 2 .. 1 ... done
650 ## Test code with syntax errors
652 Syntax errors aren't handled well yet - the result is almost always a
653 single message about the first error. So this section will be fairly
654 thin until we add proper parsing recovery in the face of common errors.
656 A special case of syntax errors is token errors, when a token is only
657 accepted because the parser doesn't know quite enough to reject it.
658 There are handled better as they are quite local, so a single test
659 program can trigger most of the possible errors.
661 To handle erronous code, we need a different set of tests, as we need to
662 capture `stderr`. The same test code will be used for type errors too.
663 As error messages contain the line number, and we don't want changes to
664 this file to change the reported numbers, we copy the code into a
665 separate file first, then run from there.
668 @for t in $(oceani_failing_tests); do \
669 echo -n "Test $$t ... "; \
670 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
671 echo '``````' > .tmp.code; \
672 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
673 ./oceani .tmp.code > .tmp.have 2>&1; \
674 if ! cmp -s .tmp.want .tmp.have; then \
675 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
677 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
681 oceani_failing_tests := syn1
682 oceani_failing_tests += tokerr
687 if then else while do
690 .tmp.code:3:11: Syntax error in statement: then
694 a := 1i // imaginary numbers aren't understood
695 b:[2i]number // array sizes are handled separately
696 c:[3.14159]Boolean // array sizes must be integers
697 d:[1_000_000_000_000]number // they mustn't be huge
698 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
701 This is a multiline string
702 With an unsupportable suffix
706 yy:[unknowable]number
708 zz:[zzsize]string // size must be constant, use ::=
710 // These numbers should be bad in all contexts: FIXME
713 ###### output: tokerr
714 .tmp.code:3:13: error: unsupported number suffix: 1i
715 .tmp.code:4:11: error: unsupported number suffix: 2i
716 .tmp.code:5:11: error: array size must be an integer: 3.14159
717 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
718 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
719 .tmp.code:9:17: error: unsupported string suffix: """
720 This is a multiline string
721 With an unsupportable suffix
723 .tmp.code:14:11: error: undefined type: unknown
724 .tmp.code:15:12: error: name undeclared: unknowable
725 .tmp.code:17:12: error: array size must be a constant: zzsize
726 .tmp.code:20:12: error: unrecognised number: 00123
728 ## Tests for type errors
730 Type error don't cause parsing to abort, so we can fit many in the
731 one test program. Some type errors are found during the parse, others
732 during type analysis which doesn't run if parsing failed. So we cannot
733 fit everything in one.
735 These programs were generated by looking for the
736 various places that `type_err()` are called.
739 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
741 ###### test: type_err1
744 print "hello" ++ 5, 5 ++ "hello"
749 if 3 * 4 and not True: print "Weird"
751 ###### output: type_err1
752 .tmp.code:3:25: error: expected string found number
753 .tmp.code:3:28: error: expected string found number
754 .tmp.code:6:8: error: Cannot assign to a constant: b
755 .tmp.code:5:8: info: name was defined as a constant here
756 .tmp.code:6:8: error: Cannot assign to a constant: b
757 .tmp.code:5:8: info: name was defined as a constant here
758 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
759 oceani: type error in program - not running.
761 ###### test: type_err2
771 ###### output: type_err2
772 .tmp.code:4:8: error: variable 'a' redeclared
773 .tmp.code:3:8: info: this is where 'a' was first declared
774 .tmp.code:5:8: error: variable 'a' redeclared
775 .tmp.code:3:8: info: this is where 'a' was first declared
776 .tmp.code:6:8: error: variable 'a' redeclared
777 .tmp.code:3:8: info: this is where 'a' was first declared
778 .tmp.code:7:8: error: variable 'a' redeclared
779 .tmp.code:3:8: info: this is where 'a' was first declared
780 .tmp.code:8:8: Variable declared with no type or value: c
782 ###### test: type_err3
791 c = "hello" ++ (True and False)
793 print 45 + ( "Hello" ++ "there")
803 case "Hello": print "Hello"
805 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
820 // trigger 'labels not permitted' error message
821 while 1 if True else False:
825 ###### output: type_err3
826 .tmp.code:8:12: error: expected number but variable 'c' is string
827 .tmp.code:7:8: info: this is where 'c' was set to string
828 .tmp.code:8:12: error: Arithmetic returns number but string expected
829 .tmp.code:7:8: info: variable 'c' was set as string here.
830 .tmp.code:9:24: error: Boolean operation found where string expected
831 .tmp.code:10:12: error: Comparison returns Boolean but string expected
832 .tmp.code:7:8: info: variable 'c' was set as string here.
833 .tmp.code:11:21: error: Concat returns string but number expected
834 .tmp.code:12:8: error: string cannot be indexed
835 .tmp.code:12:8: error: string cannot be indexed
836 .tmp.code:21:13: error: expected number found string
837 .tmp.code:17:16: error: expected number, found string
838 .tmp.code:24:8: error: cannot assign value of type [5]number
839 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
840 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
841 .tmp.code:25:8: error: cannot assign value of type [5]number
842 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
843 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
844 .tmp.code:26:8: error: cannot assign value of type [5]number
845 .tmp.code:27:16: error: expected number found string
846 .tmp.code:28:16: error: expected string found Boolean
847 .tmp.code:29:12: error: have number but need string
848 .tmp.code:7:8: info: variable 'c' was set as string here.
849 .tmp.code:32:8: error: variable used but not declared: foo
850 .tmp.code:32:8: error: field reference attempted on none, not a struct
851 .tmp.code:32:16: error: expected none found number
852 .tmp.code:33:14: error: field reference attempted on string, not a struct
853 .tmp.code:34:14: error: cannot find requested field in foo
854 .tmp.code:35:17: error: have string but need number
855 .tmp.code:39:29: error: expected number (labels not permitted) found Boolean
856 oceani: type error in program - not running.
858 ###### test: type_err4
863 ###### output: type_err4
864 .tmp.code:3:14: error: variable used but not declared: b
865 .tmp.code:3:16: error: expected none found number
866 .tmp.code:3:14: info: variable 'b' was set as none here.
867 oceani: type error in program - not running.
870 oceani_failing_tests += type_err_const type_err_const1 missing_program bad_main
872 ###### test: type_err_const
875 bar ::= "string" + 56
882 // trigger duplicate-main error
887 ###### output: type_err_const
888 .tmp.code:4:16: error: expected number found string
889 .tmp.code:6:8: error: name already declared: bar
890 .tmp.code:4:8: info: this is where 'bar' was first declared
891 .tmp.code:8:8: error: variable 'foo' redeclared
892 .tmp.code:3:8: info: this is where 'foo' was first declared
893 .tmp.code:12:5: error: function 'main' redeclared
894 .tmp.code:7:5: info: this is where 'main' was first declared
895 .tmp.code:13:8: error: variable 'foo' redeclared
896 .tmp.code:3:8: info: this is where 'foo' was first declared
898 ###### test: type_err_const1
906 ###### output: type_err_const1
907 .tmp.code:3:12: Syntax error in constant: :
908 .tmp.code:4:12: Syntax error in constant: :
910 ###### test: missing_program
914 ###### output: missing_program
915 oceani: no main function found.
917 ###### test: bad_main
918 func main(foo:string)
921 ###### output: bad_main
922 .tmp.code:??:??: error: expected argv but variable 'foo' is string
923 .tmp.code:??:??: info: this is where 'NOTVAR' was set to string
924 oceani: main has wrong type.
926 Test for type errors with functions
929 oceani_failing_tests += func_err_args func_err_redeclare
931 ###### test: func_err_args
933 func test1(a:number; b:string; c:[3]Boolean)
936 func test2(a:number; b:string; c:[3]Boolean)
944 test1(1, "two", truth)
946 test1(1, "lo", truth, 4)
947 print test(), test1(1,2,3)
951 ###### output: func_err_args
952 .tmp.code:11:14: error: insufficient arguments to function.
953 .tmp.code:12:14: error: expected number found string
954 .tmp.code:12:22: error: expected string found number
955 .tmp.code:12:14: error: insufficient arguments to function.
956 .tmp.code:14:17: error: expected string found number
957 .tmp.code:15:14: error: too many arguments to function.
958 .tmp.code:16:14: error: attempt to call a non-function.
959 .tmp.code:16:32: error: expected string found number
960 .tmp.code:16:28: error: insufficient arguments to function.
961 .tmp.code:17:20: error: expected func but variable 'test2' is func
962 .tmp.code:??:??: info: this is where 'NOTVAR' was set to func
963 oceani: type error in program - not running.
965 ###### test: func_err_redeclare
967 func test1(a:number; b:string; c:[3]Boolean)
979 ###### output: func_err_redeclare
980 .tmp.code:5:5: error: function 'test1' redeclared
981 .tmp.code:2:5: info: this is where 'test1' was first declared
982 .tmp.code:9:5: error: function 'test1' redeclared
983 .tmp.code:2:5: info: this is where 'test1' was first declared
985 ## Test erroneous command line args
987 To improve coverage, we want to test correct handling of strange command
988 line arguments. These tests won't use code, so the exiting test types
989 won't work. So we need to be able to explicitly give the command line,
990 and the expected output, and have that tested and the coverage assessed.
991 Rather than having to spell out the whole command name, just give "cmd",
992 and discard that. Requiring but discarding the command make an empty
993 command list possible.
996 @for t in $(oceani_special_tests); do \
997 echo -n "Test $$t ... ";\
998 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
999 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1000 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1001 if ! cmp -s .tmp.want .tmp.have; then \
1002 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1004 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1008 oceani_special_tests += "cmd"
1009 oceani_special_tests += "cmd,-zyx"
1010 oceani_special_tests += "cmd,nofile"
1011 oceani_special_tests += "cmd,/dev/null"
1012 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1015 oceani: no input file given
1017 ###### output: cmd,-zyx
1018 ./oceani: invalid option -- 'z'
1019 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1021 ###### output: cmd,nofile
1022 oceani: cannot open nofile
1024 ###### output: cmd,/dev/null
1025 oceani: could not find any code in /dev/null
1027 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1028 oceani: cannot find section toast:nothing