1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
83 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_tests); do \
84 echo -n "Valgrind $$T.. "; \
85 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
86 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
87 > /dev/null 2>&1 ; then \
88 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
89 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
90 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
91 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
92 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
95 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
96 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
97 if (ran < (ran + skip) *0.968) exit(1) }' \
98 coverage/oceani.mdc.gcov
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
113 func main(argv:[argc::]string)
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
194 case 1: scratch:=42; num:="one"
201 for i:=0; then i=i+1; while i < 5:
208 // re-declare a CondScope variable
213 ###### output: varmerge
214 zero , one , two , three , many ,
217 ## Conditions and Loops
219 Now we need to test if/else and some different loops
222 oceani_tests += cond_loop
224 ###### test: cond_loop
232 for b:=1; then b=b+b; while b < 100:
235 // Newtons method for square root of 2
241 current := guess * guess
242 use +(current - target) > 0.000000001
244 guess = (guess + (target / guess) ) / 2
247 print "error is ", target - guess * guess
249 for j:=0; then j = j+3 ; while j < 10:
250 if j != 0 and then 20 / j > 3:
251 print "20 /", j," =", 20 / j
253 print "I won't calculate 20 /", j
254 pi ::= 3.1415926535897
255 if 355/113 == pi or else +(pi - 355/113) < 0.001:
257 print "lower" if 355/113 < pi else "higher"
259 if pi > 3 then print "pi exceeds three"; else print "need more pie"
260 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
261 for { i := 0; sum := 0 }
267 print "sum 1..10 is", sum
276 ###### output: cond_loop
283 error is -4.51095e-12
284 I won't calculate 20 / 0
287 I won't calculate 20 / 9
297 The demonstration code presented in the interpreted is suitable for the test suite.
298 Here I break it into two parts, keeping the array code separate.
301 oceani_tests += "sayhello,55,33,hello,True"
302 oceani_tests += "sayhello,12,60,there,False"
304 ###### test: sayhello
306 func main(av:[ac::number]string)
307 A := $av[1]; B := $av[2]
309 bbool := av[ac-1] == "True"
310 print "Hello World, what lovely oceans you have!"
311 /* When a variable is defined in both branches of an 'if',
312 * and used afterwards, the variables are merged.
318 print "Is", A, "bigger than", B,"? ", bigger
319 /* If a variable is not used after the 'if', no
320 * merge happens, so types can be different
323 double:string = "yes"
324 print A, "is more than twice", B, "?", double
327 print "double", B, "is", double
332 print "still", bigger // check for regression in scoping
339 print "GCD of", A, "and", B,"is", a
341 print a, "is not positive, cannot calculate GCD"
343 print b, "is not positive, cannot calculate GCD"
348 print "Fibonacci:", f1,f2,
358 print astr ++ " was the str"
360 print "I found the str over " ++ astr
362 /* Binary search... */
380 print "Yay, I found", target
382 print "Closest I found was", lo
384 ###### output: sayhello,55,33,hello,True
385 Hello World, what lovely oceans you have!
386 Is 55 bigger than 33 ? yes
389 GCD of 55 and 33 is 11
390 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
392 Closest I found was 77.3438
394 ###### output: sayhello,12,60,there,False
395 Hello World, what lovely oceans you have!
396 Is 12 bigger than 60 ? no
399 GCD of 12 and 60 is 12
400 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
401 I found the str over there
402 Closest I found was 77.3438
405 oceani_tests += "insert_sort"
406 ###### test: insert_sort
411 for i:=1; then i = i + 1; while i < size:
412 n := list[i-1] * list[i-1]
413 list[i] = (n / 100) % 10000
416 for i:=0; then i = i + 1; while i < size:
417 print "list[",i,"]=",list[i]
419 for i := 1; then i=i+1; while i < size:
420 for j:=i-1; then j=j-1; while j >= 0:
421 if list[j] > list[j+1]:
426 for i:=0; then i = i + 1; while i < size:
427 print "list[",i,"]=",list[i]
429 ###### output: insert_sort
545 We already have some array tests, but this is where we put other
546 ad-hoc things array related.
549 oceani_tests += arrays
557 bools[3] = strings[1] == "Hello"
558 bools[1] = strings[2] <= "there"
560 for i:=0; then i=i+1; while i<5:
564 ra[6] = 42 // mustn't crash
565 print '', bools[i], ra[j-1],
568 ###### output: arrays
569 False 0 True 1 False 4 False 9 False 16
573 Time to test if structure declarations and accesses work correctly.
576 oceani_tests += structs
584 active:Boolean = True
586 struct baz { a:number; b:Boolean; }
592 for i:=0; then i=i+1; while i < 4:
600 info[i].size[0] = i*i
602 info[i].active = False
604 for i:=0; then i=i+1; while i < 4:
605 print info[i].name, info[i].active, info[i].size[0]
607 ###### output: structs
616 Test functions. They don't return anything, so we need to get them to print
619 oceani_tests += functions func_ret_type
621 ###### test: functions
641 func test(n:number; s:string)
648 for i:=0; then i = i + 1; while i < 5:
651 ###### output: functions
656 4 3 . 2 .. 1 ... done
658 ###### test: func_ret_type
660 func double(n:number):number
676 for j:=10; then j = j - 3; while j > -5:
677 print answer("dou","ble"), j, "is", double(j)
679 ###### output: func_ret_type
686 ## Test code with syntax errors
688 Syntax errors aren't handled well yet - the result is almost always a
689 single message about the first error. So this section will be fairly
690 thin until we add proper parsing recovery in the face of common errors.
692 A special case of syntax errors is token errors, when a token is only
693 accepted because the parser doesn't know quite enough to reject it.
694 There are handled better as they are quite local, so a single test
695 program can trigger most of the possible errors.
697 To handle erronous code, we need a different set of tests, as we need to
698 capture `stderr`. The same test code will be used for type errors too.
699 As error messages contain the line number, and we don't want changes to
700 this file to change the reported numbers, we copy the code into a
701 separate file first, then run from there.
704 @for t in $(oceani_failing_tests); do \
705 echo -n "Test $$t ... "; \
706 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
707 echo '``````' > .tmp.code; \
708 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
709 ./oceani .tmp.code > .tmp.have 2>&1; \
710 if ! cmp -s .tmp.want .tmp.have; then \
711 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
713 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
717 oceani_failing_tests := syn1
718 oceani_failing_tests += tokerr
723 if then else while do
726 .tmp.code:3:11: Syntax error in statement: then
730 a := 1i // imaginary numbers aren't understood
731 b:[2i]number // array sizes are handled separately
732 c:[3.14159]Boolean // array sizes must be integers
733 d:[1_000_000_000_000]number // they mustn't be huge
734 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
737 This is a multiline string
738 With an unsupportable suffix
742 yy:[unknowable]number
744 zz:[zzsize]string // size must be constant, use ::=
746 // These numbers should be bad in all contexts: FIXME
749 ###### output: tokerr
750 .tmp.code:3:13: error: unsupported number suffix: 1i
751 .tmp.code:4:11: error: unsupported number suffix: 2i
752 .tmp.code:5:11: error: array size must be an integer: 3.14159
753 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
754 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
755 .tmp.code:9:17: error: unsupported string suffix: """
756 This is a multiline string
757 With an unsupportable suffix
759 .tmp.code:14:11: error: undefined type: unknown
760 .tmp.code:15:12: error: name undeclared: unknowable
761 .tmp.code:17:12: error: array size must be a constant: zzsize
762 .tmp.code:20:12: error: unrecognised number: 00123
764 ## Tests for type errors
766 Type error don't cause parsing to abort, so we can fit many in the
767 one test program. Some type errors are found during the parse, others
768 during type analysis which doesn't run if parsing failed. So we cannot
769 fit everything in one.
771 These programs were generated by looking for the
772 various places that `type_err()` are called.
775 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
777 ###### test: type_err1
780 print "hello" ++ 5, 5 ++ "hello"
785 if 3 * 4 and not True: print "Weird"
787 ###### output: type_err1
788 .tmp.code:3:25: error: expected string found number
789 .tmp.code:3:28: error: expected string found number
790 .tmp.code:6:8: error: Cannot assign to a constant: b
791 .tmp.code:5:8: info: name was defined as a constant here
792 .tmp.code:6:8: error: Cannot assign to a constant: b
793 .tmp.code:5:8: info: name was defined as a constant here
794 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
795 oceani: type error in program - not running.
797 ###### test: type_err2
807 ###### output: type_err2
808 .tmp.code:4:8: error: variable 'a' redeclared
809 .tmp.code:3:8: info: this is where 'a' was first declared
810 .tmp.code:5:8: error: variable 'a' redeclared
811 .tmp.code:3:8: info: this is where 'a' was first declared
812 .tmp.code:6:8: error: variable 'a' redeclared
813 .tmp.code:3:8: info: this is where 'a' was first declared
814 .tmp.code:7:8: error: variable 'a' redeclared
815 .tmp.code:3:8: info: this is where 'a' was first declared
816 .tmp.code:8:8: Variable declared with no type or value: c
818 ###### test: type_err3
827 c = "hello" ++ (True and False)
829 print 45 + ( "Hello" ++ "there")
839 case "Hello": print "Hello"
841 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
855 // trigger 'labels not permitted' error message
856 while 1 if True else False:
860 ###### output: type_err3
861 .tmp.code:8:12: error: expected number but variable 'c' is string
862 .tmp.code:7:8: info: this is where 'c' was set to string
863 .tmp.code:8:12: error: Arithmetic returns number but string expected
864 .tmp.code:7:8: info: variable 'c' was set as string here.
865 .tmp.code:9:24: error: Boolean operation found where string expected
866 .tmp.code:10:12: error: Comparison returns Boolean but string expected
867 .tmp.code:7:8: info: variable 'c' was set as string here.
868 .tmp.code:11:21: error: Concat returns string but number expected
869 .tmp.code:12:8: error: string cannot be indexed
870 .tmp.code:12:8: error: string cannot be indexed
871 .tmp.code:21:13: error: expected number found string
872 .tmp.code:17:16: error: expected number, found string
873 .tmp.code:24:8: error: cannot assign value of type [5]number
874 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
875 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
876 .tmp.code:25:8: error: cannot assign value of type [5]number
877 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
878 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
879 .tmp.code:26:8: error: cannot assign value of type [5]number
880 .tmp.code:27:16: error: expected number found string
881 .tmp.code:28:16: error: expected string found Boolean
882 .tmp.code:29:12: error: have number but need string
883 .tmp.code:7:8: info: variable 'c' was set as string here.
884 .tmp.code:32:8: error: variable used but not declared: foo
885 .tmp.code:32:8: error: field reference attempted on none, not a struct
886 .tmp.code:32:16: error: expected none found number
887 .tmp.code:33:14: error: field reference attempted on string, not a struct
888 .tmp.code:34:14: error: cannot find requested field in foo
889 .tmp.code:35:17: error: have string but need number
890 .tmp.code:38:29: error: expected number (labels not permitted) found Boolean
891 oceani: type error in program - not running.
893 ###### test: type_err4
898 ###### output: type_err4
899 .tmp.code:3:14: error: variable used but not declared: b
900 .tmp.code:3:16: error: expected none found number
901 .tmp.code:3:14: info: variable 'b' was set as none here.
902 oceani: type error in program - not running.
905 oceani_failing_tests += type_err_const type_err_const1 missing_program bad_main
907 ###### test: type_err_const
910 bar ::= "string" + 56
917 // trigger duplicate-main error
922 ###### output: type_err_const
923 .tmp.code:4:16: error: expected number found string
924 .tmp.code:6:8: error: name already declared: bar
925 .tmp.code:4:8: info: this is where 'bar' was first declared
926 .tmp.code:8:8: error: variable 'foo' redeclared
927 .tmp.code:3:8: info: this is where 'foo' was first declared
928 .tmp.code:12:5: error: function 'main' redeclared
929 .tmp.code:7:5: info: this is where 'main' was first declared
930 .tmp.code:13:8: error: variable 'foo' redeclared
931 .tmp.code:3:8: info: this is where 'foo' was first declared
933 ###### test: type_err_const1
941 ###### output: type_err_const1
942 .tmp.code:3:12: Syntax error in constant: :
943 .tmp.code:4:12: Syntax error in constant: :
945 ###### test: missing_program
949 ###### output: missing_program
950 oceani: no main function found.
952 ###### test: bad_main
953 func main(foo:string)
956 ###### output: bad_main
957 .tmp.code:??:??: error: expected argv but variable 'foo' is string
958 .tmp.code:??:??: info: this is where 'NOTVAR' was set to string
959 oceani: main has wrong type.
961 Test for type errors with functions
964 oceani_failing_tests += func_err_args func_err_redeclare
966 ###### test: func_err_args
968 func test1(a:number; b:string; c:[3]Boolean)
971 func test2(a:number; b:string; c:[3]Boolean)
975 # use undefined names
983 test1(1, "two", truth)
985 test1(1, "lo", truth, 4)
986 print test(), test1(1,2,3)
990 func test4(a:number):string
993 func test5(a:number):string
996 ###### output: func_err_args
997 .tmp.code:28:14: error: expected string, found none
998 .tmp.code:25:8: error: expected string, found number
999 .tmp.code:15:14: error: insufficient arguments to function.
1000 .tmp.code:16:14: error: expected number found string
1001 .tmp.code:16:22: error: expected string found number
1002 .tmp.code:16:14: error: insufficient arguments to function.
1003 .tmp.code:18:17: error: expected string found number
1004 .tmp.code:19:14: error: too many arguments to function.
1005 .tmp.code:20:14: error: attempt to call a non-function.
1006 .tmp.code:20:32: error: expected string found number
1007 .tmp.code:20:28: error: insufficient arguments to function.
1008 .tmp.code:21:20: error: expected func but variable 'test2' is func
1009 .tmp.code:??:??: info: this is where 'NOTVAR' was set to func
1010 .tmp.code:10:14: error: variable used but not declared: a
1011 .tmp.code:10:17: error: variable used but not declared: z
1012 oceani: type error in program - not running.
1014 ###### test: func_err_redeclare
1016 func test1(a:number; b:string; c:[3]Boolean)
1028 ###### output: func_err_redeclare
1029 .tmp.code:5:5: error: function 'test1' redeclared
1030 .tmp.code:2:5: info: this is where 'test1' was first declared
1031 .tmp.code:9:5: error: function 'test1' redeclared
1032 .tmp.code:2:5: info: this is where 'test1' was first declared
1034 ## Test erroneous command line args
1036 To improve coverage, we want to test correct handling of strange command
1037 line arguments. These tests won't use code, so the exiting test types
1038 won't work. So we need to be able to explicitly give the command line,
1039 and the expected output, and have that tested and the coverage assessed.
1040 Rather than having to spell out the whole command name, just give "cmd",
1041 and discard that. Requiring but discarding the command make an empty
1042 command list possible.
1045 @for t in $(oceani_special_tests); do \
1046 echo -n "Test $$t ... ";\
1047 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1048 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1049 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1050 if ! cmp -s .tmp.want .tmp.have; then \
1051 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1053 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1057 oceani_special_tests += "cmd"
1058 oceani_special_tests += "cmd,-zyx"
1059 oceani_special_tests += "cmd,nofile"
1060 oceani_special_tests += "cmd,/dev/null"
1061 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1064 oceani: no input file given
1066 ###### output: cmd,-zyx
1067 ./oceani: invalid option -- 'z'
1068 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1070 ###### output: cmd,nofile
1071 oceani: cannot open nofile
1073 ###### output: cmd,/dev/null
1074 oceani: could not find any code in /dev/null
1076 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1077 oceani: cannot find section toast:nothing