1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.968) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
113 func main(argv:[argc::]string)
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
194 case 1: scratch:=42; num:="one"
201 for i:=0; then i=i+1; while i < 5:
208 // re-declare a CondScope variable
213 ###### output: varmerge
214 zero , one , two , three , many ,
217 ## Conditions and Loops
219 Now we need to test if/else and some different loops
222 oceani_tests += cond_loop
224 ###### test: cond_loop
232 for b:=1; then b=b+b; while b < 100:
235 // Newtons method for square root of 2
241 current := guess * guess
242 use +(current - target) > 0.000000001
244 guess = (guess + (target / guess) ) / 2
247 print "error is ", target - guess * guess
249 for j:=0; then j = j+3 ; while j < 10:
250 if j != 0 and then 20 / j > 3:
251 print "20 /", j," =", 20 / j
253 print "I won't calculate 20 /", j
254 pi ::= 3.1415926535897
255 if 355/113 == pi or else +(pi - 355/113) < 0.001:
257 print "lower" if 355/113 < pi else "higher"
259 if pi > 3 then print "pi exceeds three"; else print "need more pie"
260 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
261 for { i := 0; sum := 0 }
267 print "sum 1..10 is", sum
276 ###### output: cond_loop
283 error is -4.51095e-12
284 I won't calculate 20 / 0
287 I won't calculate 20 / 9
297 The demonstration code presented in the interpreted is suitable for the test suite.
298 Here I break it into two parts, keeping the array code separate.
301 oceani_tests += "sayhello,55,33,hello,True"
302 oceani_tests += "sayhello,12,60,there,False"
304 ###### test: sayhello
306 func main(av:[ac::number]string)
307 A := $av[1]; B := $av[2]
309 bbool := av[ac-1] == "True"
310 print "Hello World, what lovely oceans you have!"
311 /* When a variable is defined in both branches of an 'if',
312 * and used afterwards, the variables are merged.
318 print "Is", A, "bigger than", B,"? ", bigger
319 /* If a variable is not used after the 'if', no
320 * merge happens, so types can be different
323 double:string = "yes"
324 print A, "is more than twice", B, "?", double
327 print "double", B, "is", double
332 print "still", bigger // check for regression in scoping
339 print "GCD of", A, "and", B,"is", a
341 print a, "is not positive, cannot calculate GCD"
343 print b, "is not positive, cannot calculate GCD"
348 print "Fibonacci:", f1,f2,
358 print astr ++ " was the str"
360 print "I found the str over " ++ astr
362 /* Binary search... */
380 print "Yay, I found", target
382 print "Closest I found was", lo
384 ###### output: sayhello,55,33,hello,True
385 Hello World, what lovely oceans you have!
386 Is 55 bigger than 33 ? yes
389 GCD of 55 and 33 is 11
390 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
392 Closest I found was 77.3438
394 ###### output: sayhello,12,60,there,False
395 Hello World, what lovely oceans you have!
396 Is 12 bigger than 60 ? no
399 GCD of 12 and 60 is 12
400 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
401 I found the str over there
402 Closest I found was 77.3438
405 oceani_tests += "insert_sort"
406 ###### test: insert_sort
411 for i:=1; then i = i + 1; while i < size:
412 n := list[i-1] * list[i-1]
413 list[i] = (n / 100) % 10000
416 for i:=0; then i = i + 1; while i < size:
417 print "list[",i,"]=",list[i]
419 for i := 1; then i=i+1; while i < size:
420 for j:=i-1; then j=j-1; while j >= 0:
421 if list[j] > list[j+1]:
426 for i:=0; then i = i + 1; while i < size:
427 print "list[",i,"]=",list[i]
429 ###### output: insert_sort
545 We already have some array tests, but this is where we put other
546 ad-hoc things array related.
549 oceani_tests += arrays
557 bools[3] = strings[1] == "Hello"
558 bools[1] = strings[2] <= "there"
560 for i:=0; then i=i+1; while i<5:
564 ra[6] = 42 // mustn't crash
565 print '', bools[i], ra[j-1],
568 ###### output: arrays
569 False 0 True 1 False 4 False 9 False 16
573 Time to test if structure declarations and accesses work correctly.
576 oceani_tests += structs
584 active:Boolean = True
586 struct baz { a:number; b:Boolean; }
592 for i:=0; then i=i+1; while i < 4:
600 info[i].size[0] = i*i
602 info[i].active = False
604 for i:=0; then i=i+1; while i < 4:
605 print info[i].name, info[i].active, info[i].size[0]
607 ###### output: structs
616 Test functions. They don't return anything, so we need to get them to print
619 oceani_tests += functions
621 ###### test: functions
641 func test(n:number; s:string)
648 for i:=0; then i = i + 1; while i < 5:
651 ###### output: functions
656 4 3 . 2 .. 1 ... done
658 ## Test code with syntax errors
660 Syntax errors aren't handled well yet - the result is almost always a
661 single message about the first error. So this section will be fairly
662 thin until we add proper parsing recovery in the face of common errors.
664 A special case of syntax errors is token errors, when a token is only
665 accepted because the parser doesn't know quite enough to reject it.
666 There are handled better as they are quite local, so a single test
667 program can trigger most of the possible errors.
669 To handle erronous code, we need a different set of tests, as we need to
670 capture `stderr`. The same test code will be used for type errors too.
671 As error messages contain the line number, and we don't want changes to
672 this file to change the reported numbers, we copy the code into a
673 separate file first, then run from there.
676 @for t in $(oceani_failing_tests); do \
677 echo -n "Test $$t ... "; \
678 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
679 echo '``````' > .tmp.code; \
680 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
681 ./oceani .tmp.code > .tmp.have 2>&1; \
682 if ! cmp -s .tmp.want .tmp.have; then \
683 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
685 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
689 oceani_failing_tests := syn1
690 oceani_failing_tests += tokerr
695 if then else while do
698 .tmp.code:3:11: Syntax error in statement: then
702 a := 1i // imaginary numbers aren't understood
703 b:[2i]number // array sizes are handled separately
704 c:[3.14159]Boolean // array sizes must be integers
705 d:[1_000_000_000_000]number // they mustn't be huge
706 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
709 This is a multiline string
710 With an unsupportable suffix
714 yy:[unknowable]number
716 zz:[zzsize]string // size must be constant, use ::=
718 // These numbers should be bad in all contexts: FIXME
721 ###### output: tokerr
722 .tmp.code:3:13: error: unsupported number suffix: 1i
723 .tmp.code:4:11: error: unsupported number suffix: 2i
724 .tmp.code:5:11: error: array size must be an integer: 3.14159
725 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
726 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
727 .tmp.code:9:17: error: unsupported string suffix: """
728 This is a multiline string
729 With an unsupportable suffix
731 .tmp.code:14:11: error: undefined type: unknown
732 .tmp.code:15:12: error: name undeclared: unknowable
733 .tmp.code:17:12: error: array size must be a constant: zzsize
734 .tmp.code:20:12: error: unrecognised number: 00123
736 ## Tests for type errors
738 Type error don't cause parsing to abort, so we can fit many in the
739 one test program. Some type errors are found during the parse, others
740 during type analysis which doesn't run if parsing failed. So we cannot
741 fit everything in one.
743 These programs were generated by looking for the
744 various places that `type_err()` are called.
747 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
749 ###### test: type_err1
752 print "hello" ++ 5, 5 ++ "hello"
757 if 3 * 4 and not True: print "Weird"
759 ###### output: type_err1
760 .tmp.code:3:25: error: expected string found number
761 .tmp.code:3:28: error: expected string found number
762 .tmp.code:6:8: error: Cannot assign to a constant: b
763 .tmp.code:5:8: info: name was defined as a constant here
764 .tmp.code:6:8: error: Cannot assign to a constant: b
765 .tmp.code:5:8: info: name was defined as a constant here
766 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
767 oceani: type error in program - not running.
769 ###### test: type_err2
779 ###### output: type_err2
780 .tmp.code:4:8: error: variable 'a' redeclared
781 .tmp.code:3:8: info: this is where 'a' was first declared
782 .tmp.code:5:8: error: variable 'a' redeclared
783 .tmp.code:3:8: info: this is where 'a' was first declared
784 .tmp.code:6:8: error: variable 'a' redeclared
785 .tmp.code:3:8: info: this is where 'a' was first declared
786 .tmp.code:7:8: error: variable 'a' redeclared
787 .tmp.code:3:8: info: this is where 'a' was first declared
788 .tmp.code:8:8: Variable declared with no type or value: c
790 ###### test: type_err3
799 c = "hello" ++ (True and False)
801 print 45 + ( "Hello" ++ "there")
811 case "Hello": print "Hello"
813 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
827 // trigger 'labels not permitted' error message
828 while 1 if True else False:
832 ###### output: type_err3
833 .tmp.code:8:12: error: expected number but variable 'c' is string
834 .tmp.code:7:8: info: this is where 'c' was set to string
835 .tmp.code:8:12: error: Arithmetic returns number but string expected
836 .tmp.code:7:8: info: variable 'c' was set as string here.
837 .tmp.code:9:24: error: Boolean operation found where string expected
838 .tmp.code:10:12: error: Comparison returns Boolean but string expected
839 .tmp.code:7:8: info: variable 'c' was set as string here.
840 .tmp.code:11:21: error: Concat returns string but number expected
841 .tmp.code:12:8: error: string cannot be indexed
842 .tmp.code:12:8: error: string cannot be indexed
843 .tmp.code:21:13: error: expected number found string
844 .tmp.code:17:16: error: expected number, found string
845 .tmp.code:24:8: error: cannot assign value of type [5]number
846 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
847 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
848 .tmp.code:25:8: error: cannot assign value of type [5]number
849 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
850 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
851 .tmp.code:26:8: error: cannot assign value of type [5]number
852 .tmp.code:27:16: error: expected number found string
853 .tmp.code:28:16: error: expected string found Boolean
854 .tmp.code:29:12: error: have number but need string
855 .tmp.code:7:8: info: variable 'c' was set as string here.
856 .tmp.code:32:8: error: variable used but not declared: foo
857 .tmp.code:32:8: error: field reference attempted on none, not a struct
858 .tmp.code:32:16: error: expected none found number
859 .tmp.code:33:14: error: field reference attempted on string, not a struct
860 .tmp.code:34:14: error: cannot find requested field in foo
861 .tmp.code:35:17: error: have string but need number
862 .tmp.code:38:29: error: expected number (labels not permitted) found Boolean
863 oceani: type error in program - not running.
865 ###### test: type_err4
870 ###### output: type_err4
871 .tmp.code:3:14: error: variable used but not declared: b
872 .tmp.code:3:16: error: expected none found number
873 .tmp.code:3:14: info: variable 'b' was set as none here.
874 oceani: type error in program - not running.
877 oceani_failing_tests += type_err_const type_err_const1 missing_program bad_main
879 ###### test: type_err_const
882 bar ::= "string" + 56
889 // trigger duplicate-main error
894 ###### output: type_err_const
895 .tmp.code:4:16: error: expected number found string
896 .tmp.code:6:8: error: name already declared: bar
897 .tmp.code:4:8: info: this is where 'bar' was first declared
898 .tmp.code:8:8: error: variable 'foo' redeclared
899 .tmp.code:3:8: info: this is where 'foo' was first declared
900 .tmp.code:12:5: error: function 'main' redeclared
901 .tmp.code:7:5: info: this is where 'main' was first declared
902 .tmp.code:13:8: error: variable 'foo' redeclared
903 .tmp.code:3:8: info: this is where 'foo' was first declared
905 ###### test: type_err_const1
913 ###### output: type_err_const1
914 .tmp.code:3:12: Syntax error in constant: :
915 .tmp.code:4:12: Syntax error in constant: :
917 ###### test: missing_program
921 ###### output: missing_program
922 oceani: no main function found.
924 ###### test: bad_main
925 func main(foo:string)
928 ###### output: bad_main
929 .tmp.code:??:??: error: expected argv but variable 'foo' is string
930 .tmp.code:??:??: info: this is where 'NOTVAR' was set to string
931 oceani: main has wrong type.
933 Test for type errors with functions
936 oceani_failing_tests += func_err_args func_err_redeclare
938 ###### test: func_err_args
940 func test1(a:number; b:string; c:[3]Boolean)
943 func test2(a:number; b:string; c:[3]Boolean)
947 # use undefined names
955 test1(1, "two", truth)
957 test1(1, "lo", truth, 4)
958 print test(), test1(1,2,3)
962 ###### output: func_err_args
963 .tmp.code:15:14: error: insufficient arguments to function.
964 .tmp.code:16:14: error: expected number found string
965 .tmp.code:16:22: error: expected string found number
966 .tmp.code:16:14: error: insufficient arguments to function.
967 .tmp.code:18:17: error: expected string found number
968 .tmp.code:19:14: error: too many arguments to function.
969 .tmp.code:20:14: error: attempt to call a non-function.
970 .tmp.code:20:32: error: expected string found number
971 .tmp.code:20:28: error: insufficient arguments to function.
972 .tmp.code:21:20: error: expected func but variable 'test2' is func
973 .tmp.code:??:??: info: this is where 'NOTVAR' was set to func
974 .tmp.code:10:14: error: variable used but not declared: a
975 .tmp.code:10:17: error: variable used but not declared: z
976 oceani: type error in program - not running.
978 ###### test: func_err_redeclare
980 func test1(a:number; b:string; c:[3]Boolean)
992 ###### output: func_err_redeclare
993 .tmp.code:5:5: error: function 'test1' redeclared
994 .tmp.code:2:5: info: this is where 'test1' was first declared
995 .tmp.code:9:5: error: function 'test1' redeclared
996 .tmp.code:2:5: info: this is where 'test1' was first declared
998 ## Test erroneous command line args
1000 To improve coverage, we want to test correct handling of strange command
1001 line arguments. These tests won't use code, so the exiting test types
1002 won't work. So we need to be able to explicitly give the command line,
1003 and the expected output, and have that tested and the coverage assessed.
1004 Rather than having to spell out the whole command name, just give "cmd",
1005 and discard that. Requiring but discarding the command make an empty
1006 command list possible.
1009 @for t in $(oceani_special_tests); do \
1010 echo -n "Test $$t ... ";\
1011 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1012 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1013 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1014 if ! cmp -s .tmp.want .tmp.have; then \
1015 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1017 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1021 oceani_special_tests += "cmd"
1022 oceani_special_tests += "cmd,-zyx"
1023 oceani_special_tests += "cmd,nofile"
1024 oceani_special_tests += "cmd,/dev/null"
1025 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1028 oceani: no input file given
1030 ###### output: cmd,-zyx
1031 ./oceani: invalid option -- 'z'
1032 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1034 ###### output: cmd,nofile
1035 oceani: cannot open nofile
1037 ###### output: cmd,/dev/null
1038 oceani: could not find any code in /dev/null
1040 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1041 oceani: cannot find section toast:nothing