1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
38 oceani_valg_tests := $(oceani_tests)
41 tests:: oceani_test_suite
42 oceani_test_suite: oceani coverage_oceani
43 @echo -n Checking grammar ...
44 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
45 { echo "Grammar contains conflicts, please review" ; exit 1; }
47 @rm -rf coverage; mkdir -p coverage
49 @for T in $(oceani_tests); do \
50 echo -n "Test $$T.. "; \
51 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
52 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
53 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
54 if ! cmp -s .tmp.want .tmp.have; then \
55 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
56 echo -n "printing.. "; \
57 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
58 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
59 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
60 if ! cmp -s .tmp.code1 .tmp.code2; then \
61 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
62 echo -n "extra-newlines.. "; \
64 ' .tmp.code1 > .tmp.code1a; \
65 echo '``````' > .tmp.code2a ;\
66 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
67 if ! cmp -s .tmp.code1 .tmp.code2a; then \
68 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
69 echo -n "exec-after-print.. "; \
70 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
71 if ! cmp -s .tmp.want .tmp.have; then \
72 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
74 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
75 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
80 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
81 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
82 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
85 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_valg_tests); do \
86 echo -n "Valgrind $$T.. "; \
87 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
88 if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
89 > /dev/null 2>&1 ; [ $$? -eq 42 ] ; then \
90 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
91 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
92 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
93 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
94 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
97 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
98 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
99 if (ran < (ran + skip) *0.968) exit(1) }' \
100 coverage/oceani.mdc.gcov
102 coverage_oceani: oceani.c
103 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
105 ## Values and variables
107 The first test stores values in variables and performs various
108 calculations on them.
111 oceani_tests += "valvar"
115 func main(argv:[argc::]string)
116 a := 23; b:=12 ; b1 := -b
117 print a, b, a+b, a-b, a*b, a/b, a%b
118 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
119 print +a, +b, +b1, -a, -b, -b1
120 x := True; y := False
121 print x and y, x or y, x and x, y or y, x and not x, x < y
123 c ::= "This is a string"
124 d ::= " field theory"
127 aconst :: string = "unchanging"
136 ###### output: valvar
138 23 12 35 11 276 1.91667 11
139 False False True True False False True
141 False True True False False False
142 This is a string field theory This is a string field theory
145 Next we change the value of variables
148 oceani_tests += "setvar"
155 a = (a + a) * (a + a)
159 ###### output: setvar
165 oceani_tests += "consts"
170 four ::= 2 + 2 ; five ::= 10/2
171 const pie ::= "I like Pie";
172 cake ::= "The cake is"
176 print "Hello World, what lovely oceans you have!"
177 print "are there", five, "?"
178 print pi, pie, "but", cake
180 ###### output: consts
181 Hello World, what lovely oceans you have!
183 3.14159 I like Pie but The cake is a lie
185 Test merging of variables from multiple cases
188 oceani_tests += varmerge
190 ###### test: varmerge
193 for i:=0; then i=i+1; while i < 5:
196 case 1: scratch:=42; num:="one"
203 for i:=0; then i=i+1; while i < 5:
210 // re-declare a CondScope variable
215 ###### output: varmerge
216 zero , one , two , three , many ,
219 ## Conditions and Loops
221 Now we need to test if/else and some different loops
224 oceani_tests += cond_loop
226 ###### test: cond_loop
234 for b:=1; then b=b+b; while b < 100:
237 // Newtons method for square root of 2
243 current := guess * guess
244 use +(current - target) > 0.000000001
246 guess = (guess + (target / guess) ) / 2
249 print "error is ", target - guess * guess
251 for j:=0; then j = j+3 ; while j < 10:
252 if j != 0 and then 20 / j > 3:
253 print "20 /", j," =", 20 / j
255 print "I won't calculate 20 /", j
256 pi ::= 3.1415926535897
257 if 355/113 == pi or else +(pi - 355/113) < 0.001:
259 print "lower" if 355/113 < pi else "higher"
261 if pi > 3 then print "pi exceeds three"; else print "need more pie"
262 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
263 for { i := 0; sum := 0 }
269 print "sum 1..10 is", sum
278 ###### output: cond_loop
285 error is -4.51095e-12
286 I won't calculate 20 / 0
289 I won't calculate 20 / 9
299 The demonstration code presented in the interpreted is suitable for the test suite.
300 Here I break it into two parts, keeping the array code separate.
303 oceani_tests += "sayhello,55,33,hello,True"
304 oceani_tests += "sayhello,12,60,there,False"
306 ###### test: sayhello
308 func main(av:[ac::number]string)
309 A := $av[1]; B := $av[2]
311 bbool := av[ac-1] == "True"
312 print "Hello World, what lovely oceans you have!"
313 /* When a variable is defined in both branches of an 'if',
314 * and used afterwards, the variables are merged.
320 print "Is", A, "bigger than", B,"? ", bigger
321 /* If a variable is not used after the 'if', no
322 * merge happens, so types can be different
325 double:string = "yes"
326 print A, "is more than twice", B, "?", double
329 print "double", B, "is", double
334 print "still", bigger // check for regression in scoping
341 print "GCD of", A, "and", B,"is", a
343 print a, "is not positive, cannot calculate GCD"
345 print b, "is not positive, cannot calculate GCD"
350 print "Fibonacci:", f1,f2,
360 print astr ++ " was the str"
362 print "I found the str over " ++ astr
364 /* Binary search... */
382 print "Yay, I found", target
384 print "Closest I found was", lo
386 ###### output: sayhello,55,33,hello,True
387 Hello World, what lovely oceans you have!
388 Is 55 bigger than 33 ? yes
391 GCD of 55 and 33 is 11
392 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
394 Closest I found was 77.3438
396 ###### output: sayhello,12,60,there,False
397 Hello World, what lovely oceans you have!
398 Is 12 bigger than 60 ? no
401 GCD of 12 and 60 is 12
402 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
403 I found the str over there
404 Closest I found was 77.3438
407 oceani_tests += "insert_sort"
408 ###### test: insert_sort
413 for i:=1; then i = i + 1; while i < size:
414 n := list[i-1] * list[i-1]
415 list[i] = (n / 100) % 10000
418 for i:=0; then i = i + 1; while i < size:
419 print "list[",i,"]=",list[i]
421 for i := 1; then i=i+1; while i < size:
422 for j:=i-1; then j=j-1; while j >= 0:
423 if list[j] > list[j+1]:
428 for i:=0; then i = i + 1; while i < size:
429 print "list[",i,"]=",list[i]
431 ###### output: insert_sort
547 We already have some array tests, but this is where we put other
548 ad-hoc things array related.
551 oceani_tests += arrays
559 bools[3] = strings[1] == "Hello"
560 bools[1] = strings[2] <= "there"
562 for i:=0; then i=i+1; while i<5:
566 ra[6] = 42 // mustn't crash
567 print '', bools[i], ra[j-1],
570 ###### output: arrays
571 False 0 True 1 False 4 False 9 False 16
575 Time to test if structure declarations and accesses work correctly.
578 oceani_tests += structs
586 active:Boolean = True
588 struct baz { a:number; b:Boolean; }
594 for i:=0; then i=i+1; while i < 4:
602 info[i].size[0] = i*i
604 info[i].active = False
606 for i:=0; then i=i+1; while i < 4:
607 print info[i].name, info[i].active, info[i].size[0]
609 ###### output: structs
618 Test functions. They don't return anything, so we need to get them to print
621 oceani_tests += functions func_ret_type
623 ###### test: functions
643 func test(n:number; s:string)
650 for i:=0; then i = i + 1; while i < 5:
653 ###### output: functions
658 4 3 . 2 .. 1 ... done
660 ###### test: func_ret_type
662 func double(n:number):number
678 for j:=10; then j = j - 3; while j > -5:
679 print answer("dou","ble"), j, "is", double(j)
681 ###### output: func_ret_type
688 ## Test code with syntax errors
690 Syntax errors aren't handled well yet - the result is almost always a
691 single message about the first error. So this section will be fairly
692 thin until we add proper parsing recovery in the face of common errors.
694 A special case of syntax errors is token errors, when a token is only
695 accepted because the parser doesn't know quite enough to reject it.
696 There are handled better as they are quite local, so a single test
697 program can trigger most of the possible errors.
699 To handle erronous code, we need a different set of tests, as we need to
700 capture `stderr`. The same test code will be used for type errors too.
701 As error messages contain the line number, and we don't want changes to
702 this file to change the reported numbers, we copy the code into a
703 separate file first, then run from there.
706 @for t in $(oceani_failing_tests); do \
707 echo -n "Test $$t ... "; \
708 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
709 echo '``````' > .tmp.code; \
710 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
711 ./oceani .tmp.code > .tmp.have 2>&1; \
712 if ! cmp -s .tmp.want .tmp.have; then \
713 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
715 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
718 ###### combine test lists
719 oceani_valg_tests += $(oceani_failing_tests)
722 oceani_failing_tests := syn1
723 oceani_failing_tests += tokerr
728 if then else while do
731 .tmp.code:3:11: Syntax error in statement: then
735 a := 1i // imaginary numbers aren't understood
736 b:[2i]number // array sizes are handled separately
737 c:[3.14159]Boolean // array sizes must be integers
738 d:[1_000_000_000_000]number // they mustn't be huge
739 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
742 This is a multiline string
743 With an unsupportable suffix
747 yy:[unknowable]number
749 zz:[zzsize]string // size must be constant, use ::=
751 // These numbers should be bad in all contexts: FIXME
754 ###### output: tokerr
755 .tmp.code:3:13: error: unsupported number suffix: 1i
756 .tmp.code:4:11: error: unsupported number suffix: 2i
757 .tmp.code:5:11: error: array size must be an integer: 3.14159
758 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
759 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
760 .tmp.code:9:17: error: unsupported string suffix: """
761 This is a multiline string
762 With an unsupportable suffix
764 .tmp.code:14:11: error: undefined type: unknown
765 .tmp.code:15:12: error: name undeclared: unknowable
766 .tmp.code:17:12: error: array size must be a constant: zzsize
767 .tmp.code:20:12: error: unrecognised number: 00123
769 ## Tests for type errors
771 Type error don't cause parsing to abort, so we can fit many in the
772 one test program. Some type errors are found during the parse, others
773 during type analysis which doesn't run if parsing failed. So we cannot
774 fit everything in one.
776 These programs were generated by looking for the
777 various places that `type_err()` are called.
780 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
782 ###### test: type_err1
785 print "hello" ++ 5, 5 ++ "hello"
790 if 3 * 4 and not True: print "Weird"
792 ###### output: type_err1
793 .tmp.code:3:25: error: expected string found number
794 .tmp.code:3:28: error: expected string found number
795 .tmp.code:6:8: error: Cannot assign to a constant: b
796 .tmp.code:5:8: info: name was defined as a constant here
797 .tmp.code:6:8: error: Cannot assign to a constant: b
798 .tmp.code:5:8: info: name was defined as a constant here
799 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
800 oceani: type error in program - not running.
802 ###### test: type_err2
812 ###### output: type_err2
813 .tmp.code:4:8: error: variable 'a' redeclared
814 .tmp.code:3:8: info: this is where 'a' was first declared
815 .tmp.code:5:8: error: variable 'a' redeclared
816 .tmp.code:3:8: info: this is where 'a' was first declared
817 .tmp.code:6:8: error: variable 'a' redeclared
818 .tmp.code:3:8: info: this is where 'a' was first declared
819 .tmp.code:7:8: error: variable 'a' redeclared
820 .tmp.code:3:8: info: this is where 'a' was first declared
821 .tmp.code:8:8: Variable declared with no type or value: c
823 ###### test: type_err3
832 c = "hello" ++ (True and False)
834 print 45 + ( "Hello" ++ "there")
844 case "Hello": print "Hello"
846 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
860 // trigger 'labels not permitted' error message
861 while 1 if True else False:
865 ###### output: type_err3
866 .tmp.code:8:12: error: expected number but variable 'c' is string
867 .tmp.code:7:8: info: this is where 'c' was set to string
868 .tmp.code:8:12: error: Arithmetic returns number but string expected
869 .tmp.code:7:8: info: variable 'c' was set as string here.
870 .tmp.code:9:24: error: Boolean operation found where string expected
871 .tmp.code:10:12: error: Comparison returns Boolean but string expected
872 .tmp.code:7:8: info: variable 'c' was set as string here.
873 .tmp.code:11:21: error: Concat returns string but number expected
874 .tmp.code:12:8: error: string cannot be indexed
875 .tmp.code:12:8: error: string cannot be indexed
876 .tmp.code:21:13: error: expected number found string
877 .tmp.code:17:16: error: expected number, found string
878 .tmp.code:24:8: error: cannot assign value of type [5]number
879 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
880 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
881 .tmp.code:25:8: error: cannot assign value of type [5]number
882 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
883 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
884 .tmp.code:26:8: error: cannot assign value of type [5]number
885 .tmp.code:27:16: error: expected number found string
886 .tmp.code:28:16: error: expected string found Boolean
887 .tmp.code:29:12: error: have number but need string
888 .tmp.code:7:8: info: variable 'c' was set as string here.
889 .tmp.code:32:8: error: variable used but not declared: foo
890 .tmp.code:32:8: error: field reference attempted on none, not a struct
891 .tmp.code:32:16: error: expected none found number
892 .tmp.code:33:14: error: field reference attempted on string, not a struct
893 .tmp.code:34:14: error: cannot find requested field in foo
894 .tmp.code:35:17: error: have string but need number
895 .tmp.code:38:29: error: expected number (labels not permitted) found Boolean
896 oceani: type error in program - not running.
898 ###### test: type_err4
903 ###### output: type_err4
904 .tmp.code:3:14: error: variable used but not declared: b
905 .tmp.code:3:16: error: expected none found number
906 .tmp.code:3:14: info: variable 'b' was set as none here.
907 oceani: type error in program - not running.
910 oceani_failing_tests += type_err_const type_err_const1 missing_program bad_main
912 ###### test: type_err_const
915 bar ::= "string" + 56
922 // trigger duplicate-main error
927 ###### output: type_err_const
928 .tmp.code:4:16: error: expected number found string
929 .tmp.code:6:8: error: name already declared: bar
930 .tmp.code:4:8: info: this is where 'bar' was first declared
931 .tmp.code:8:8: error: variable 'foo' redeclared
932 .tmp.code:3:8: info: this is where 'foo' was first declared
933 .tmp.code:12:5: error: function 'main' redeclared
934 .tmp.code:7:5: info: this is where 'main' was first declared
935 .tmp.code:13:8: error: variable 'foo' redeclared
936 .tmp.code:3:8: info: this is where 'foo' was first declared
938 ###### test: type_err_const1
946 ###### output: type_err_const1
947 .tmp.code:3:12: Syntax error in constant: :
948 .tmp.code:4:12: Syntax error in constant: :
950 ###### test: missing_program
954 ###### output: missing_program
955 oceani: no main function found.
957 ###### test: bad_main
958 func main(foo:string)
961 ###### output: bad_main
962 .tmp.code:??:??: error: expected argv but variable 'foo' is string
963 .tmp.code:??:??: info: this is where 'NOTVAR' was set to string
964 oceani: main has wrong type.
966 Test for type errors with functions
969 oceani_failing_tests += func_err_args func_err_redeclare
971 ###### test: func_err_args
973 func test1(a:number; b:string; c:[3]Boolean)
976 func test2(a:number; b:string; c:[3]Boolean)
980 # use undefined names
988 test1(1, "two", truth)
990 test1(1, "lo", truth, 4)
991 print test(), test1(1,2,3)
995 func test4(a:number):string
998 func test5(a:number):string
1001 ###### output: func_err_args
1002 .tmp.code:28:14: error: expected string, found none
1003 .tmp.code:25:8: error: expected string, found number
1004 .tmp.code:15:14: error: insufficient arguments to function.
1005 .tmp.code:16:14: error: expected number found string
1006 .tmp.code:16:22: error: expected string found number
1007 .tmp.code:16:14: error: insufficient arguments to function.
1008 .tmp.code:18:17: error: expected string found number
1009 .tmp.code:19:14: error: too many arguments to function.
1010 .tmp.code:20:14: error: attempt to call a non-function.
1011 .tmp.code:20:32: error: expected string found number
1012 .tmp.code:20:28: error: insufficient arguments to function.
1013 .tmp.code:21:20: error: expected func but variable 'test2' is func
1014 .tmp.code:??:??: info: this is where 'NOTVAR' was set to func
1015 .tmp.code:10:14: error: variable used but not declared: a
1016 .tmp.code:10:17: error: variable used but not declared: z
1017 oceani: type error in program - not running.
1019 ###### test: func_err_redeclare
1021 func test1(a:number; b:string; c:[3]Boolean)
1033 ###### output: func_err_redeclare
1034 .tmp.code:5:5: error: function 'test1' redeclared
1035 .tmp.code:2:5: info: this is where 'test1' was first declared
1036 .tmp.code:9:5: error: function 'test1' redeclared
1037 .tmp.code:2:5: info: this is where 'test1' was first declared
1039 ## Test erroneous command line args
1041 To improve coverage, we want to test correct handling of strange command
1042 line arguments. These tests won't use code, so the exiting test types
1043 won't work. So we need to be able to explicitly give the command line,
1044 and the expected output, and have that tested and the coverage assessed.
1045 Rather than having to spell out the whole command name, just give "cmd",
1046 and discard that. Requiring but discarding the command make an empty
1047 command list possible.
1050 @for t in $(oceani_special_tests); do \
1051 echo -n "Test $$t ... ";\
1052 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1053 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1054 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1055 if ! cmp -s .tmp.want .tmp.have; then \
1056 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1058 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1062 oceani_special_tests += "cmd"
1063 oceani_special_tests += "cmd,-zyx"
1064 oceani_special_tests += "cmd,nofile"
1065 oceani_special_tests += "cmd,/dev/null"
1066 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1069 oceani: no input file given
1071 ###### output: cmd,-zyx
1072 ./oceani: invalid option -- 'z'
1073 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1075 ###### output: cmd,nofile
1076 oceani: cannot open nofile
1078 ###### output: cmd,/dev/null
1079 oceani: could not find any code in /dev/null
1081 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1082 oceani: cannot find section toast:nothing