1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
38 oceani_valg_tests := $(oceani_tests)
41 tests:: oceani_test_suite
42 oceani_test_suite: oceani coverage_oceani
43 @echo -n Checking grammar ...
44 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
45 { echo "Grammar contains conflicts, please review" ; exit 1; }
47 @rm -rf coverage; mkdir -p coverage
49 @for T in $(oceani_tests); do \
50 echo -n "Test $$T.. "; \
51 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
52 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
53 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
54 if ! cmp -s .tmp.want .tmp.have; then \
55 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
56 echo -n "printing.. "; \
57 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
58 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
59 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
60 if ! cmp -s .tmp.code1 .tmp.code2; then \
61 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
62 echo -n "extra-newlines.. "; \
64 ' .tmp.code1 > .tmp.code1a; \
65 echo '``````' > .tmp.code2a ;\
66 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
67 if ! cmp -s .tmp.code1 .tmp.code2a; then \
68 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
69 echo -n "exec-after-print.. "; \
70 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
71 if ! cmp -s .tmp.want .tmp.have; then \
72 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
74 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
75 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
80 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
81 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
82 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
85 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_valg_tests); do \
86 echo -n "Valgrind $$T.. "; \
87 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
88 if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
89 > /dev/null 2>&1 ; [ $$? -eq 42 ] ; then \
90 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
91 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
92 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
93 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
94 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
98 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
99 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
100 if (ran < (ran + skip) *0.99) exit(1) }' \
101 coverage/oceani.mdc.gcov
103 coverage_oceani: oceani.c
104 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
106 ## Values and variables
108 The first test stores values in variables and performs various
109 calculations on them.
112 oceani_tests += "valvar"
116 func main(argv:[]string)
117 a := 23; b:=12 ; b1 := -b
118 print a, b, a+b, a-b, a*b, a/b, a%b
119 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
120 print +a, +b, +b1, -a, -b, -b1
121 x := True; y := False
122 print x and y, x or y, x and x, y or y, x and not x, x < y
124 c ::= "This is a string"
125 d ::= " field theory"
128 aconst :: string = "unchanging"
142 print "z??False", z??False, "w??False", w??False
145 print $"-34.56", $"not-a-number"
147 if True { pass; pass }
149 ###### output: valvar
151 23 12 35 11 276 1.916666667 11
152 False False True True False False True
154 False True True False False False
155 This is a string field theory This is a string field theory
159 z??False True w??False False
161 Unsupported suffix: 4i
164 Next we change the value of variables
167 oceani_tests += "setvar"
174 a = (a + a) * (a + a)
178 ###### output: setvar
184 oceani_tests += "consts"
190 four ::= 2 + 2 ; five ::= 10/2
191 const pie ::= "I like Pie";
192 cake ::= "The cake is"
196 print "Hello World, what lovely oceans you have!"
197 print "are there", five, "?"
198 print pi, pie, "but", cake, "Tau is", tau
200 ###### output: consts
201 Hello World, what lovely oceans you have!
203 3.141592653 I like Pie but The cake is a lie Tau is 6.283185306
205 Test merging of variables from multiple cases
208 oceani_tests += varmerge
210 ###### test: varmerge
213 for i:=0; then i=i+1; while i < 5:
216 case 1: scratch:=42; num:="one"
223 for i:=0; then i=i+1; while i < 5:
225 case 0: num:="zero" ;
227 case 2 { num:="two" } case 3 { num:="three" }
229 // re-declare a CondScope variable
234 ###### output: varmerge
235 zero , one , two , three , many ,
238 ## Conditions and Loops
240 Now we need to test if/else and some different loops
243 oceani_tests += cond_loop
245 ###### test: cond_loop
253 for b:=1; then b=b+b; while b < 100:
256 // Newtons method for square root of 2
262 current := guess * guess
263 use +(current - target) > 0.000000001
265 guess = (guess + (target / guess) ) / 2
268 print "error is ", target - guess * guess
270 for j:=0; then j = j+3 ; while j < 10:
271 if j != 0 and then 20 / j > 3:
272 print "20 /", j," =", 20 / j
274 print "I won't calculate 20 /", j
275 pi ::= 3.1415926535897
276 if 355/113 == pi or else +(pi - 355/113) < 0.001:
278 print "lower" if 355/113 < pi else "higher"
279 print "higher" if 355/113 > pi else "lower"
281 if pi > 3 then print "pi exceeds three"; else print "need more pie"
282 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
283 for { i := 0; sum := 0 }
289 print "sum 1..10 is", sum
298 // different parsing options
311 ###### output: cond_loop
318 error is -4.510950445e-12
319 I won't calculate 20 / 0
322 I won't calculate 20 / 9
333 The demonstration code presented in the interpreted is suitable for the test suite.
334 Here I break it into two parts, keeping the array code separate.
337 oceani_tests += "sayhello,55,33,hello,True"
338 oceani_tests += "sayhello,12,60,there,False"
340 ###### test: sayhello
342 func main(av:[]string)
343 A := $av[1]; B := $av[2]
346 bbool := av[l-1] == "True"
347 print "Hello World, what lovely oceans you have!"
348 /* When a variable is defined in both branches of an 'if',
349 * and used afterwards, the variables are merged.
355 print "Is", A, "bigger than", B,"? ", bigger
356 /* If a variable is not used after the 'if', no
357 * merge happens, so types can be different
360 double:string = "yes"
361 print A, "is more than twice", B, "?", double
364 print "double", B, "is", double
369 print "still", bigger // check for regression in scoping
376 print "GCD of", A, "and", B,"is", a
378 print a, "is not positive, cannot calculate GCD"
380 print b, "is not positive, cannot calculate GCD"
385 print "Fibonacci:", f1,f2,
395 print astr ++ " was the str"
397 print "I found the str over " ++ astr
399 /* Binary search... */
417 print "Yay, I found", target
419 print "Closest I found was", lo
421 ###### output: sayhello,55,33,hello,True
422 Hello World, what lovely oceans you have!
423 Is 55 bigger than 33 ? yes
426 GCD of 55 and 33 is 11
427 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
429 Closest I found was 77.34375
431 ###### output: sayhello,12,60,there,False
432 Hello World, what lovely oceans you have!
433 Is 12 bigger than 60 ? no
436 GCD of 12 and 60 is 12
437 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
438 I found the str over there
439 Closest I found was 77.34375
442 oceani_tests += "insert_sort"
443 ###### test: insert_sort
448 for i:=1; then i = i + 1; while i < size:
449 n := list[i-1] * list[i-1]
450 list[i] = (n / 100) % 10000
453 for i:=0; then i = i + 1; while i < size:
454 print "list[",i,"]=",list[i]
456 for i := 1; then i=i+1; while i < size:
457 for j:=i-1; then j=j-1; while j >= 0:
458 if list[j] > list[j+1]:
463 for i:=0; then i = i + 1; while i < size:
464 print "list[",i,"]=",list[i]
466 ###### output: insert_sort
582 We already have some array tests, but this is where we put other
583 ad-hoc things array related.
586 oceani_tests += arrays
594 bools[3] = strings[1] == "Hello"
595 bools[1] = strings[2] <= "there"
597 for i:=0; then i=i+1; while i<5:
601 ra[6] = 42 // mustn't crash
602 print '', bools[i], ra[j-1],
605 ###### output: arrays
606 False 0 True 1 False 4 False 9 False 16
610 Time to test if structure declarations and accesses work correctly.
613 oceani_tests += structs
620 name:string = "Hello"
622 active:Boolean = True
624 struct baz { a:number; b:Boolean; }
629 struct bat2 a:string; b:Boolean
635 for i:=0; then i=i+1; while i < 4:
643 info[i].size[0] = i*i
645 info[i].active = False
647 for i:=0; then i=i+1; while i < 4:
648 print info[i].name, info[i].active, info[i].size[0]
649 info[0].thing.b = True
653 ###### output: structs
662 Test functions. They don't return anything, so we need to get them to print
665 oceani_tests += functions func_ret_type
667 ###### test: functions
687 func test(n:number; s:string)
704 for i:=0; then i = i + 1; while i < 5:
706 angular := to_polar(32, 23)
707 print angular.rho, angular.theta
709 func test2(n:number; s:string;) : (ret:number)
718 // exercise the parsing options
722 func t2 (a:string) {print "string"}
725 ###### output: functions
730 4 3 . 2 .. 1 ... done
733 ###### test: func_ret_type
735 func double(n:number):number
751 for j:=10; then j = j - 3; while j > -5:
752 print answer("dou","ble"), j, "is", double(j)
754 ###### output: func_ret_type
763 A simple linked list example
766 oceani_tests += "linked_list,one,two,three,four"
768 ###### test: linked_list
776 func insert(list:@linkage; new:string)
778 while ?p.next and then p.next.this < new:
785 func printlist(list:@linkage)
787 print list@.next.this
788 list = list@.next.list
790 func freelist(list:@linkage)
791 if list.next != @nil:
792 lp:@linkage = list.next.list
796 func main(argv:[]string)
799 insert(list, "@start");
801 for i:=1; then i=i+1; while i < argv[]:
802 insert(list, argv[i])
803 insert(list, "Hello!")
807 ###### output: linked_list,one,two,three,four
816 ## Test code with syntax errors
818 Syntax errors aren't handled well yet - the result is almost always a
819 single message about the first error. So this section will be fairly
820 thin until we add proper parsing recovery in the face of common errors.
822 A special case of syntax errors is token errors, when a token is only
823 accepted because the parser doesn't know quite enough to reject it.
824 There are handled better as they are quite local, so a single test
825 program can trigger most of the possible errors.
827 To handle erronous code, we need a different set of tests, as we need to
828 capture `stderr`. The same test code will be used for type errors too.
829 As error messages contain the line number, and we don't want changes to
830 this file to change the reported numbers, we copy the code into a
831 separate file first, then run from there.
834 @for t in $(oceani_failing_tests); do \
835 echo -n "Test $$t ... "; \
836 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
837 echo '``````' > .tmp.code; \
838 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
839 ./oceani .tmp.code > .tmp.have 2>&1; \
840 if ! cmp -s .tmp.want .tmp.have; then \
841 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
843 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
846 ###### combine test lists
847 oceani_valg_tests += $(oceani_failing_tests)
850 oceani_failing_tests := syn1
851 oceani_failing_tests += tokerr
856 if then else while do
859 .tmp.code:3:11: Syntax error in statement: then
863 a := 1i // imaginary numbers aren't understood
864 b:[2i]number // array sizes are handled separately
865 c:[3.14159]Boolean // array sizes must be integers
866 d:[1_000_000_000_000]number // they mustn't be huge
867 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
870 This is a multiline string
871 With an unsupportable suffix
875 yy:[unknowable]number
877 zz:[zzsize]string // size must be constant, use ::=
879 // These numbers should be bad in all contexts: FIXME
882 ###### output: tokerr
883 .tmp.code:3:13: error: unsupported number suffix: 1i
884 .tmp.code:4:11: error: unsupported number suffix: 2i
885 .tmp.code:5:11: error: array size must be an integer: 3.14159
886 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
887 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
888 .tmp.code:9:17: error: unsupported string suffix: """
889 This is a multiline string
890 With an unsupportable suffix
892 .tmp.code:15:12: error: name undeclared: unknowable
893 .tmp.code:17:12: error: array size must be a constant: zzsize
894 .tmp.code:20:12: error: unrecognised number: 00123
895 .tmp.code:14:11: error: type used but not declared: unknown
897 ## Tests for type errors
899 Type error don't cause parsing to abort, so we can fit many in the
900 one test program. Some type errors are found during the parse, others
901 during type analysis which doesn't run if parsing failed. So we cannot
902 fit everything in one.
904 These programs were generated by looking for the
905 various places that `type_err()` are called.
908 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4 type_err5 type_err6
910 ###### test: type_err1
913 print "hello" ++ 5, 5 ++ "hello"
918 if 3 * 4 and not True: print "Weird"
922 ###### output: type_err1
923 .tmp.code:3:25: error: expected string found number
924 .tmp.code:3:28: error: expected string found number
925 .tmp.code:6:8: error: Cannot assign to a constant: b
926 .tmp.code:5:8: info: name was defined as a constant here
927 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
928 .tmp.code:9:20: error: expected number found label
929 .tmp.code:9:8: info: variable 'd' was set as number here.
930 .tmp.code:10:8: error: cannot assign to an rval
931 oceani: type error in program - not running.
933 ###### test: type_err2
944 ###### output: type_err2
945 .tmp.code:4:8: error: variable 'a' redeclared
946 .tmp.code:3:8: info: this is where 'a' was first declared
947 .tmp.code:5:8: error: variable 'a' redeclared
948 .tmp.code:3:8: info: this is where 'a' was first declared
949 .tmp.code:6:8: error: variable 'a' redeclared
950 .tmp.code:3:8: info: this is where 'a' was first declared
951 .tmp.code:7:8: error: variable 'a' redeclared
952 .tmp.code:3:8: info: this is where 'a' was first declared
953 .tmp.code:8:8: Variable declared with no type or value: c
954 .tmp.code:9:19: error: unsupported number format: 02
956 ###### test: type_err3
960 b:string = "hello"; d:Boolean
965 c = "hello" ++ (True and False)
967 print 45 + ( "Hello" ++ "there")
977 case "Hello": print "Hello"
979 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
993 // trigger 'labels not permitted' error message
994 while 1 if True else False:
997 print "one" ++ a4[], c[]
1007 ###### output: type_err3
1008 .tmp.code:8:12: error: expected number but variable 'c' is string
1009 .tmp.code:7:8: info: this is where 'c' was set to string
1010 .tmp.code:8:12: error: Arithmetic returns number but string expected
1011 .tmp.code:7:8: info: variable 'c' was set as string here.
1012 .tmp.code:9:24: error: Boolean operation found where string expected
1013 .tmp.code:7:8: info: variable 'c' was set as string here.
1014 .tmp.code:10:12: error: Comparison returns Boolean but string expected
1015 .tmp.code:7:8: info: variable 'c' was set as string here.
1016 .tmp.code:11:21: error: Concat returns string but number expected
1017 .tmp.code:12:8: error: string cannot be indexed
1018 .tmp.code:12:8: error: string cannot be indexed
1019 .tmp.code:21:13: error: expected number found string
1020 .tmp.code:17:16: error: expected number, found string
1021 .tmp.code:24:8: error: cannot assign value of type [5]number
1022 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
1023 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
1024 .tmp.code:25:8: error: cannot assign value of type [5]number
1025 .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
1026 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
1027 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
1028 .tmp.code:26:8: error: cannot assign value of type [5]number
1029 .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
1030 .tmp.code:27:16: error: expected number found string
1031 .tmp.code:28:16: error: expected string found Boolean
1032 .tmp.code:29:12: error: have number but need string
1033 .tmp.code:7:8: info: variable 'c' was set as string here.
1034 .tmp.code:32:8: error: variable used but not declared: foo
1035 .tmp.code:32:8: error: field reference on none is not supported
1036 .tmp.code:32:16: error: expected none found number
1037 .tmp.code:33:14: error: field reference on string is not supported
1038 .tmp.code:34:14: error: cannot find requested field in foo
1039 .tmp.code:35:17: error: have string but need number
1040 .tmp.code:38:29: error: expected number found Boolean
1041 .tmp.code:41:23: error: have number but need string
1042 .tmp.code:41:29: error: string cannot provide length
1043 .tmp.code:43:21: error: Can only convert string to number, not Boolean
1044 .tmp.code:43:8: info: variable 'x' was set as Boolean here.
1045 .tmp.code:49:13: error: expected [five]number but variable 'x2' is [four]number
1046 .tmp.code:48:8: info: this is where 'x2' was set to [four]number
1047 .tmp.code:49:8: error: cannot assign value of type [five]number
1048 .tmp.code:47:8: info: variable 'x1' was set as [five]number here.
1049 oceani: type error in program - not running.
1051 ###### test: type_err4
1056 ###### output: type_err4
1057 .tmp.code:3:14: error: variable used but not declared: b
1058 .tmp.code:3:16: error: expected none found number
1059 .tmp.code:3:14: info: variable 'b' was set as none here.
1060 oceani: type error in program - not running.
1062 ###### test: type_err5
1073 ###### output: type_err5
1074 .tmp.code:8:7: error: type already declared: foo
1075 .tmp.code:2:7: info: this is location of declartion: foo
1076 .tmp.code:10:13: Syntax error in struct field: :
1077 .tmp.code:5:7: error: type has recursive definition: baz
1078 .tmp.code:2:7: error: type has recursive definition: foo
1080 ###### test: type_err6
1088 ###### output: type_err6
1089 .tmp.code:4:12: error: '?' requires a testable value, not string
1090 .tmp.code:6:14: error: "??" requires a testable value, not string
1091 oceani: type error in program - not running.
1095 oceani_failing_tests += type_err_const type_err_const1 type_err_const2 missing_program bad_main
1097 ###### test: type_err_const
1100 bar ::= "string" + 56
1107 // trigger duplicate-main error
1112 ###### output: type_err_const
1113 .tmp.code:6:8: error: name already declared: bar
1114 .tmp.code:4:8: info: this is where 'bar' was first declared
1115 .tmp.code:8:8: error: variable 'foo' redeclared
1116 .tmp.code:3:8: info: this is where 'foo' was first declared
1117 .tmp.code:12:5: error: function 'main' redeclared
1118 .tmp.code:7:5: info: this is where 'main' was first declared
1119 .tmp.code:13:8: error: variable 'foo' redeclared
1120 .tmp.code:3:8: info: this is where 'foo' was first declared
1121 .tmp.code:4:16: error: expected number found string
1123 ###### test: type_err_const1
1131 ###### output: type_err_const1
1132 .tmp.code:3:12: Syntax error in constant: :
1133 .tmp.code:4:12: Syntax error in constant: :
1135 ###### test: type_err_const2
1140 ###### output: type_err_const2
1141 .tmp.code:3:8: error: const four cannot be resolved.
1142 .tmp.code:4:8: error: const two cannot be resolved.
1144 ###### test: missing_program
1148 ###### output: missing_program
1149 oceani: no main function found.
1151 ###### test: bad_main
1152 func main(foo:string)
1155 ###### output: bad_main
1156 .tmp.code:2:10: error: expected []string but variable 'foo' is string
1157 .tmp.code:2:10: info: this is where 'foo' was set to string
1158 oceani: main has wrong type.
1160 Test for type errors with functions
1163 oceani_failing_tests += func_err_args func_err_redeclare
1165 ###### test: func_err_args
1167 func test1(a:number; b:string; c:[3]Boolean)
1170 func test2(a:number; b:string; c:[3]Boolean)
1174 # use undefined names
1182 test1(1, "two", truth)
1184 test1(1, "lo", truth, 4)
1185 print test(), test1(1,2,3)
1189 func test4(a:number):string
1192 func test5(a:number):string
1199 func test6(a:number):foo
1204 func test7(a:@number)
1208 ###### output: func_err_args
1209 .tmp.code:40:14: error: cannot pass rval when reference expected
1210 .tmp.code:41:14: error: expected @number found string
1211 .tmp.code:34:5: error: function cannot return value of type foo
1212 .tmp.code:28:8: error: expected string, found none
1213 .tmp.code:25:8: error: expected string, found number
1214 .tmp.code:15:14: error: insufficient arguments to function.
1215 .tmp.code:16:14: error: expected number found string
1216 .tmp.code:16:22: error: expected string found number
1217 .tmp.code:16:14: error: insufficient arguments to function.
1218 .tmp.code:18:17: error: expected string found number
1219 .tmp.code:19:14: error: too many arguments to function.
1220 .tmp.code:20:14: error: attempt to call a non-function.
1221 .tmp.code:20:32: error: expected string found number
1222 .tmp.code:20:28: error: insufficient arguments to function.
1223 .tmp.code:21:20: error: expected "func test1" but variable 'test2' is "func test2"
1224 .tmp.code:5:5: info: this is where 'test2' was set to "func test2"
1225 .tmp.code:10:14: error: variable used but not declared: a
1226 .tmp.code:10:17: error: variable used but not declared: z
1227 oceani: type error in program - not running.
1229 ###### test: func_err_redeclare
1231 func test1(a:number; b:string; c:[3]Boolean)
1243 ###### output: func_err_redeclare
1244 .tmp.code:5:5: error: function 'test1' redeclared
1245 .tmp.code:2:5: info: this is where 'test1' was first declared
1246 .tmp.code:9:5: error: function 'test1' redeclared
1247 .tmp.code:2:5: info: this is where 'test1' was first declared
1249 Test for errors with references
1252 oceani_failing_tests += ref_err1 ref_err2
1254 ###### test: ref_err1
1262 ###### output: ref_err1
1263 .tmp.code:4:9: error: only "@free" makes sense here: foo
1264 .tmp.code:5:15: error: Only reference function is "@new()": old
1265 .tmp.code:6:19: error: Only reference value is "@nil": null
1267 ###### test: ref_err2
1273 if num == @nil or ref == ref2 or ref == 2 or ref.foo:
1277 ###### output: ref_err2
1278 .tmp.code:5:22: error: @new() can only be used with references, not number
1279 .tmp.code:5:8: info: variable 'num' was set as number here.
1280 .tmp.code:6:14: error: Cannot dereference number
1281 .tmp.code:7:19: error: @nil can only be used with reference, not number
1282 .tmp.code:7:33: error: expected @number but variable 'ref2' is @string
1283 .tmp.code:4:8: info: this is where 'ref2' was set to @string
1284 .tmp.code:7:48: error: expected @number found number
1285 .tmp.code:7:53: error: field reference on number is not supported
1286 .tmp.code:7:56: error: have none but need Boolean
1287 .tmp.code:8:17: error: @free can only be assigned a reference, not number
1288 .tmp.code:8:17: error: @free can only be assigned a reference, not number
1289 .tmp.code:9:8: error: Cannot assign an rval to a reference.
1290 oceani: type error in program - not running.
1292 ## Test erroneous command line args
1294 To improve coverage, we want to test correct handling of strange command
1295 line arguments. These tests won't use code, so the exiting test types
1296 won't work. So we need to be able to explicitly give the command line,
1297 and the expected output, and have that tested and the coverage assessed.
1298 Rather than having to spell out the whole command name, just give "cmd",
1299 and discard that. Requiring but discarding the command make an empty
1300 command list possible.
1303 @for t in $(oceani_special_tests); do \
1304 echo -n "Test $$t ... ";\
1305 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1306 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1307 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1308 if ! cmp -s .tmp.want .tmp.have; then \
1309 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1311 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1313 ###### valgrind test code
1314 @[ -n "$$SKIP_VALGRIND" ] || for t in $(oceani_special_tests); do\
1315 echo -n "Valgrind $$t.. "; \
1316 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1317 if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1318 [ $$? -eq 42 ]; then \
1319 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
1320 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
1321 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
1322 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
1323 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
1328 oceani_special_tests += "cmd"
1329 oceani_special_tests += "cmd,-zyx"
1330 oceani_special_tests += "cmd,nofile"
1331 oceani_special_tests += "cmd,/dev/null"
1332 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1335 oceani: no input file given
1337 ###### output: cmd,-zyx
1338 ./oceani: invalid option -- 'z'
1339 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1341 ###### output: cmd,nofile
1342 oceani: cannot open nofile
1344 ###### output: cmd,/dev/null
1345 oceani: could not find any code in /dev/null
1347 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1348 oceani: cannot find section toast:nothing