1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
38 oceani_valg_tests := $(oceani_tests)
41 tests:: oceani_test_suite
42 oceani_test_suite: oceani coverage_oceani
43 @echo -n Checking grammar ...
44 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
45 { echo "Grammar contains conflicts, please review" ; exit 1; }
47 @rm -rf coverage; mkdir -p coverage
49 @for T in $(oceani_tests); do \
50 echo -n "Test $$T.. "; \
51 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
52 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
53 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
54 if ! cmp -s .tmp.want .tmp.have; then \
55 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
56 echo -n "printing.. "; \
57 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
58 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
59 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
60 if ! cmp -s .tmp.code1 .tmp.code2; then \
61 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
62 echo -n "extra-newlines.. "; \
64 ' .tmp.code1 > .tmp.code1a; \
65 echo '``````' > .tmp.code2a ;\
66 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
67 if ! cmp -s .tmp.code1 .tmp.code2a; then \
68 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
69 echo -n "exec-after-print.. "; \
70 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
71 if ! cmp -s .tmp.want .tmp.have; then \
72 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
74 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
75 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
80 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
81 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
82 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
85 @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_valg_tests); do \
86 echo -n "Valgrind $$T.. "; \
87 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
88 if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
89 > /dev/null 2>&1 ; [ $$? -eq 42 ] ; then \
90 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
91 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
92 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
93 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
94 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
98 @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
99 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
100 if (ran < (ran + skip) *0.972) exit(1) }' \
101 coverage/oceani.mdc.gcov
103 coverage_oceani: oceani.c
104 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
106 ## Values and variables
108 The first test stores values in variables and performs various
109 calculations on them.
112 oceani_tests += "valvar"
116 func main(argv:[argc::]string)
117 a := 23; b:=12 ; b1 := -b
118 print a, b, a+b, a-b, a*b, a/b, a%b
119 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
120 print +a, +b, +b1, -a, -b, -b1
121 x := True; y := False
122 print x and y, x or y, x and x, y or y, x and not x, x < y
124 c ::= "This is a string"
125 d ::= " field theory"
128 aconst :: string = "unchanging"
142 print "z??False", z??False, "w??False", w??False
146 ###### output: valvar
148 23 12 35 11 276 1.916666667 11
149 False False True True False False True
151 False True True False False False
152 This is a string field theory This is a string field theory
156 z??False True w??False False
158 Next we change the value of variables
161 oceani_tests += "setvar"
168 a = (a + a) * (a + a)
172 ###### output: setvar
178 oceani_tests += "consts"
184 four ::= 2 + 2 ; five ::= 10/2
185 const pie ::= "I like Pie";
186 cake ::= "The cake is"
190 print "Hello World, what lovely oceans you have!"
191 print "are there", five, "?"
192 print pi, pie, "but", cake, "Tau is", tau
194 ###### output: consts
195 Hello World, what lovely oceans you have!
197 3.141592653 I like Pie but The cake is a lie Tau is 6.283185306
199 Test merging of variables from multiple cases
202 oceani_tests += varmerge
204 ###### test: varmerge
207 for i:=0; then i=i+1; while i < 5:
210 case 1: scratch:=42; num:="one"
217 for i:=0; then i=i+1; while i < 5:
224 // re-declare a CondScope variable
229 ###### output: varmerge
230 zero , one , two , three , many ,
233 ## Conditions and Loops
235 Now we need to test if/else and some different loops
238 oceani_tests += cond_loop
240 ###### test: cond_loop
248 for b:=1; then b=b+b; while b < 100:
251 // Newtons method for square root of 2
257 current := guess * guess
258 use +(current - target) > 0.000000001
260 guess = (guess + (target / guess) ) / 2
263 print "error is ", target - guess * guess
265 for j:=0; then j = j+3 ; while j < 10:
266 if j != 0 and then 20 / j > 3:
267 print "20 /", j," =", 20 / j
269 print "I won't calculate 20 /", j
270 pi ::= 3.1415926535897
271 if 355/113 == pi or else +(pi - 355/113) < 0.001:
273 print "lower" if 355/113 < pi else "higher"
275 if pi > 3 then print "pi exceeds three"; else print "need more pie"
276 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
277 for { i := 0; sum := 0 }
283 print "sum 1..10 is", sum
292 ###### output: cond_loop
299 error is -4.510950445e-12
300 I won't calculate 20 / 0
303 I won't calculate 20 / 9
313 The demonstration code presented in the interpreted is suitable for the test suite.
314 Here I break it into two parts, keeping the array code separate.
317 oceani_tests += "sayhello,55,33,hello,True"
318 oceani_tests += "sayhello,12,60,there,False"
320 ###### test: sayhello
322 func main(av:[ac::number]string)
323 A := $av[1]; B := $av[2]
326 bbool := av[l-1] == "True"
327 print "Hello World, what lovely oceans you have!"
328 /* When a variable is defined in both branches of an 'if',
329 * and used afterwards, the variables are merged.
335 print "Is", A, "bigger than", B,"? ", bigger
336 /* If a variable is not used after the 'if', no
337 * merge happens, so types can be different
340 double:string = "yes"
341 print A, "is more than twice", B, "?", double
344 print "double", B, "is", double
349 print "still", bigger // check for regression in scoping
356 print "GCD of", A, "and", B,"is", a
358 print a, "is not positive, cannot calculate GCD"
360 print b, "is not positive, cannot calculate GCD"
365 print "Fibonacci:", f1,f2,
375 print astr ++ " was the str"
377 print "I found the str over " ++ astr
379 /* Binary search... */
397 print "Yay, I found", target
399 print "Closest I found was", lo
401 ###### output: sayhello,55,33,hello,True
402 Hello World, what lovely oceans you have!
403 Is 55 bigger than 33 ? yes
406 GCD of 55 and 33 is 11
407 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
409 Closest I found was 77.34375
411 ###### output: sayhello,12,60,there,False
412 Hello World, what lovely oceans you have!
413 Is 12 bigger than 60 ? no
416 GCD of 12 and 60 is 12
417 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
418 I found the str over there
419 Closest I found was 77.34375
422 oceani_tests += "insert_sort"
423 ###### test: insert_sort
428 for i:=1; then i = i + 1; while i < size:
429 n := list[i-1] * list[i-1]
430 list[i] = (n / 100) % 10000
433 for i:=0; then i = i + 1; while i < size:
434 print "list[",i,"]=",list[i]
436 for i := 1; then i=i+1; while i < size:
437 for j:=i-1; then j=j-1; while j >= 0:
438 if list[j] > list[j+1]:
443 for i:=0; then i = i + 1; while i < size:
444 print "list[",i,"]=",list[i]
446 ###### output: insert_sort
562 We already have some array tests, but this is where we put other
563 ad-hoc things array related.
566 oceani_tests += arrays
574 bools[3] = strings[1] == "Hello"
575 bools[1] = strings[2] <= "there"
577 for i:=0; then i=i+1; while i<5:
581 ra[6] = 42 // mustn't crash
582 print '', bools[i], ra[j-1],
585 ###### output: arrays
586 False 0 True 1 False 4 False 9 False 16
590 Time to test if structure declarations and accesses work correctly.
593 oceani_tests += structs
602 active:Boolean = True
604 struct baz { a:number; b:Boolean; }
610 for i:=0; then i=i+1; while i < 4:
618 info[i].size[0] = i*i
620 info[i].active = False
622 for i:=0; then i=i+1; while i < 4:
623 print info[i].name, info[i].active, info[i].size[0]
624 info[0].thing.b = True
626 ###### output: structs
635 Test functions. They don't return anything, so we need to get them to print
638 oceani_tests += functions func_ret_type
640 ###### test: functions
660 func test(n:number; s:string)
677 for i:=0; then i = i + 1; while i < 5:
679 angular := to_polar(32, 23)
680 print angular.rho, angular.theta
682 ###### output: functions
687 4 3 . 2 .. 1 ... done
690 ###### test: func_ret_type
692 func double(n:number):number
708 for j:=10; then j = j - 3; while j > -5:
709 print answer("dou","ble"), j, "is", double(j)
711 ###### output: func_ret_type
720 A simple linked list example
723 oceani_tests += "linked_list,one,two,three,four"
725 ###### test: linked_list
733 func insert(list:@linkage; new:string)
735 while ?p.next and then p.next.this < new:
742 func printlist(list:@linkage)
744 print list@.next.this
745 list = list@.next.list
747 func freelist(list:@linkage)
748 if list.next != @nil:
749 lp:@linkage = list.next.list
753 func main(argv:[ac::]string)
756 insert(list, "@start");
758 for i:=1; then i=i+1; while i < argv[]:
759 insert(list, argv[i])
760 insert(list, "Hello!")
764 ###### output: linked_list,one,two,three,four
773 ## Test code with syntax errors
775 Syntax errors aren't handled well yet - the result is almost always a
776 single message about the first error. So this section will be fairly
777 thin until we add proper parsing recovery in the face of common errors.
779 A special case of syntax errors is token errors, when a token is only
780 accepted because the parser doesn't know quite enough to reject it.
781 There are handled better as they are quite local, so a single test
782 program can trigger most of the possible errors.
784 To handle erronous code, we need a different set of tests, as we need to
785 capture `stderr`. The same test code will be used for type errors too.
786 As error messages contain the line number, and we don't want changes to
787 this file to change the reported numbers, we copy the code into a
788 separate file first, then run from there.
791 @for t in $(oceani_failing_tests); do \
792 echo -n "Test $$t ... "; \
793 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
794 echo '``````' > .tmp.code; \
795 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
796 ./oceani .tmp.code > .tmp.have 2>&1; \
797 if ! cmp -s .tmp.want .tmp.have; then \
798 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
800 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
803 ###### combine test lists
804 oceani_valg_tests += $(oceani_failing_tests)
807 oceani_failing_tests := syn1
808 oceani_failing_tests += tokerr
813 if then else while do
816 .tmp.code:3:11: Syntax error in statement: then
820 a := 1i // imaginary numbers aren't understood
821 b:[2i]number // array sizes are handled separately
822 c:[3.14159]Boolean // array sizes must be integers
823 d:[1_000_000_000_000]number // they mustn't be huge
824 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
827 This is a multiline string
828 With an unsupportable suffix
832 yy:[unknowable]number
834 zz:[zzsize]string // size must be constant, use ::=
836 // These numbers should be bad in all contexts: FIXME
839 ###### output: tokerr
840 .tmp.code:3:13: error: unsupported number suffix: 1i
841 .tmp.code:4:11: error: unsupported number suffix: 2i
842 .tmp.code:5:11: error: array size must be an integer: 3.14159
843 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
844 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
845 .tmp.code:9:17: error: unsupported string suffix: """
846 This is a multiline string
847 With an unsupportable suffix
849 .tmp.code:15:12: error: name undeclared: unknowable
850 .tmp.code:17:12: error: array size must be a constant: zzsize
851 .tmp.code:20:12: error: unrecognised number: 00123
852 .tmp.code:14:11: error: type used but not declared: unknown
854 ## Tests for type errors
856 Type error don't cause parsing to abort, so we can fit many in the
857 one test program. Some type errors are found during the parse, others
858 during type analysis which doesn't run if parsing failed. So we cannot
859 fit everything in one.
861 These programs were generated by looking for the
862 various places that `type_err()` are called.
865 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4 type_err5 type_err6
867 ###### test: type_err1
870 print "hello" ++ 5, 5 ++ "hello"
875 if 3 * 4 and not True: print "Weird"
879 ###### output: type_err1
880 .tmp.code:3:25: error: expected string found number
881 .tmp.code:3:28: error: expected string found number
882 .tmp.code:6:8: error: Cannot assign to a constant: b
883 .tmp.code:5:8: info: name was defined as a constant here
884 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
885 .tmp.code:9:20: error: expected number found label
886 .tmp.code:9:8: info: variable 'd' was set as number here.
887 .tmp.code:10:8: error: cannot assign to an rval
888 oceani: type error in program - not running.
890 ###### test: type_err2
900 ###### output: type_err2
901 .tmp.code:4:8: error: variable 'a' redeclared
902 .tmp.code:3:8: info: this is where 'a' was first declared
903 .tmp.code:5:8: error: variable 'a' redeclared
904 .tmp.code:3:8: info: this is where 'a' was first declared
905 .tmp.code:6:8: error: variable 'a' redeclared
906 .tmp.code:3:8: info: this is where 'a' was first declared
907 .tmp.code:7:8: error: variable 'a' redeclared
908 .tmp.code:3:8: info: this is where 'a' was first declared
909 .tmp.code:8:8: Variable declared with no type or value: c
911 ###### test: type_err3
920 c = "hello" ++ (True and False)
922 print 45 + ( "Hello" ++ "there")
932 case "Hello": print "Hello"
934 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
948 // trigger 'labels not permitted' error message
949 while 1 if True else False:
952 print "one" ++ a4[], c[]
954 ###### output: type_err3
955 .tmp.code:8:12: error: expected number but variable 'c' is string
956 .tmp.code:7:8: info: this is where 'c' was set to string
957 .tmp.code:8:12: error: Arithmetic returns number but string expected
958 .tmp.code:7:8: info: variable 'c' was set as string here.
959 .tmp.code:9:24: error: Boolean operation found where string expected
960 .tmp.code:7:8: info: variable 'c' was set as string here.
961 .tmp.code:10:12: error: Comparison returns Boolean but string expected
962 .tmp.code:7:8: info: variable 'c' was set as string here.
963 .tmp.code:11:21: error: Concat returns string but number expected
964 .tmp.code:12:8: error: string cannot be indexed
965 .tmp.code:12:8: error: string cannot be indexed
966 .tmp.code:21:13: error: expected number found string
967 .tmp.code:17:16: error: expected number, found string
968 .tmp.code:24:8: error: cannot assign value of type [5]number
969 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
970 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
971 .tmp.code:25:8: error: cannot assign value of type [5]number
972 .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
973 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
974 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
975 .tmp.code:26:8: error: cannot assign value of type [5]number
976 .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
977 .tmp.code:27:16: error: expected number found string
978 .tmp.code:28:16: error: expected string found Boolean
979 .tmp.code:29:12: error: have number but need string
980 .tmp.code:7:8: info: variable 'c' was set as string here.
981 .tmp.code:32:8: error: variable used but not declared: foo
982 .tmp.code:32:8: error: field reference on none is not supported
983 .tmp.code:32:16: error: expected none found number
984 .tmp.code:33:14: error: field reference on string is not supported
985 .tmp.code:34:14: error: cannot find requested field in foo
986 .tmp.code:35:17: error: have string but need number
987 .tmp.code:38:29: error: expected number found Boolean
988 .tmp.code:41:23: error: have number but need string
989 .tmp.code:41:29: error: string cannot provide length
990 oceani: type error in program - not running.
992 ###### test: type_err4
997 ###### output: type_err4
998 .tmp.code:3:14: error: variable used but not declared: b
999 .tmp.code:3:16: error: expected none found number
1000 .tmp.code:3:14: info: variable 'b' was set as none here.
1001 oceani: type error in program - not running.
1003 ###### test: type_err5
1013 ###### output: type_err5
1014 .tmp.code:8:7: error: type already declared: foo
1015 .tmp.code:2:7: info: this is location of declartion: foo
1016 .tmp.code:5:7: error: type has recursive definition: baz
1017 .tmp.code:2:7: error: type has recursive definition: foo
1019 ###### test: type_err6
1027 ###### output: type_err6
1028 .tmp.code:4:12: error: '?' requires a testable value, not string
1029 .tmp.code:6:14: error: "??" requires a testable value, not string
1030 oceani: type error in program - not running.
1034 oceani_failing_tests += type_err_const type_err_const1 type_err_const2 missing_program bad_main
1036 ###### test: type_err_const
1039 bar ::= "string" + 56
1046 // trigger duplicate-main error
1051 ###### output: type_err_const
1052 .tmp.code:6:8: error: name already declared: bar
1053 .tmp.code:4:8: info: this is where 'bar' was first declared
1054 .tmp.code:8:8: error: variable 'foo' redeclared
1055 .tmp.code:3:8: info: this is where 'foo' was first declared
1056 .tmp.code:12:5: error: function 'main' redeclared
1057 .tmp.code:7:5: info: this is where 'main' was first declared
1058 .tmp.code:13:8: error: variable 'foo' redeclared
1059 .tmp.code:3:8: info: this is where 'foo' was first declared
1060 .tmp.code:4:16: error: expected number found string
1062 ###### test: type_err_const1
1070 ###### output: type_err_const1
1071 .tmp.code:3:12: Syntax error in constant: :
1072 .tmp.code:4:12: Syntax error in constant: :
1074 ###### test: type_err_const2
1079 ###### output: type_err_const2
1080 .tmp.code:3:8: error: const four cannot be resolved.
1081 .tmp.code:4:8: error: const two cannot be resolved.
1083 ###### test: missing_program
1087 ###### output: missing_program
1088 oceani: no main function found.
1090 ###### test: bad_main
1091 func main(foo:string)
1094 ###### output: bad_main
1095 .tmp.code:2:10: error: expected []string but variable 'foo' is string
1096 .tmp.code:2:10: info: this is where 'foo' was set to string
1097 oceani: main has wrong type.
1099 Test for type errors with functions
1102 oceani_failing_tests += func_err_args func_err_redeclare
1104 ###### test: func_err_args
1106 func test1(a:number; b:string; c:[3]Boolean)
1109 func test2(a:number; b:string; c:[3]Boolean)
1113 # use undefined names
1121 test1(1, "two", truth)
1123 test1(1, "lo", truth, 4)
1124 print test(), test1(1,2,3)
1128 func test4(a:number):string
1131 func test5(a:number):string
1138 func test6(a:number):foo
1143 func test7(a:@number)
1147 ###### output: func_err_args
1148 .tmp.code:40:14: error: cannot pass rval when reference expected
1149 .tmp.code:41:14: error: expected @number found string
1150 .tmp.code:34:5: error: function cannot return value of type foo
1151 .tmp.code:28:8: error: expected string, found none
1152 .tmp.code:25:8: error: expected string, found number
1153 .tmp.code:15:14: error: insufficient arguments to function.
1154 .tmp.code:16:14: error: expected number found string
1155 .tmp.code:16:22: error: expected string found number
1156 .tmp.code:16:14: error: insufficient arguments to function.
1157 .tmp.code:18:17: error: expected string found number
1158 .tmp.code:19:14: error: too many arguments to function.
1159 .tmp.code:20:14: error: attempt to call a non-function.
1160 .tmp.code:20:32: error: expected string found number
1161 .tmp.code:20:28: error: insufficient arguments to function.
1162 .tmp.code:21:20: error: expected "func test1" but variable 'test2' is "func test2"
1163 .tmp.code:5:5: info: this is where 'test2' was set to "func test2"
1164 .tmp.code:10:14: error: variable used but not declared: a
1165 .tmp.code:10:17: error: variable used but not declared: z
1166 oceani: type error in program - not running.
1168 ###### test: func_err_redeclare
1170 func test1(a:number; b:string; c:[3]Boolean)
1182 ###### output: func_err_redeclare
1183 .tmp.code:5:5: error: function 'test1' redeclared
1184 .tmp.code:2:5: info: this is where 'test1' was first declared
1185 .tmp.code:9:5: error: function 'test1' redeclared
1186 .tmp.code:2:5: info: this is where 'test1' was first declared
1188 Test for errors with references
1191 oceani_failing_tests += ref_err1 ref_err2
1193 ###### test: ref_err1
1201 ###### output: ref_err1
1202 .tmp.code:4:9: error: only "@free" makes sense here: foo
1203 .tmp.code:5:15: error: Only reference function is "@new()": old
1204 .tmp.code:6:19: error: Only reference value is "@nil": null
1206 ###### test: ref_err2
1212 if num == @nil or ref == ref2 or ref == 2 or ref.foo:
1216 ###### output: ref_err2
1217 .tmp.code:5:22: error: @new() can only be used with references, not number
1218 .tmp.code:5:8: info: variable 'num' was set as number here.
1219 .tmp.code:6:14: error: Cannot dereference number
1220 .tmp.code:7:19: error: @nil can only be used with reference, not number
1221 .tmp.code:7:33: error: expected @number but variable 'ref2' is @string
1222 .tmp.code:4:8: info: this is where 'ref2' was set to @string
1223 .tmp.code:7:48: error: expected @number found number
1224 .tmp.code:7:53: error: field reference on number is not supported
1225 .tmp.code:7:56: error: have none but need Boolean
1226 .tmp.code:8:17: error: @free can only be assigned a reference, not number
1227 .tmp.code:8:17: error: @free can only be assigned a reference, not number
1228 .tmp.code:9:8: error: Cannot assign an rval to a reference.
1229 oceani: type error in program - not running.
1231 ## Test erroneous command line args
1233 To improve coverage, we want to test correct handling of strange command
1234 line arguments. These tests won't use code, so the exiting test types
1235 won't work. So we need to be able to explicitly give the command line,
1236 and the expected output, and have that tested and the coverage assessed.
1237 Rather than having to spell out the whole command name, just give "cmd",
1238 and discard that. Requiring but discarding the command make an empty
1239 command list possible.
1242 @for t in $(oceani_special_tests); do \
1243 echo -n "Test $$t ... ";\
1244 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1245 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
1246 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1247 if ! cmp -s .tmp.want .tmp.have; then \
1248 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
1250 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
1252 ###### valgrind test code
1253 @[ -n "$$SKIP_VALGRIND" ] || for t in $(oceani_special_tests); do\
1254 echo -n "Valgrind $$t.. "; \
1255 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
1256 if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
1257 [ $$? -eq 42 ]; then \
1258 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
1259 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
1260 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
1261 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
1262 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
1267 oceani_special_tests += "cmd"
1268 oceani_special_tests += "cmd,-zyx"
1269 oceani_special_tests += "cmd,nofile"
1270 oceani_special_tests += "cmd,/dev/null"
1271 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
1274 oceani: no input file given
1276 ###### output: cmd,-zyx
1277 ./oceani: invalid option -- 'z'
1278 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
1280 ###### output: cmd,nofile
1281 oceani: cannot open nofile
1283 ###### output: cmd,/dev/null
1284 oceani: could not find any code in /dev/null
1286 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
1287 oceani: cannot find section toast:nothing