1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
42 { echo "Grammar container conflicts, please review" ; exit 1; }
43 @rm -rf coverage; mkdir -p coverage
45 @for T in $(oceani_tests); do \
46 echo -n "Test $$T.. "; \
47 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
48 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
49 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
50 if ! cmp -s .tmp.want .tmp.have; then \
51 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
52 echo -n "printing.. "; \
53 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
54 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
55 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
56 if ! cmp -s .tmp.code1 .tmp.code2; then \
57 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
58 echo -n "exec-after-print.. "; \
59 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
60 if ! cmp -s .tmp.want .tmp.have; then \
61 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
63 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
64 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
69 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
70 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
71 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
72 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
73 if (ran < (ran + skip) *0.94) exit(1) }' \
74 coverage/oceani.mdc.gcov
77 @for T in $(oceani_tests); do \
78 echo -n "Valgrind $$T.. "; \
79 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
80 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
81 > /dev/null 2>&1 ; then \
82 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
83 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
84 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
85 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
86 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
90 coverage_oceani: oceani.c
91 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
93 ## Values and variables
95 The first test stores values in variables and performs various
99 oceani_tests += "valvar"
105 a := 23; b:=12 ; b1 := -b
106 print a, b, a+b, a-b, a*b, a/b, a%b
107 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
108 print +a, +b, +b1, -a, -b, -b1
109 x := True; y := False
110 print x and y, x or y, x and x, y or y, x and not x, x < y
112 c ::= "This is a string"
113 d ::= " field theory"
116 aconst :: string = "unchanging"
118 ###### output: valvar
120 23 12 35 11 276 1.91667 11
121 False False True True False False True
123 False True True False False False
124 This is a string field theory This is a string field theory
126 Next we change the value of variables
129 oceani_tests += "setvar"
136 a = (a + a) * (a + a)
140 ###### output: setvar
146 oceani_tests += "consts"
151 four ::= 2 + 2 ; five ::= 10/2
152 const pie ::= "I like Pie";
153 cake ::= "The cake is"
157 print "Hello World, what lovely oceans you have!"
158 print "are there", five, "?"
159 print pi, pie, "but", cake
161 ###### output: consts
162 Hello World, what lovely oceans you have!
164 3.14159 I like Pie but The cake is a lie
166 Test merging of variables from multiple cases
169 oceani_tests += varmerge
171 ###### test: varmerge
174 for i:=0; then i=i+1; while i < 5:
184 ###### output: varmerge
185 zero , one , two , three , many ,
187 ## Conditions and Loops
189 Now we need to test if/else and some different loops
192 oceani_tests += cond_loop
194 ###### test: cond_loop
202 for b:=1; then b=b+b; while b < 100:
205 // Newtons method for square root of 2
211 current := guess * guess
212 use +(current - target) > 0.000000001
214 guess = (guess + (target / guess) ) / 2
217 print "error is ", target - guess * guess
219 for j:=0; then j = j+3 ; while j < 10:
220 if j != 0 and then 20 / j > 3:
221 print "20 /", j," =", 20 / j
223 print "I won't calculate 20 /", j
224 pi ::= 3.1415926535897
225 if 355/113 == pi or else +(pi - 355/113) < 0.001:
227 print "lower" if 355/113 < pi else "higher"
229 ###### output: cond_loop
236 error is -4.51095e-12
237 I won't calculate 20 / 0
240 I won't calculate 20 / 9
246 The demonstration code presented in the interpreted is suitable for the test suite.
247 Here I break it into two parts, keeping the array code separate.
250 oceani_tests += "sayhello,55,33,hello,True"
251 oceani_tests += "sayhello,12,60,there,False"
253 ###### test: sayhello
255 program A B astr bbool:
256 print "Hello World, what lovely oceans you have!"
257 /* When a variable is defined in both branches of an 'if',
258 * and used afterwards, the variables are merged.
264 print "Is", A, "bigger than", B,"? ", bigger
265 /* If a variable is not used after the 'if', no
266 * merge happens, so types can be different
269 double:string = "yes"
270 print A, "is more than twice", B, "?", double
273 print "double", B, "is", double
284 print "GCD of", A, "and", B,"is", a
286 print a, "is not positive, cannot calculate GCD"
288 print b, "is not positive, cannot calculate GCD"
293 print "Fibonacci:", f1,f2,
303 print astr ++ " was the str"
305 print "I found the str over " ++ astr
307 /* Binary search... */
324 print "Yay, I found", target
326 print "Closest I found was", mid
328 ###### output: sayhello,55,33,hello,True
329 Hello World, what lovely oceans you have!
330 Is 55 bigger than 33 ? yes
332 GCD of 55 and 33 is 11
333 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
335 Closest I found was 77.3438
337 ###### output: sayhello,12,60,there,False
338 Hello World, what lovely oceans you have!
339 Is 12 bigger than 60 ? no
341 GCD of 12 and 60 is 12
342 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
343 I found the str over there
344 Closest I found was 77.3438
347 oceani_tests += "insert_sort"
348 ###### test: insert_sort
353 for i:=1; then i = i + 1; while i < size:
354 n := list[i-1] * list[i-1]
355 list[i] = (n / 100) % 10000
358 for i:=0; then i = i + 1; while i < size:
359 print "list[",i,"]=",list[i]
361 for i := 1; then i=i+1; while i < size:
362 for j:=i-1; then j=j-1; while j >= 0:
363 if list[j] > list[j+1]:
368 for i:=0; then i = i + 1; while i < size:
369 print "list[",i,"]=",list[i]
371 ###### output: insert_sort
487 We already have some array tests, but this is where we put other
488 ad-hoc things array related.
491 oceani_tests += arrays
499 bools[3] = strings[1] == "Hello"
500 bools[1] = strings[2] <= "there"
502 for i:=0; then i=i+1; while i<5:
506 ###### output: arrays
507 False True False False False
511 Time to test if structure declarations and accesses work correctly.
514 oceani_tests += structs
523 struct baz { a:number; b:Boolean; }
528 for i:=0; then i=i+1; while i < 4:
536 info[i].size[0] = i*i
537 info[i].active = nm == "jane"
539 for i:=0; then i=i+1; while i < 4:
540 print info[i].name, info[i].active, info[i].size[0]
542 ###### output: structs
549 ## Test code with syntax errors
551 Syntax errors aren't handled well yet - the result is almost always a
552 single message about the first error. So this section will be fairly
553 thin until we add proper parsing recovery in the face of common errors.
555 A special case of syntax errors is token errors, when a token is only
556 accepted because the parser doesn't know quite enough to reject it.
557 There are handled better as they are quite local, so a single test
558 program can trigger most of the possible errors.
560 To handle erronous code, we need a different set of tests, as we need to
561 capture `stderr`. The same test code will be used for type errors too.
562 As error messages contain the line number, and we don't want changes to
563 this file to change the reported numbers, we copy the code into a
564 separate file first, then run from there.
567 @for t in $(oceani_failing_tests); do \
568 echo -n "Test $$t ... "; \
569 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
570 echo '``````' > .tmp.code; \
571 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
572 ./oceani .tmp.code > .tmp.have 2>&1; \
573 if ! cmp -s .tmp.want .tmp.have; then \
574 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
576 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
580 oceani_failing_tests := syn1
581 oceani_failing_tests += tokerr
586 if then else while do
589 .tmp.code:3:11: error: unhandled parse error: then
590 oceani: no program found.
594 a := 1i // imaginary numbers aren't understood
595 b:[2i]number // array sizes are handled separately
596 c:[3.14159]Boolean // array sizes must be integers
597 d:[1_000_000_000_000]number // they mustn't be huge
598 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
601 This is a multiline string
602 With an unsupportable suffix
606 yy:[unknowable]number
608 zz:[zzsize]string // size must be constant, use ::=
610 // These numbers should be bad in all contexts: FIXME
613 ###### output: tokerr
614 .tmp.code:3:13: error: unsupported number suffix: 1i
615 .tmp.code:4:11: error: unsupported number suffix: 2i
616 .tmp.code:5:11: error: array size must be an integer: 3.14159
617 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
618 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
619 .tmp.code:9:17: error: unsupported string suffix: """
620 This is a multiline string
621 With an unsupportable suffix
623 .tmp.code:14:11: error: undefined type: unknown
624 .tmp.code:15:12: error: name undeclared: unknowable
625 .tmp.code:17:12: error: array size must be a constant: zzsize
626 .tmp.code:20:12: error: unrecognised number: 00123
628 ## Tests for type errors
630 Type error don't cause parsing to abort, so we can fit many in the
631 one test program. Some type errors are found during the parse, others
632 during type analysis which doesn't run if parsing failed. So we cannot
633 fit everything in one.
635 These programs were generated by looking for the
636 various places that `type_err()` are called.
639 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
641 ###### test: type_err1
644 print "hello" ++ 5, 5 ++ "hello"
649 if 3 * 4 and not True: print "Weird"
651 ###### output: type_err1
652 .tmp.code:3:25: error: expected string found number
653 .tmp.code:3:28: error: expected string found number
654 .tmp.code:6:8: error: Cannot assign to a constant: b
655 .tmp.code:5:8: info: name was defined as a constant here
656 .tmp.code:6:8: error: Cannot assign to a constant: b
657 .tmp.code:5:8: info: name was defined as a constant here
658 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
659 oceani: type error in program - not running.
661 ###### test: type_err2
671 ###### output: type_err2
672 .tmp.code:4:8: error: variable 'a' redeclared
673 .tmp.code:3:8: info: this is where 'a' was first declared
674 .tmp.code:5:8: error: variable 'a' redeclared
675 .tmp.code:3:8: info: this is where 'a' was first declared
676 .tmp.code:6:8: error: variable 'a' redeclared
677 .tmp.code:3:8: info: this is where 'a' was first declared
678 .tmp.code:7:8: error: variable 'a' redeclared
679 .tmp.code:3:8: info: this is where 'a' was first declared
680 .tmp.code:8:8: Variable declared with no type or value: c
682 ###### test: type_err3
691 c = "hello" ++ (True and False)
693 print 45 + ( "Hello" ++ "there")
703 case "Hello": print "Hello"
705 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
717 ###### output: type_err3
718 .tmp.code:8:12: error: expected number but variable 'c' is string
719 .tmp.code:7:8: info: this is where 'c' was set to string
720 .tmp.code:8:12: error: Arithmetic returns number but string expected
721 .tmp.code:7:8: info: variable 'c' was set as string here.
722 .tmp.code:9:24: error: Boolean operation found where string expected
723 .tmp.code:10:12: error: Comparison returns Boolean but string expected
724 .tmp.code:7:8: info: variable 'c' was set as string here.
725 .tmp.code:11:21: error: Concat returns string but number expected
726 .tmp.code:12:8: error: string cannot be indexed
727 .tmp.code:12:8: error: string cannot be indexed
728 .tmp.code:21:13: error: expected number found string
729 .tmp.code:17:16: error: expected number, found string
730 .tmp.code:24:8: error: cannot assign value of type [5]number
731 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
732 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
733 .tmp.code:25:8: error: cannot assign value of type [5]number
734 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
735 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
736 .tmp.code:26:8: error: cannot assign value of type [5]number
737 .tmp.code:27:16: error: expected number found string
738 .tmp.code:28:16: error: expected string found Boolean
739 .tmp.code:29:12: error: have number but need string
740 .tmp.code:7:8: info: variable 'c' was set as string here.
741 .tmp.code:32:8: error: field reference attempted on label, not a struct
742 .tmp.code:32:16: error: expected none found number
743 .tmp.code:33:14: error: field reference attempted on string, not a struct
744 oceani: type error in program - not running.
746 ###### test: type_err4
751 ###### output: type_err4
752 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
753 .tmp.code:3:14: info: this is where 'b' was set to label
754 .tmp.code:3:16: error: expected label found number
755 .tmp.code:3:14: info: variable 'b' was set as label here.
756 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
757 .tmp.code:3:14: info: this is where 'b' was set to label
758 oceani: type error in program - not running.
761 oceani_failing_tests += type_err_const type_err_const1
763 ###### test: type_err_const
766 bar ::= "string" + 56
773 ###### output: type_err_const
774 .tmp.code:4:16: error: expected number found string
775 .tmp.code:6:8: error: name already declared: bar
776 .tmp.code:4:8: info: this is where 'bar' was first declared
777 .tmp.code:8:8: error: variable 'foo' redeclared
778 .tmp.code:3:8: info: this is where 'foo' was first declared
780 ###### test: type_err_const1
788 ###### output: type_err_const1
789 .tmp.code:3:12: error: unhandled parse error: :
790 oceani: no program found.
793 ## Test erroneous command line args
795 To improve coverage, we want to test correct handling of strange command
796 line arguments. These tests won't use code, so the exiting test types
797 won't work. So we need to be able to explicitly give the command line,
798 and the expected output, and have that tested and the coverage assessed.
799 Rather than having to spell out the whole command name, just give "cmd",
800 and discard that. Requiring but discarding the command make an empty
801 command list possible.
804 @for t in $(oceani_special_tests); do \
805 echo -n "Test $$t ... ";\
806 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
807 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
808 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
809 if ! cmp -s .tmp.want .tmp.have; then \
810 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
812 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
817 oceani_special_tests += "cmd"
818 oceani_special_tests += "cmd,-zyx"
819 oceani_special_tests += "cmd,nofile"
820 oceani_special_tests += "cmd,/dev/null"
821 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
824 oceani: no input file given
826 ###### output: cmd,-zyx
827 ./oceani: invalid option -- 'z'
828 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
830 ###### output: cmd,nofile
831 oceani: cannot open nofile
833 ###### output: cmd,/dev/null
834 oceani: could not find any code in /dev/null
836 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
837 oceani: cannot find section toast:nothing