1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "exec-after-print.. "; \
61 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
62 if ! cmp -s .tmp.want .tmp.have; then \
63 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
65 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
66 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
71 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
72 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
73 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
74 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
75 if (ran < (ran + skip) *0.94) exit(1) }' \
76 coverage/oceani.mdc.gcov
79 @for T in $(oceani_tests); do \
80 echo -n "Valgrind $$T.. "; \
81 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
82 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
83 > /dev/null 2>&1 ; then \
84 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
85 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
86 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
87 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
88 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
92 coverage_oceani: oceani.c
93 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
95 ## Values and variables
97 The first test stores values in variables and performs various
101 oceani_tests += "valvar"
106 a := 23; b:=12 ; b1 := -b
107 print a, b, a+b, a-b, a*b, a/b, a%b
108 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
109 print +a, +b, +b1, -a, -b, -b1
110 x := True; y := False
111 print x and y, x or y, x and x, y or y, x and not x, x < y
113 c ::= "This is a string"
114 d ::= " field theory"
117 aconst :: string = "unchanging"
126 ###### output: valvar
128 23 12 35 11 276 1.91667 11
129 False False True True False False True
131 False True True False False False
132 This is a string field theory This is a string field theory
135 Next we change the value of variables
138 oceani_tests += "setvar"
145 a = (a + a) * (a + a)
149 ###### output: setvar
155 oceani_tests += "consts"
160 four ::= 2 + 2 ; five ::= 10/2
161 const pie ::= "I like Pie";
162 cake ::= "The cake is"
166 print "Hello World, what lovely oceans you have!"
167 print "are there", five, "?"
168 print pi, pie, "but", cake
170 ###### output: consts
171 Hello World, what lovely oceans you have!
173 3.14159 I like Pie but The cake is a lie
175 Test merging of variables from multiple cases
178 oceani_tests += varmerge
180 ###### test: varmerge
183 for i:=0; then i=i+1; while i < 5:
193 ###### output: varmerge
194 zero , one , two , three , many ,
196 ## Conditions and Loops
198 Now we need to test if/else and some different loops
201 oceani_tests += cond_loop
203 ###### test: cond_loop
211 for b:=1; then b=b+b; while b < 100:
214 // Newtons method for square root of 2
220 current := guess * guess
221 use +(current - target) > 0.000000001
223 guess = (guess + (target / guess) ) / 2
226 print "error is ", target - guess * guess
228 for j:=0; then j = j+3 ; while j < 10:
229 if j != 0 and then 20 / j > 3:
230 print "20 /", j," =", 20 / j
232 print "I won't calculate 20 /", j
233 pi ::= 3.1415926535897
234 if 355/113 == pi or else +(pi - 355/113) < 0.001:
236 print "lower" if 355/113 < pi else "higher"
238 ###### output: cond_loop
245 error is -4.51095e-12
246 I won't calculate 20 / 0
249 I won't calculate 20 / 9
255 The demonstration code presented in the interpreted is suitable for the test suite.
256 Here I break it into two parts, keeping the array code separate.
259 oceani_tests += "sayhello,55,33,hello,True"
260 oceani_tests += "sayhello,12,60,there,False"
262 ###### test: sayhello
264 program A B astr bbool:
265 print "Hello World, what lovely oceans you have!"
266 /* When a variable is defined in both branches of an 'if',
267 * and used afterwards, the variables are merged.
273 print "Is", A, "bigger than", B,"? ", bigger
274 /* If a variable is not used after the 'if', no
275 * merge happens, so types can be different
278 double:string = "yes"
279 print A, "is more than twice", B, "?", double
282 print "double", B, "is", double
293 print "GCD of", A, "and", B,"is", a
295 print a, "is not positive, cannot calculate GCD"
297 print b, "is not positive, cannot calculate GCD"
302 print "Fibonacci:", f1,f2,
312 print astr ++ " was the str"
314 print "I found the str over " ++ astr
316 /* Binary search... */
333 print "Yay, I found", target
335 print "Closest I found was", mid
337 ###### output: sayhello,55,33,hello,True
338 Hello World, what lovely oceans you have!
339 Is 55 bigger than 33 ? yes
341 GCD of 55 and 33 is 11
342 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
344 Closest I found was 77.3438
346 ###### output: sayhello,12,60,there,False
347 Hello World, what lovely oceans you have!
348 Is 12 bigger than 60 ? no
350 GCD of 12 and 60 is 12
351 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
352 I found the str over there
353 Closest I found was 77.3438
356 oceani_tests += "insert_sort"
357 ###### test: insert_sort
362 for i:=1; then i = i + 1; while i < size:
363 n := list[i-1] * list[i-1]
364 list[i] = (n / 100) % 10000
367 for i:=0; then i = i + 1; while i < size:
368 print "list[",i,"]=",list[i]
370 for i := 1; then i=i+1; while i < size:
371 for j:=i-1; then j=j-1; while j >= 0:
372 if list[j] > list[j+1]:
377 for i:=0; then i = i + 1; while i < size:
378 print "list[",i,"]=",list[i]
380 ###### output: insert_sort
496 We already have some array tests, but this is where we put other
497 ad-hoc things array related.
500 oceani_tests += arrays
508 bools[3] = strings[1] == "Hello"
509 bools[1] = strings[2] <= "there"
511 for i:=0; then i=i+1; while i<5:
515 ###### output: arrays
516 False True False False False
520 Time to test if structure declarations and accesses work correctly.
523 oceani_tests += structs
532 struct baz { a:number; b:Boolean; }
537 for i:=0; then i=i+1; while i < 4:
545 info[i].size[0] = i*i
546 info[i].active = nm == "jane"
548 for i:=0; then i=i+1; while i < 4:
549 print info[i].name, info[i].active, info[i].size[0]
551 ###### output: structs
558 ## Test code with syntax errors
560 Syntax errors aren't handled well yet - the result is almost always a
561 single message about the first error. So this section will be fairly
562 thin until we add proper parsing recovery in the face of common errors.
564 A special case of syntax errors is token errors, when a token is only
565 accepted because the parser doesn't know quite enough to reject it.
566 There are handled better as they are quite local, so a single test
567 program can trigger most of the possible errors.
569 To handle erronous code, we need a different set of tests, as we need to
570 capture `stderr`. The same test code will be used for type errors too.
571 As error messages contain the line number, and we don't want changes to
572 this file to change the reported numbers, we copy the code into a
573 separate file first, then run from there.
576 @for t in $(oceani_failing_tests); do \
577 echo -n "Test $$t ... "; \
578 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
579 echo '``````' > .tmp.code; \
580 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
581 ./oceani .tmp.code > .tmp.have 2>&1; \
582 if ! cmp -s .tmp.want .tmp.have; then \
583 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
585 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
589 oceani_failing_tests := syn1
590 oceani_failing_tests += tokerr
595 if then else while do
598 .tmp.code:3:11: error: unhandled parse error: then
599 oceani: no program found.
603 a := 1i // imaginary numbers aren't understood
604 b:[2i]number // array sizes are handled separately
605 c:[3.14159]Boolean // array sizes must be integers
606 d:[1_000_000_000_000]number // they mustn't be huge
607 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
610 This is a multiline string
611 With an unsupportable suffix
615 yy:[unknowable]number
617 zz:[zzsize]string // size must be constant, use ::=
619 // These numbers should be bad in all contexts: FIXME
622 ###### output: tokerr
623 .tmp.code:3:13: error: unsupported number suffix: 1i
624 .tmp.code:4:11: error: unsupported number suffix: 2i
625 .tmp.code:5:11: error: array size must be an integer: 3.14159
626 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
627 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
628 .tmp.code:9:17: error: unsupported string suffix: """
629 This is a multiline string
630 With an unsupportable suffix
632 .tmp.code:14:11: error: undefined type: unknown
633 .tmp.code:15:12: error: name undeclared: unknowable
634 .tmp.code:17:12: error: array size must be a constant: zzsize
635 .tmp.code:20:12: error: unrecognised number: 00123
637 ## Tests for type errors
639 Type error don't cause parsing to abort, so we can fit many in the
640 one test program. Some type errors are found during the parse, others
641 during type analysis which doesn't run if parsing failed. So we cannot
642 fit everything in one.
644 These programs were generated by looking for the
645 various places that `type_err()` are called.
648 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
650 ###### test: type_err1
653 print "hello" ++ 5, 5 ++ "hello"
658 if 3 * 4 and not True: print "Weird"
660 ###### output: type_err1
661 .tmp.code:3:25: error: expected string found number
662 .tmp.code:3:28: error: expected string found number
663 .tmp.code:6:8: error: Cannot assign to a constant: b
664 .tmp.code:5:8: info: name was defined as a constant here
665 .tmp.code:6:8: error: Cannot assign to a constant: b
666 .tmp.code:5:8: info: name was defined as a constant here
667 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
668 oceani: type error in program - not running.
670 ###### test: type_err2
680 ###### output: type_err2
681 .tmp.code:4:8: error: variable 'a' redeclared
682 .tmp.code:3:8: info: this is where 'a' was first declared
683 .tmp.code:5:8: error: variable 'a' redeclared
684 .tmp.code:3:8: info: this is where 'a' was first declared
685 .tmp.code:6:8: error: variable 'a' redeclared
686 .tmp.code:3:8: info: this is where 'a' was first declared
687 .tmp.code:7:8: error: variable 'a' redeclared
688 .tmp.code:3:8: info: this is where 'a' was first declared
689 .tmp.code:8:8: Variable declared with no type or value: c
691 ###### test: type_err3
700 c = "hello" ++ (True and False)
702 print 45 + ( "Hello" ++ "there")
712 case "Hello": print "Hello"
714 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
726 ###### output: type_err3
727 .tmp.code:8:12: error: expected number but variable 'c' is string
728 .tmp.code:7:8: info: this is where 'c' was set to string
729 .tmp.code:8:12: error: Arithmetic returns number but string expected
730 .tmp.code:7:8: info: variable 'c' was set as string here.
731 .tmp.code:9:24: error: Boolean operation found where string expected
732 .tmp.code:10:12: error: Comparison returns Boolean but string expected
733 .tmp.code:7:8: info: variable 'c' was set as string here.
734 .tmp.code:11:21: error: Concat returns string but number expected
735 .tmp.code:12:8: error: string cannot be indexed
736 .tmp.code:12:8: error: string cannot be indexed
737 .tmp.code:21:13: error: expected number found string
738 .tmp.code:17:16: error: expected number, found string
739 .tmp.code:24:8: error: cannot assign value of type [5]number
740 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
741 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
742 .tmp.code:25:8: error: cannot assign value of type [5]number
743 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
744 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
745 .tmp.code:26:8: error: cannot assign value of type [5]number
746 .tmp.code:27:16: error: expected number found string
747 .tmp.code:28:16: error: expected string found Boolean
748 .tmp.code:29:12: error: have number but need string
749 .tmp.code:7:8: info: variable 'c' was set as string here.
750 .tmp.code:32:8: error: field reference attempted on label, not a struct
751 .tmp.code:32:16: error: expected none found number
752 .tmp.code:33:14: error: field reference attempted on string, not a struct
753 oceani: type error in program - not running.
755 ###### test: type_err4
760 ###### output: type_err4
761 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
762 .tmp.code:3:14: info: this is where 'b' was set to label
763 .tmp.code:3:16: error: expected label found number
764 .tmp.code:3:14: info: variable 'b' was set as label here.
765 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
766 .tmp.code:3:14: info: this is where 'b' was set to label
767 oceani: type error in program - not running.
770 oceani_failing_tests += type_err_const type_err_const1
772 ###### test: type_err_const
775 bar ::= "string" + 56
782 ###### output: type_err_const
783 .tmp.code:4:16: error: expected number found string
784 .tmp.code:6:8: error: name already declared: bar
785 .tmp.code:4:8: info: this is where 'bar' was first declared
786 .tmp.code:8:8: error: variable 'foo' redeclared
787 .tmp.code:3:8: info: this is where 'foo' was first declared
789 ###### test: type_err_const1
797 ###### output: type_err_const1
798 .tmp.code:3:12: error: unhandled parse error: :
799 oceani: no program found.
801 ## Test erroneous command line args
803 To improve coverage, we want to test correct handling of strange command
804 line arguments. These tests won't use code, so the exiting test types
805 won't work. So we need to be able to explicitly give the command line,
806 and the expected output, and have that tested and the coverage assessed.
807 Rather than having to spell out the whole command name, just give "cmd",
808 and discard that. Requiring but discarding the command make an empty
809 command list possible.
812 @for t in $(oceani_special_tests); do \
813 echo -n "Test $$t ... ";\
814 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
815 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
816 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
817 if ! cmp -s .tmp.want .tmp.have; then \
818 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
820 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
824 oceani_special_tests += "cmd"
825 oceani_special_tests += "cmd,-zyx"
826 oceani_special_tests += "cmd,nofile"
827 oceani_special_tests += "cmd,/dev/null"
828 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
831 oceani: no input file given
833 ###### output: cmd,-zyx
834 ./oceani: invalid option -- 'z'
835 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
837 ###### output: cmd,nofile
838 oceani: cannot open nofile
840 ###### output: cmd,/dev/null
841 oceani: could not find any code in /dev/null
843 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
844 oceani: cannot find section toast:nothing