1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
42 { echo "Grammar container conflicts, please review" ; exit 1; }
43 @rm -rf coverage; mkdir -p coverage
45 @for T in $(oceani_tests); do \
46 echo -n "Test $$T.. "; \
47 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
48 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
49 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
50 if ! cmp -s .tmp.want .tmp.have; then \
51 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
52 echo -n "printing.. "; \
53 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
54 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
55 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
56 if ! cmp -s .tmp.code1 .tmp.code2; then \
57 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
58 echo -n "exec-after-print.. "; \
59 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
60 if ! cmp -s .tmp.want .tmp.have; then \
61 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
63 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
64 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
69 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
70 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
71 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
72 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
73 if (ran < (ran + skip) *0.93) exit(1) }' \
74 coverage/oceani.mdc.gcov
77 @for T in $(oceani_tests); do \
78 echo -n "Valgrind $$T.. "; \
79 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
80 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
81 > /dev/null 2>&1 ; then \
82 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
83 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
84 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
85 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
86 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
90 coverage_oceani: oceani.c
91 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
93 ## Values and variables
95 The first test stores values in variables and performs various
99 oceani_tests += "valvar"
105 a := 23; b:=12 ; b1 := -b
106 print a, b, a+b, a-b, a*b, a/b, a%b
107 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
108 print +a, +b, +b1, -a, -b, -b1
109 x := True; y := False
110 print x and y, x or y, x and x, y or y, x and not x, x < y
112 c ::= "This is a string"
113 d ::= " field theory"
116 aconst :: string = "unchanging"
118 ###### output: valvar
120 23 12 35 11 276 1.91667 11
121 False False True True False False True
123 False True True False False False
124 This is a string field theory This is a string field theory
126 Next we change the value of variables
129 oceani_tests += "setvar"
136 a = (a + a) * (a + a)
140 ###### output: setvar
146 oceani_tests += "consts"
151 four ::= 2 + 2 ; five ::= 10/2
152 const pie ::= "I like Pie";
153 cake ::= "The cake is"
157 print "Hello World, what lovely oceans you have!"
158 print "are there", five, "?"
159 print pi, pie, "but", cake
161 ###### output: consts
162 Hello World, what lovely oceans you have!
164 3.14159 I like Pie but The cake is a lie
166 Test merging of variables from multiple cases
169 oceani_tests += varmerge
171 ###### test: varmerge
174 for i:=0; then i=i+1; while i < 5:
184 ###### output: varmerge
185 zero , one , two , three , many ,
187 ## Conditions and Loops
189 Now we need to test if/else and some different loops
192 oceani_tests += cond_loop
194 ###### test: cond_loop
202 for b:=1; then b=b+b; while b < 100:
205 // Newtons method for square root of 2
211 current := guess * guess
212 use +(current - target) > 0.000000001
214 guess = (guess + (target / guess) ) / 2
217 print "error is ", target - guess * guess
219 for j:=0; then j = j+3 ; while j < 10:
220 if j != 0 and then 20 / j > 3:
221 print "20 /", j," =", 20 / j
223 print "I won't calculate 20 /", j
224 pi ::= 3.1415926535897
225 if 355/113 == pi or else +(pi - 355/113) < 0.001:
227 print "lower" if 355/113 < pi else "higher"
229 ###### output: cond_loop
236 error is -4.51095e-12
237 I won't calculate 20 / 0
240 I won't calculate 20 / 9
246 The demonstration code presented in the interpreted is suitable for the test suite.
247 Here I break it into two parts, keeping the array code separate.
250 oceani_tests += "sayhello,55,33"
251 oceani_tests += "sayhello,12,60"
253 ###### test: sayhello
256 print "Hello World, what lovely oceans you have!"
257 /* When a variable is defined in both branches of an 'if',
258 * and used afterwards, the variables are merged.
264 print "Is", A, "bigger than", B,"? ", bigger
265 /* If a variable is not used after the 'if', no
266 * merge happens, so types can be different
269 double:string = "yes"
270 print A, "is more than twice", B, "?", double
273 print "double", B, "is", double
284 print "GCD of", A, "and", B,"is", a
286 print a, "is not positive, cannot calculate GCD"
288 print b, "is not positive, cannot calculate GCD"
293 print "Fibonacci:", f1,f2,
302 /* Binary search... */
319 print "Yay, I found", target
321 print "Closest I found was", mid
323 ###### output: sayhello,55,33
324 Hello World, what lovely oceans you have!
325 Is 55 bigger than 33 ? yes
327 GCD of 55 and 33 is 11
328 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
329 Closest I found was 77.3438
331 ###### output: sayhello,12,60
332 Hello World, what lovely oceans you have!
333 Is 12 bigger than 60 ? no
335 GCD of 12 and 60 is 12
336 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
337 Closest I found was 77.3438
340 oceani_tests += "insert_sort"
341 ###### test: insert_sort
346 for i:=1; then i = i + 1; while i < size:
347 n := list[i-1] * list[i-1]
348 list[i] = (n / 100) % 10000
351 for i:=0; then i = i + 1; while i < size:
352 print "list[",i,"]=",list[i]
354 for i := 1; then i=i+1; while i < size:
355 for j:=i-1; then j=j-1; while j >= 0:
356 if list[j] > list[j+1]:
361 for i:=0; then i = i + 1; while i < size:
362 print "list[",i,"]=",list[i]
364 ###### output: insert_sort
480 We already have some array tests, but this is where we put other
481 ad-hoc things array related.
484 oceani_tests += arrays
492 bools[3] = strings[1] == "Hello"
493 bools[1] = strings[2] <= "there"
495 for i:=0; then i=i+1; while i<5:
499 ###### output: arrays
500 False True False False False
504 Time to test if structure declarations and accesses work correctly.
507 oceani_tests += structs
516 struct baz { a:number; b:Boolean; }
521 for i:=0; then i=i+1; while i < 4:
529 info[i].size[0] = i*i
530 info[i].active = nm == "jane"
532 for i:=0; then i=i+1; while i < 4:
533 print info[i].name, info[i].active, info[i].size[0]
535 ###### output: structs
542 ## Test code with syntax errors
544 Syntax errors aren't handled well yet - the result is almost always a
545 single message about the first error. So this section will be fairly
546 thin until we add proper parsing recovery in the face of common errors.
548 A special case of syntax errors is token errors, when a token is only
549 accepted because the parser doesn't know quite enough to reject it.
550 There are handled better as they are quite local, so a single test
551 program can trigger most of the possible errors.
553 To handle erronous code, we need a different set of tests, as we need to
554 capture `stderr`. The same test code will be used for type errors too.
555 As error messages contain the line number, and we don't want changes to
556 this file to change the reported numbers, we copy the code into a
557 separate file first, then run from there.
560 @for t in $(oceani_failing_tests); do \
561 echo -n "Test $$t ... "; \
562 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
563 echo '``````' > .tmp.code; \
564 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
565 ./oceani .tmp.code > .tmp.have 2>&1; \
566 if ! cmp -s .tmp.want .tmp.have; then \
567 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
569 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
573 oceani_failing_tests := syn1
574 oceani_failing_tests += tokerr
579 if then else while do
582 .tmp.code:3:11: error: unhandled parse error: then
583 oceani: no program found.
587 a := 1i // imaginary numbers aren't understood
588 b:[2i]number // array sizes are handled separately
589 c:[3.14159]Boolean // array sizes must be integers
590 d:[1_000_000_000_000]number // they mustn't be huge
591 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
594 This is a multiline string
595 With an unsupportable suffix
599 yy:[unknowable]number
601 zz:[zzsize]string // size must be constant, use ::=
603 // These numbers should be bad in all contexts: FIXME
606 ###### output: tokerr
607 .tmp.code:3:13: error: unsupported number suffix: 1i
608 .tmp.code:4:11: error: unsupported number suffix: 2i
609 .tmp.code:5:11: error: array size must be an integer: 3.14159
610 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
611 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
612 .tmp.code:9:17: error: unsupported string suffix: """
613 This is a multiline string
614 With an unsupportable suffix
616 .tmp.code:14:11: error: undefined type: unknown
617 .tmp.code:15:12: error: name undeclared: unknowable
618 .tmp.code:17:12: error: array size must be a constant: zzsize
619 .tmp.code:20:12: error: unrecognised number: 00123
621 ## Tests for type errors
623 Type error don't cause parsing to abort, so we can fit many in the
624 one test program. Some type errors are found during the parse, others
625 during type analysis which doesn't run if parsing failed. So we cannot
626 fit everything in one.
628 These programs were generated by looking for the
629 various places that `type_err()` are called.
632 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
634 ###### test: type_err1
637 print "hello" ++ 5, 5 ++ "hello"
642 if 3 * 4 and not True: print "Weird"
644 ###### output: type_err1
645 .tmp.code:3:25: error: expected string found number
646 .tmp.code:3:28: error: expected string found number
647 .tmp.code:6:8: error: Cannot assign to a constant: b
648 .tmp.code:5:8: info: name was defined as a constant here
649 .tmp.code:6:8: error: Cannot assign to a constant: b
650 .tmp.code:5:8: info: name was defined as a constant here
651 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
652 oceani: type error in program - not running.
654 ###### test: type_err2
664 ###### output: type_err2
665 .tmp.code:4:8: error: variable 'a' redeclared
666 .tmp.code:3:8: info: this is where 'a' was first declared
667 .tmp.code:5:8: error: variable 'a' redeclared
668 .tmp.code:3:8: info: this is where 'a' was first declared
669 .tmp.code:6:8: error: variable 'a' redeclared
670 .tmp.code:3:8: info: this is where 'a' was first declared
671 .tmp.code:7:8: error: variable 'a' redeclared
672 .tmp.code:3:8: info: this is where 'a' was first declared
673 .tmp.code:8:8: Variable declared with no type or value: c
675 ###### test: type_err3
684 c = "hello" ++ (True and False)
686 print 45 + ( "Hello" ++ "there")
696 case "Hello": print "Hello"
698 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
710 ###### output: type_err3
711 .tmp.code:8:12: error: expected number but variable 'c' is string
712 .tmp.code:7:8: info: this is where 'c' was set to string
713 .tmp.code:8:12: error: Arithmetic returns number but string expected
714 .tmp.code:7:8: info: variable 'c' was set as string here.
715 .tmp.code:9:24: error: Boolean operation found where string expected
716 .tmp.code:10:12: error: Comparison returns Boolean but string expected
717 .tmp.code:7:8: info: variable 'c' was set as string here.
718 .tmp.code:11:21: error: Concat returns string but number expected
719 .tmp.code:12:8: error: string cannot be indexed
720 .tmp.code:12:8: error: string cannot be indexed
721 .tmp.code:21:13: error: expected number found string
722 .tmp.code:17:16: error: expected number, found string
723 .tmp.code:24:8: error: cannot assign value of type [5]number
724 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
725 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
726 .tmp.code:25:8: error: cannot assign value of type [5]number
727 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
728 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
729 .tmp.code:26:8: error: cannot assign value of type [5]number
730 .tmp.code:27:16: error: expected number found string
731 .tmp.code:28:16: error: expected string found Boolean
732 .tmp.code:29:12: error: have number but need string
733 .tmp.code:7:8: info: variable 'c' was set as string here.
734 .tmp.code:32:8: error: field reference attempted on label, not a struct
735 .tmp.code:32:16: error: expected none found number
736 .tmp.code:33:14: error: field reference attempted on string, not a struct
737 oceani: type error in program - not running.
739 ###### test: type_err4
744 ###### output: type_err4
745 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
746 .tmp.code:3:14: info: this is where 'b' was set to label
747 .tmp.code:3:16: error: expected label found number
748 .tmp.code:3:14: info: variable 'b' was set as label here.
749 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
750 .tmp.code:3:14: info: this is where 'b' was set to label
751 oceani: type error in program - not running.
754 oceani_failing_tests += type_err_const type_err_const1
756 ###### test: type_err_const
759 bar ::= "string" + 56
766 ###### output: type_err_const
767 .tmp.code:4:16: error: expected number found string
768 .tmp.code:6:8: error: name already declared: bar
769 .tmp.code:4:8: info: this is where 'bar' was first declared
770 .tmp.code:8:8: error: variable 'foo' redeclared
771 .tmp.code:3:8: info: this is where 'foo' was first declared
773 ###### test: type_err_const1
781 ###### output: type_err_const1
782 .tmp.code:3:12: error: unhandled parse error: :
783 oceani: no program found.
786 ## Test erroneous command line args
788 To improve coverage, we want to test correct handling of strange command
789 line arguments. These tests won't use code, so the exiting test types
790 won't work. So we need to be able to explicitly give the command line,
791 and the expected output, and have that tested and the coverage assessed.
792 Rather than having to spell out the whole command name, just give "cmd",
793 and discard that. Requiring but discarding the command make an empty
794 command list possible.
797 @for t in $(oceani_special_tests); do \
798 echo -n "Test $$t ... ";\
799 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
800 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
801 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
802 if ! cmp -s .tmp.want .tmp.have; then \
803 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
805 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
810 oceani_special_tests += "cmd"
811 oceani_special_tests += "cmd,-zyx"
812 oceani_special_tests += "cmd,nofile"
813 oceani_special_tests += "cmd,/dev/null"
814 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
817 oceani: no input file given
819 ###### output: cmd,-zyx
820 ./oceani: invalid option -- 'z'
821 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
823 ###### output: cmd,nofile
824 oceani: cannot open nofile
826 ###### output: cmd,/dev/null
827 oceani: could not find any code in /dev/null
829 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
830 oceani: cannot find section toast:nothing