1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
42 { echo "Grammar container conflicts, please review" ; exit 1; }
43 @rm -rf coverage; mkdir -p coverage
45 @for T in $(oceani_tests); do \
46 echo -n "Test $$T.. "; \
47 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
48 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
49 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
50 if ! cmp -s .tmp.want .tmp.have; then \
51 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
52 echo -n "printing.. "; \
53 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
54 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
55 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
56 if ! cmp -s .tmp.code1 .tmp.code2; then \
57 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
58 echo -n "exec-after-print.. "; \
59 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
60 if ! cmp -s .tmp.want .tmp.have; then \
61 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
63 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
64 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
69 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
70 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
71 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
72 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
73 if (ran < (ran + skip) *0.94) exit(1) }' \
74 coverage/oceani.mdc.gcov
77 @for T in $(oceani_tests); do \
78 echo -n "Valgrind $$T.. "; \
79 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
80 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
81 > /dev/null 2>&1 ; then \
82 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
83 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
84 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
85 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
86 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
90 coverage_oceani: oceani.c
91 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
93 ## Values and variables
95 The first test stores values in variables and performs various
99 oceani_tests += "valvar"
105 a := 23; b:=12 ; b1 := -b
106 print a, b, a+b, a-b, a*b, a/b, a%b
107 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
108 print +a, +b, +b1, -a, -b, -b1
109 x := True; y := False
110 print x and y, x or y, x and x, y or y, x and not x, x < y
112 c ::= "This is a string"
113 d ::= " field theory"
116 aconst :: string = "unchanging"
125 ###### output: valvar
127 23 12 35 11 276 1.91667 11
128 False False True True False False True
130 False True True False False False
131 This is a string field theory This is a string field theory
134 Next we change the value of variables
137 oceani_tests += "setvar"
144 a = (a + a) * (a + a)
148 ###### output: setvar
154 oceani_tests += "consts"
159 four ::= 2 + 2 ; five ::= 10/2
160 const pie ::= "I like Pie";
161 cake ::= "The cake is"
165 print "Hello World, what lovely oceans you have!"
166 print "are there", five, "?"
167 print pi, pie, "but", cake
169 ###### output: consts
170 Hello World, what lovely oceans you have!
172 3.14159 I like Pie but The cake is a lie
174 Test merging of variables from multiple cases
177 oceani_tests += varmerge
179 ###### test: varmerge
182 for i:=0; then i=i+1; while i < 5:
192 ###### output: varmerge
193 zero , one , two , three , many ,
195 ## Conditions and Loops
197 Now we need to test if/else and some different loops
200 oceani_tests += cond_loop
202 ###### test: cond_loop
210 for b:=1; then b=b+b; while b < 100:
213 // Newtons method for square root of 2
219 current := guess * guess
220 use +(current - target) > 0.000000001
222 guess = (guess + (target / guess) ) / 2
225 print "error is ", target - guess * guess
227 for j:=0; then j = j+3 ; while j < 10:
228 if j != 0 and then 20 / j > 3:
229 print "20 /", j," =", 20 / j
231 print "I won't calculate 20 /", j
232 pi ::= 3.1415926535897
233 if 355/113 == pi or else +(pi - 355/113) < 0.001:
235 print "lower" if 355/113 < pi else "higher"
237 ###### output: cond_loop
244 error is -4.51095e-12
245 I won't calculate 20 / 0
248 I won't calculate 20 / 9
254 The demonstration code presented in the interpreted is suitable for the test suite.
255 Here I break it into two parts, keeping the array code separate.
258 oceani_tests += "sayhello,55,33,hello,True"
259 oceani_tests += "sayhello,12,60,there,False"
261 ###### test: sayhello
263 program A B astr bbool:
264 print "Hello World, what lovely oceans you have!"
265 /* When a variable is defined in both branches of an 'if',
266 * and used afterwards, the variables are merged.
272 print "Is", A, "bigger than", B,"? ", bigger
273 /* If a variable is not used after the 'if', no
274 * merge happens, so types can be different
277 double:string = "yes"
278 print A, "is more than twice", B, "?", double
281 print "double", B, "is", double
292 print "GCD of", A, "and", B,"is", a
294 print a, "is not positive, cannot calculate GCD"
296 print b, "is not positive, cannot calculate GCD"
301 print "Fibonacci:", f1,f2,
311 print astr ++ " was the str"
313 print "I found the str over " ++ astr
315 /* Binary search... */
332 print "Yay, I found", target
334 print "Closest I found was", mid
336 ###### output: sayhello,55,33,hello,True
337 Hello World, what lovely oceans you have!
338 Is 55 bigger than 33 ? yes
340 GCD of 55 and 33 is 11
341 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
343 Closest I found was 77.3438
345 ###### output: sayhello,12,60,there,False
346 Hello World, what lovely oceans you have!
347 Is 12 bigger than 60 ? no
349 GCD of 12 and 60 is 12
350 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
351 I found the str over there
352 Closest I found was 77.3438
355 oceani_tests += "insert_sort"
356 ###### test: insert_sort
361 for i:=1; then i = i + 1; while i < size:
362 n := list[i-1] * list[i-1]
363 list[i] = (n / 100) % 10000
366 for i:=0; then i = i + 1; while i < size:
367 print "list[",i,"]=",list[i]
369 for i := 1; then i=i+1; while i < size:
370 for j:=i-1; then j=j-1; while j >= 0:
371 if list[j] > list[j+1]:
376 for i:=0; then i = i + 1; while i < size:
377 print "list[",i,"]=",list[i]
379 ###### output: insert_sort
495 We already have some array tests, but this is where we put other
496 ad-hoc things array related.
499 oceani_tests += arrays
507 bools[3] = strings[1] == "Hello"
508 bools[1] = strings[2] <= "there"
510 for i:=0; then i=i+1; while i<5:
514 ###### output: arrays
515 False True False False False
519 Time to test if structure declarations and accesses work correctly.
522 oceani_tests += structs
531 struct baz { a:number; b:Boolean; }
536 for i:=0; then i=i+1; while i < 4:
544 info[i].size[0] = i*i
545 info[i].active = nm == "jane"
547 for i:=0; then i=i+1; while i < 4:
548 print info[i].name, info[i].active, info[i].size[0]
550 ###### output: structs
557 ## Test code with syntax errors
559 Syntax errors aren't handled well yet - the result is almost always a
560 single message about the first error. So this section will be fairly
561 thin until we add proper parsing recovery in the face of common errors.
563 A special case of syntax errors is token errors, when a token is only
564 accepted because the parser doesn't know quite enough to reject it.
565 There are handled better as they are quite local, so a single test
566 program can trigger most of the possible errors.
568 To handle erronous code, we need a different set of tests, as we need to
569 capture `stderr`. The same test code will be used for type errors too.
570 As error messages contain the line number, and we don't want changes to
571 this file to change the reported numbers, we copy the code into a
572 separate file first, then run from there.
575 @for t in $(oceani_failing_tests); do \
576 echo -n "Test $$t ... "; \
577 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
578 echo '``````' > .tmp.code; \
579 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
580 ./oceani .tmp.code > .tmp.have 2>&1; \
581 if ! cmp -s .tmp.want .tmp.have; then \
582 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
584 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
588 oceani_failing_tests := syn1
589 oceani_failing_tests += tokerr
594 if then else while do
597 .tmp.code:3:11: error: unhandled parse error: then
598 oceani: no program found.
602 a := 1i // imaginary numbers aren't understood
603 b:[2i]number // array sizes are handled separately
604 c:[3.14159]Boolean // array sizes must be integers
605 d:[1_000_000_000_000]number // they mustn't be huge
606 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
609 This is a multiline string
610 With an unsupportable suffix
614 yy:[unknowable]number
616 zz:[zzsize]string // size must be constant, use ::=
618 // These numbers should be bad in all contexts: FIXME
621 ###### output: tokerr
622 .tmp.code:3:13: error: unsupported number suffix: 1i
623 .tmp.code:4:11: error: unsupported number suffix: 2i
624 .tmp.code:5:11: error: array size must be an integer: 3.14159
625 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
626 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
627 .tmp.code:9:17: error: unsupported string suffix: """
628 This is a multiline string
629 With an unsupportable suffix
631 .tmp.code:14:11: error: undefined type: unknown
632 .tmp.code:15:12: error: name undeclared: unknowable
633 .tmp.code:17:12: error: array size must be a constant: zzsize
634 .tmp.code:20:12: error: unrecognised number: 00123
636 ## Tests for type errors
638 Type error don't cause parsing to abort, so we can fit many in the
639 one test program. Some type errors are found during the parse, others
640 during type analysis which doesn't run if parsing failed. So we cannot
641 fit everything in one.
643 These programs were generated by looking for the
644 various places that `type_err()` are called.
647 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
649 ###### test: type_err1
652 print "hello" ++ 5, 5 ++ "hello"
657 if 3 * 4 and not True: print "Weird"
659 ###### output: type_err1
660 .tmp.code:3:25: error: expected string found number
661 .tmp.code:3:28: error: expected string found number
662 .tmp.code:6:8: error: Cannot assign to a constant: b
663 .tmp.code:5:8: info: name was defined as a constant here
664 .tmp.code:6:8: error: Cannot assign to a constant: b
665 .tmp.code:5:8: info: name was defined as a constant here
666 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
667 oceani: type error in program - not running.
669 ###### test: type_err2
679 ###### output: type_err2
680 .tmp.code:4:8: error: variable 'a' redeclared
681 .tmp.code:3:8: info: this is where 'a' was first declared
682 .tmp.code:5:8: error: variable 'a' redeclared
683 .tmp.code:3:8: info: this is where 'a' was first declared
684 .tmp.code:6:8: error: variable 'a' redeclared
685 .tmp.code:3:8: info: this is where 'a' was first declared
686 .tmp.code:7:8: error: variable 'a' redeclared
687 .tmp.code:3:8: info: this is where 'a' was first declared
688 .tmp.code:8:8: Variable declared with no type or value: c
690 ###### test: type_err3
699 c = "hello" ++ (True and False)
701 print 45 + ( "Hello" ++ "there")
711 case "Hello": print "Hello"
713 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
725 ###### output: type_err3
726 .tmp.code:8:12: error: expected number but variable 'c' is string
727 .tmp.code:7:8: info: this is where 'c' was set to string
728 .tmp.code:8:12: error: Arithmetic returns number but string expected
729 .tmp.code:7:8: info: variable 'c' was set as string here.
730 .tmp.code:9:24: error: Boolean operation found where string expected
731 .tmp.code:10:12: error: Comparison returns Boolean but string expected
732 .tmp.code:7:8: info: variable 'c' was set as string here.
733 .tmp.code:11:21: error: Concat returns string but number expected
734 .tmp.code:12:8: error: string cannot be indexed
735 .tmp.code:12:8: error: string cannot be indexed
736 .tmp.code:21:13: error: expected number found string
737 .tmp.code:17:16: error: expected number, found string
738 .tmp.code:24:8: error: cannot assign value of type [5]number
739 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
740 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
741 .tmp.code:25:8: error: cannot assign value of type [5]number
742 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
743 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
744 .tmp.code:26:8: error: cannot assign value of type [5]number
745 .tmp.code:27:16: error: expected number found string
746 .tmp.code:28:16: error: expected string found Boolean
747 .tmp.code:29:12: error: have number but need string
748 .tmp.code:7:8: info: variable 'c' was set as string here.
749 .tmp.code:32:8: error: field reference attempted on label, not a struct
750 .tmp.code:32:16: error: expected none found number
751 .tmp.code:33:14: error: field reference attempted on string, not a struct
752 oceani: type error in program - not running.
754 ###### test: type_err4
759 ###### output: type_err4
760 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
761 .tmp.code:3:14: info: this is where 'b' was set to label
762 .tmp.code:3:16: error: expected label found number
763 .tmp.code:3:14: info: variable 'b' was set as label here.
764 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
765 .tmp.code:3:14: info: this is where 'b' was set to label
766 oceani: type error in program - not running.
769 oceani_failing_tests += type_err_const type_err_const1
771 ###### test: type_err_const
774 bar ::= "string" + 56
781 ###### output: type_err_const
782 .tmp.code:4:16: error: expected number found string
783 .tmp.code:6:8: error: name already declared: bar
784 .tmp.code:4:8: info: this is where 'bar' was first declared
785 .tmp.code:8:8: error: variable 'foo' redeclared
786 .tmp.code:3:8: info: this is where 'foo' was first declared
788 ###### test: type_err_const1
796 ###### output: type_err_const1
797 .tmp.code:3:12: error: unhandled parse error: :
798 oceani: no program found.
801 ## Test erroneous command line args
803 To improve coverage, we want to test correct handling of strange command
804 line arguments. These tests won't use code, so the exiting test types
805 won't work. So we need to be able to explicitly give the command line,
806 and the expected output, and have that tested and the coverage assessed.
807 Rather than having to spell out the whole command name, just give "cmd",
808 and discard that. Requiring but discarding the command make an empty
809 command list possible.
812 @for t in $(oceani_special_tests); do \
813 echo -n "Test $$t ... ";\
814 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
815 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
816 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
817 if ! cmp -s .tmp.want .tmp.have; then \
818 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
820 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
825 oceani_special_tests += "cmd"
826 oceani_special_tests += "cmd,-zyx"
827 oceani_special_tests += "cmd,nofile"
828 oceani_special_tests += "cmd,/dev/null"
829 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
832 oceani: no input file given
834 ###### output: cmd,-zyx
835 ./oceani: invalid option -- 'z'
836 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
838 ###### output: cmd,nofile
839 oceani: cannot open nofile
841 ###### output: cmd,/dev/null
842 oceani: could not find any code in /dev/null
844 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
845 oceani: cannot find section toast:nothing