1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @echo -n Checking grammar ...
42 @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
43 { echo "Grammar contains conflicts, please review" ; exit 1; }
45 @rm -rf coverage; mkdir -p coverage
47 @for T in $(oceani_tests); do \
48 echo -n "Test $$T.. "; \
49 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
50 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
51 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
52 if ! cmp -s .tmp.want .tmp.have; then \
53 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
54 echo -n "printing.. "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
60 echo -n "extra-newlines.. "; \
62 ' .tmp.code1 > .tmp.code1a; \
63 echo '``````' > .tmp.code2a ;\
64 ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
65 if ! cmp -s .tmp.code1 .tmp.code2a; then \
66 echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
67 echo -n "exec-after-print.. "; \
68 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
69 if ! cmp -s .tmp.want .tmp.have; then \
70 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
72 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
73 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
78 @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
79 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
80 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
81 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
82 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
83 if (ran < (ran + skip) *0.94) exit(1) }' \
84 coverage/oceani.mdc.gcov
87 @for T in $(oceani_tests); do \
88 echo -n "Valgrind $$T.. "; \
89 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
90 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
91 > /dev/null 2>&1 ; then \
92 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
93 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
94 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
95 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
96 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
100 coverage_oceani: oceani.c
101 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
103 ## Values and variables
105 The first test stores values in variables and performs various
106 calculations on them.
109 oceani_tests += "valvar"
114 a := 23; b:=12 ; b1 := -b
115 print a, b, a+b, a-b, a*b, a/b, a%b
116 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
117 print +a, +b, +b1, -a, -b, -b1
118 x := True; y := False
119 print x and y, x or y, x and x, y or y, x and not x, x < y
121 c ::= "This is a string"
122 d ::= " field theory"
125 aconst :: string = "unchanging"
134 ###### output: valvar
136 23 12 35 11 276 1.91667 11
137 False False True True False False True
139 False True True False False False
140 This is a string field theory This is a string field theory
143 Next we change the value of variables
146 oceani_tests += "setvar"
153 a = (a + a) * (a + a)
157 ###### output: setvar
163 oceani_tests += "consts"
168 four ::= 2 + 2 ; five ::= 10/2
169 const pie ::= "I like Pie";
170 cake ::= "The cake is"
174 print "Hello World, what lovely oceans you have!"
175 print "are there", five, "?"
176 print pi, pie, "but", cake
178 ###### output: consts
179 Hello World, what lovely oceans you have!
181 3.14159 I like Pie but The cake is a lie
183 Test merging of variables from multiple cases
186 oceani_tests += varmerge
188 ###### test: varmerge
191 for i:=0; then i=i+1; while i < 5:
201 ###### output: varmerge
202 zero , one , two , three , many ,
204 ## Conditions and Loops
206 Now we need to test if/else and some different loops
209 oceani_tests += cond_loop
211 ###### test: cond_loop
219 for b:=1; then b=b+b; while b < 100:
222 // Newtons method for square root of 2
228 current := guess * guess
229 use +(current - target) > 0.000000001
231 guess = (guess + (target / guess) ) / 2
234 print "error is ", target - guess * guess
236 for j:=0; then j = j+3 ; while j < 10:
237 if j != 0 and then 20 / j > 3:
238 print "20 /", j," =", 20 / j
240 print "I won't calculate 20 /", j
241 pi ::= 3.1415926535897
242 if 355/113 == pi or else +(pi - 355/113) < 0.001:
244 print "lower" if 355/113 < pi else "higher"
246 if pi > 3 then print "pi exceeds three"; else print "need more pie"
247 if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
248 for { i := 0; sum := 0 }
254 print "sum 1..10 is", sum
256 ###### output: cond_loop
263 error is -4.51095e-12
264 I won't calculate 20 / 0
267 I won't calculate 20 / 9
276 The demonstration code presented in the interpreted is suitable for the test suite.
277 Here I break it into two parts, keeping the array code separate.
280 oceani_tests += "sayhello,55,33,hello,True"
281 oceani_tests += "sayhello,12,60,there,False"
283 ###### test: sayhello
285 program As Bs astr bbools:
287 bbool := bbools == "True"
288 print "Hello World, what lovely oceans you have!"
289 /* When a variable is defined in both branches of an 'if',
290 * and used afterwards, the variables are merged.
296 print "Is", A, "bigger than", B,"? ", bigger
297 /* If a variable is not used after the 'if', no
298 * merge happens, so types can be different
301 double:string = "yes"
302 print A, "is more than twice", B, "?", double
305 print "double", B, "is", double
316 print "GCD of", A, "and", B,"is", a
318 print a, "is not positive, cannot calculate GCD"
320 print b, "is not positive, cannot calculate GCD"
325 print "Fibonacci:", f1,f2,
335 print astr ++ " was the str"
337 print "I found the str over " ++ astr
339 /* Binary search... */
356 print "Yay, I found", target
358 print "Closest I found was", mid
360 ###### output: sayhello,55,33,hello,True
361 Hello World, what lovely oceans you have!
362 Is 55 bigger than 33 ? yes
364 GCD of 55 and 33 is 11
365 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
367 Closest I found was 77.3438
369 ###### output: sayhello,12,60,there,False
370 Hello World, what lovely oceans you have!
371 Is 12 bigger than 60 ? no
373 GCD of 12 and 60 is 12
374 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
375 I found the str over there
376 Closest I found was 77.3438
379 oceani_tests += "insert_sort"
380 ###### test: insert_sort
385 for i:=1; then i = i + 1; while i < size:
386 n := list[i-1] * list[i-1]
387 list[i] = (n / 100) % 10000
390 for i:=0; then i = i + 1; while i < size:
391 print "list[",i,"]=",list[i]
393 for i := 1; then i=i+1; while i < size:
394 for j:=i-1; then j=j-1; while j >= 0:
395 if list[j] > list[j+1]:
400 for i:=0; then i = i + 1; while i < size:
401 print "list[",i,"]=",list[i]
403 ###### output: insert_sort
519 We already have some array tests, but this is where we put other
520 ad-hoc things array related.
523 oceani_tests += arrays
531 bools[3] = strings[1] == "Hello"
532 bools[1] = strings[2] <= "there"
534 for i:=0; then i=i+1; while i<5:
538 ra[6] = 42 // mustn't crash
539 print '', bools[i], ra[j-1],
542 ###### output: arrays
543 False 0 True 1 False 4 False 9 False 16
547 Time to test if structure declarations and accesses work correctly.
550 oceani_tests += structs
558 active:Boolean = True
560 struct baz { a:number; b:Boolean; }
565 for i:=0; then i=i+1; while i < 4:
573 info[i].size[0] = i*i
575 info[i].active = False
577 for i:=0; then i=i+1; while i < 4:
578 print info[i].name, info[i].active, info[i].size[0]
580 ###### output: structs
587 ## Test code with syntax errors
589 Syntax errors aren't handled well yet - the result is almost always a
590 single message about the first error. So this section will be fairly
591 thin until we add proper parsing recovery in the face of common errors.
593 A special case of syntax errors is token errors, when a token is only
594 accepted because the parser doesn't know quite enough to reject it.
595 There are handled better as they are quite local, so a single test
596 program can trigger most of the possible errors.
598 To handle erronous code, we need a different set of tests, as we need to
599 capture `stderr`. The same test code will be used for type errors too.
600 As error messages contain the line number, and we don't want changes to
601 this file to change the reported numbers, we copy the code into a
602 separate file first, then run from there.
605 @for t in $(oceani_failing_tests); do \
606 echo -n "Test $$t ... "; \
607 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
608 echo '``````' > .tmp.code; \
609 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
610 ./oceani .tmp.code > .tmp.have 2>&1; \
611 if ! cmp -s .tmp.want .tmp.have; then \
612 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
614 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
618 oceani_failing_tests := syn1
619 oceani_failing_tests += tokerr
624 if then else while do
627 .tmp.code:3:11: Syntax error in statement: then
631 a := 1i // imaginary numbers aren't understood
632 b:[2i]number // array sizes are handled separately
633 c:[3.14159]Boolean // array sizes must be integers
634 d:[1_000_000_000_000]number // they mustn't be huge
635 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
638 This is a multiline string
639 With an unsupportable suffix
643 yy:[unknowable]number
645 zz:[zzsize]string // size must be constant, use ::=
647 // These numbers should be bad in all contexts: FIXME
650 ###### output: tokerr
651 .tmp.code:3:13: error: unsupported number suffix: 1i
652 .tmp.code:4:11: error: unsupported number suffix: 2i
653 .tmp.code:5:11: error: array size must be an integer: 3.14159
654 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
655 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
656 .tmp.code:9:17: error: unsupported string suffix: """
657 This is a multiline string
658 With an unsupportable suffix
660 .tmp.code:14:11: error: undefined type: unknown
661 .tmp.code:15:12: error: name undeclared: unknowable
662 .tmp.code:17:12: error: array size must be a constant: zzsize
663 .tmp.code:20:12: error: unrecognised number: 00123
665 ## Tests for type errors
667 Type error don't cause parsing to abort, so we can fit many in the
668 one test program. Some type errors are found during the parse, others
669 during type analysis which doesn't run if parsing failed. So we cannot
670 fit everything in one.
672 These programs were generated by looking for the
673 various places that `type_err()` are called.
676 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
678 ###### test: type_err1
681 print "hello" ++ 5, 5 ++ "hello"
686 if 3 * 4 and not True: print "Weird"
688 ###### output: type_err1
689 .tmp.code:3:25: error: expected string found number
690 .tmp.code:3:28: error: expected string found number
691 .tmp.code:6:8: error: Cannot assign to a constant: b
692 .tmp.code:5:8: info: name was defined as a constant here
693 .tmp.code:6:8: error: Cannot assign to a constant: b
694 .tmp.code:5:8: info: name was defined as a constant here
695 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
696 oceani: type error in program - not running.
698 ###### test: type_err2
708 ###### output: type_err2
709 .tmp.code:4:8: error: variable 'a' redeclared
710 .tmp.code:3:8: info: this is where 'a' was first declared
711 .tmp.code:5:8: error: variable 'a' redeclared
712 .tmp.code:3:8: info: this is where 'a' was first declared
713 .tmp.code:6:8: error: variable 'a' redeclared
714 .tmp.code:3:8: info: this is where 'a' was first declared
715 .tmp.code:7:8: error: variable 'a' redeclared
716 .tmp.code:3:8: info: this is where 'a' was first declared
717 .tmp.code:8:8: Variable declared with no type or value: c
719 ###### test: type_err3
728 c = "hello" ++ (True and False)
730 print 45 + ( "Hello" ++ "there")
740 case "Hello": print "Hello"
742 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
754 ###### output: type_err3
755 .tmp.code:8:12: error: expected number but variable 'c' is string
756 .tmp.code:7:8: info: this is where 'c' was set to string
757 .tmp.code:8:12: error: Arithmetic returns number but string expected
758 .tmp.code:7:8: info: variable 'c' was set as string here.
759 .tmp.code:9:24: error: Boolean operation found where string expected
760 .tmp.code:10:12: error: Comparison returns Boolean but string expected
761 .tmp.code:7:8: info: variable 'c' was set as string here.
762 .tmp.code:11:21: error: Concat returns string but number expected
763 .tmp.code:12:8: error: string cannot be indexed
764 .tmp.code:12:8: error: string cannot be indexed
765 .tmp.code:21:13: error: expected number found string
766 .tmp.code:17:16: error: expected number, found string
767 .tmp.code:24:8: error: cannot assign value of type [5]number
768 .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
769 .tmp.code:23:36: info: this is where 'a3' was set to [10]number
770 .tmp.code:25:8: error: cannot assign value of type [5]number
771 .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
772 .tmp.code:23:51: info: this is where 'a4' was set to [5]string
773 .tmp.code:26:8: error: cannot assign value of type [5]number
774 .tmp.code:27:16: error: expected number found string
775 .tmp.code:28:16: error: expected string found Boolean
776 .tmp.code:29:12: error: have number but need string
777 .tmp.code:7:8: info: variable 'c' was set as string here.
778 .tmp.code:32:8: error: variable used but not declared: foo
779 .tmp.code:32:8: error: field reference attempted on none, not a struct
780 .tmp.code:32:16: error: expected none found number
781 .tmp.code:33:14: error: field reference attempted on string, not a struct
782 oceani: type error in program - not running.
784 ###### test: type_err4
789 ###### output: type_err4
790 .tmp.code:3:14: error: variable used but not declared: b
791 .tmp.code:3:16: error: expected none found number
792 .tmp.code:3:14: info: variable 'b' was set as none here.
793 oceani: type error in program - not running.
796 oceani_failing_tests += type_err_const type_err_const1
798 ###### test: type_err_const
801 bar ::= "string" + 56
808 ###### output: type_err_const
809 .tmp.code:4:16: error: expected number found string
810 .tmp.code:6:8: error: name already declared: bar
811 .tmp.code:4:8: info: this is where 'bar' was first declared
812 .tmp.code:8:8: error: variable 'foo' redeclared
813 .tmp.code:3:8: info: this is where 'foo' was first declared
815 ###### test: type_err_const1
823 ###### output: type_err_const1
824 .tmp.code:3:12: Syntax error in constant: :
825 .tmp.code:4:12: Syntax error in constant: :
827 ## Test erroneous command line args
829 To improve coverage, we want to test correct handling of strange command
830 line arguments. These tests won't use code, so the exiting test types
831 won't work. So we need to be able to explicitly give the command line,
832 and the expected output, and have that tested and the coverage assessed.
833 Rather than having to spell out the whole command name, just give "cmd",
834 and discard that. Requiring but discarding the command make an empty
835 command list possible.
838 @for t in $(oceani_special_tests); do \
839 echo -n "Test $$t ... ";\
840 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
841 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
842 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
843 if ! cmp -s .tmp.want .tmp.have; then \
844 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
846 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
850 oceani_special_tests += "cmd"
851 oceani_special_tests += "cmd,-zyx"
852 oceani_special_tests += "cmd,nofile"
853 oceani_special_tests += "cmd,/dev/null"
854 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
857 oceani: no input file given
859 ###### output: cmd,-zyx
860 ./oceani: invalid option -- 'z'
861 Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn
863 ###### output: cmd,nofile
864 oceani: cannot open nofile
866 ###### output: cmd,/dev/null
867 oceani: could not find any code in /dev/null
869 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
870 oceani: cannot find section toast:nothing