1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @./parsergen --report --LR1 --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
42 { echo "Grammar container conflicts, please review" ; exit 1; }
43 @rm -rf coverage; mkdir -p coverage
45 @for T in $(oceani_tests); do \
46 echo -n "Test $$T.. "; \
47 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
48 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
49 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
50 if ! cmp -s .tmp.want .tmp.have; then \
51 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
52 echo -n "valgrind.. "; \
53 if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
54 > /dev/null 2>&1 ; then \
55 echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
56 if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
57 echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
58 if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
59 echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
60 echo -n "printing.. "; \
61 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
62 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
63 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
64 if ! cmp -s .tmp.code1 .tmp.code2; then \
65 echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
66 echo -n "exec-after-print.. "; \
67 ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
68 if ! cmp -s .tmp.want .tmp.have; then \
69 echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
71 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
72 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
77 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
78 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
79 @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
80 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
81 if (ran < (ran + skip) *0.93) exit(1) }' \
82 coverage/oceani.mdc.gcov
85 coverage_oceani: oceani.c
86 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
88 ## Values and variables
90 The first test stores values in variables and performs various
94 oceani_tests += "valvar"
100 a := 23; b:=12 ; b1 := -b
101 print a, b, a+b, a-b, a*b, a/b, a%b
102 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
103 print +a, +b, +b1, -a, -b, -b1
104 x := True; y := False
105 print x and y, x or y, x and x, y or y, x and not x, x < y
107 c ::= "This is a string"
108 d ::= " field theory"
111 aconst :: string = "unchanging"
113 ###### output: valvar
115 23 12 35 11 276 1.91667 11
116 False False True True False False True
118 False True True False False False
119 This is a string field theory This is a string field theory
121 Next we change the value of variables
124 oceani_tests += "setvar"
131 a = (a + a) * (a + a)
135 ###### output: setvar
141 oceani_tests += "consts"
146 four ::= 2 + 2 ; five ::= 10/2
147 const pie ::= "I like Pie";
148 cake ::= "The cake is"
152 print "Hello World, what lovely oceans you have!"
153 print "are there", five, "?"
154 print pi, pie, "but", cake
156 ###### output: consts
157 Hello World, what lovely oceans you have!
159 3.14159 I like Pie but The cake is a lie
161 ## Conditions and Loops
163 Now we need to test if/else and some different loops
166 oceani_tests += cond_loop
168 ###### test: cond_loop
176 for b:=1; then b=b+b; while b < 100:
179 // Newtons method for square root of 2
185 current := guess * guess
186 use +(current - target) > 0.000000001
188 guess = (guess + (target / guess) ) / 2
191 print "error is ", target - guess * guess
193 for j:=0; then j = j+3 ; while j < 10:
194 if j != 0 and then 20 / j > 3:
195 print "20 /", j," =", 20 / j
197 print "I won't calculate 20 /", j
198 pi ::= 3.1415926535897
199 if 355/113 == pi or else +(pi - 355/113) < 0.001:
201 print "lower" if 355/113 < pi else "higher"
203 ###### output: cond_loop
210 error is -4.51095e-12
211 I won't calculate 20 / 0
214 I won't calculate 20 / 9
220 The demonstration code presented in the interpreted is suitable for the test suite.
221 Here I break it into two parts, keeping the array code separate.
224 oceani_tests += "sayhello,55,33"
225 oceani_tests += "sayhello,12,60"
227 ###### test: sayhello
230 print "Hello World, what lovely oceans you have!"
231 /* When a variable is defined in both branches of an 'if',
232 * and used afterwards, the variables are merged.
238 print "Is", A, "bigger than", B,"? ", bigger
239 /* If a variable is not used after the 'if', no
240 * merge happens, so types can be different
243 double:string = "yes"
244 print A, "is more than twice", B, "?", double
247 print "double", B, "is", double
258 print "GCD of", A, "and", B,"is", a
260 print a, "is not positive, cannot calculate GCD"
262 print b, "is not positive, cannot calculate GCD"
267 print "Fibonacci:", f1,f2,
276 /* Binary search... */
293 print "Yay, I found", target
295 print "Closest I found was", mid
297 ###### output: sayhello,55,33
298 Hello World, what lovely oceans you have!
299 Is 55 bigger than 33 ? yes
301 GCD of 55 and 33 is 11
302 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
303 Closest I found was 77.3438
305 ###### output: sayhello,12,60
306 Hello World, what lovely oceans you have!
307 Is 12 bigger than 60 ? no
309 GCD of 12 and 60 is 12
310 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
311 Closest I found was 77.3438
314 oceani_tests += "insert_sort"
315 ###### test: insert_sort
320 for i:=1; then i = i + 1; while i < size:
321 n := list[i-1] * list[i-1]
322 list[i] = (n / 100) % 10000
325 for i:=0; then i = i + 1; while i < size:
326 print "list[",i,"]=",list[i]
328 for i := 1; then i=i+1; while i < size:
329 for j:=i-1; then j=j-1; while j >= 0:
330 if list[j] > list[j+1]:
335 for i:=0; then i = i + 1; while i < size:
336 print "list[",i,"]=",list[i]
338 ###### output: insert_sort
454 We already have some array tests, but this is where we put other
455 ad-hoc things array related.
458 oceani_tests += arrays
466 bools[3] = strings[1] == "Hello"
467 bools[1] = strings[2] <= "there"
469 for i:=0; then i=i+1; while i<5:
473 ###### output: arrays
474 False True False False False
476 ## Test code with syntax errors
478 Syntax errors aren't handled well yet - the result is almost always a
479 single message about the first error. So this section will be fairly
480 thin until we add proper parsing recovery in the face of common errors.
482 A special case of syntax errors is token errors, when a token is only
483 accepted because the parser doesn't know quite enough to reject it.
484 There are handled better as they are quite local, so a single test
485 program can trigger most of the possible errors.
487 To handle erronous code, we need a different set of tests, as we need to
488 capture `stderr`. The same test code will be used for type errors too.
489 As error messages contain the line number, and we don't want changes to
490 this file to change the reported numbers, we copy the code into a
491 separate file first, then run from there.
494 @for t in $(oceani_failing_tests); do \
495 echo -n "Test $$t ... "; \
496 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
497 echo '``````' > .tmp.code; \
498 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
499 ./oceani .tmp.code > .tmp.have 2>&1; \
500 if ! cmp -s .tmp.want .tmp.have; then \
501 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
503 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
507 oceani_failing_tests := syn1
508 oceani_failing_tests += tokerr
513 if then else while do
516 .tmp.code:3:11: error: unhandled parse error: then
517 oceani: no program found.
521 a := 1i // imaginary numbers aren't understood
522 b:[2i]number // array sizes are handled separately
523 c:[3.14159]Boolean // array sizes must be integers
524 d:[1_000_000_000_000]number // they mustn't be huge
525 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
528 This is a multiline string
529 With an unsupportable suffix
533 yy:[unknowable]number
535 zz:[zzsize]string // size must be constant, use ::=
537 // These numbers should be bad in all contexts: FIXME
540 ###### output: tokerr
541 .tmp.code:3:13: error: unsupported number suffix: 1i
542 .tmp.code:4:11: error: unsupported number suffix: 2i
543 .tmp.code:5:11: error: array size must be an integer: 3.14159
544 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
545 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
546 .tmp.code:9:17: error: unsupported string suffix: """
547 This is a multiline string
548 With an unsupportable suffix
550 .tmp.code:14:11: error: undefined type: unknown
551 .tmp.code:15:12: error: name undeclared: unknowable
552 .tmp.code:17:12: error: array size must be a constant: zzsize
553 .tmp.code:20:12: error: unrecognised number: 00123
555 ## Tests for type errors
557 Type error don't cause parsing to abort, so we can fit many in the
558 one test program. Some type errors are found during the parse, others
559 during type analysis which doesn't run if parsing failed. So we cannot
560 fit everything in one.
562 These programs were generated by looking for the
563 various places that `type_err()` are called.
566 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
568 ###### test: type_err1
571 print "hello" ++ 5, 5 ++ "hello"
576 if 3 * 4 and not True: print "Weird"
578 ###### output: type_err1
579 .tmp.code:3:25: error: expected string found number
580 .tmp.code:3:28: error: expected string found number
581 .tmp.code:6:8: error: Cannot assign to a constant: b
582 .tmp.code:5:8: info: name was defined as a constant here
583 .tmp.code:6:8: error: Cannot assign to a constant: b
584 .tmp.code:5:8: info: name was defined as a constant here
585 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
586 oceani: type error in program - not running.
588 ###### test: type_err2
598 ###### output: type_err2
599 .tmp.code:4:8: error: variable 'a' redeclared
600 .tmp.code:3:8: info: this is where 'a' was first declared
601 .tmp.code:5:8: error: variable 'a' redeclared
602 .tmp.code:3:8: info: this is where 'a' was first declared
603 .tmp.code:6:8: error: variable 'a' redeclared
604 .tmp.code:3:8: info: this is where 'a' was first declared
605 .tmp.code:7:8: error: variable 'a' redeclared
606 .tmp.code:3:8: info: this is where 'a' was first declared
607 .tmp.code:8:8: Variable declared with no type or value: c
609 ###### test: type_err3
614 c = "hello" ++ (True and False)
616 print 45 + ( "Hello" ++ "there")
626 case "Hello": print "Hello"
628 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
636 ###### output: type_err3
637 .tmp.code:4:12: error: expected number but variable 'c' is string
638 .tmp.code:3:8: info: this is where 'c' was set to string
639 .tmp.code:4:12: error: Arithmetic returns number but string expected
640 .tmp.code:3:8: info: variable 'c' was set as string here.
641 .tmp.code:5:24: error: Boolean operation found where string expected
642 .tmp.code:6:12: error: Comparison returns Boolean but string expected
643 .tmp.code:3:8: info: variable 'c' was set as string here.
644 .tmp.code:7:21: error: Concat returns string but number expected
645 .tmp.code:8:8: error: string cannot be indexed
646 .tmp.code:8:8: error: string cannot be indexed
647 .tmp.code:17:13: error: expected number found string
648 .tmp.code:13:16: error: expected number, found string
649 .tmp.code:20:8: error: cannot assign value of type [5]number
650 .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
651 .tmp.code:19:36: info: this is where 'a3' was set to [10]number
652 .tmp.code:21:8: error: cannot assign value of type [5]number
653 .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
654 .tmp.code:19:51: info: this is where 'a4' was set to [5]string
655 .tmp.code:22:8: error: cannot assign value of type [5]number
656 .tmp.code:23:16: error: expected number found string
657 .tmp.code:24:16: error: expected string found Boolean
658 .tmp.code:25:12: error: have number but need string
659 .tmp.code:3:8: info: variable 'c' was set as string here.
660 oceani: type error in program - not running.
662 ###### test: type_err4
667 ###### output: type_err4
668 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
669 .tmp.code:3:14: info: this is where 'b' was set to label
670 .tmp.code:3:16: error: expected label found number
671 .tmp.code:3:14: info: variable 'b' was set as label here.
672 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
673 .tmp.code:3:14: info: this is where 'b' was set to label
674 oceani: type error in program - not running.
677 oceani_failing_tests += type_err_const type_err_const1
679 ###### test: type_err_const
682 bar ::= "string" + 56
689 ###### output: type_err_const
690 .tmp.code:4:16: error: expected number found string
691 .tmp.code:6:8: error: name already declared: bar
692 .tmp.code:4:8: info: this is where 'bar' was first declared
693 .tmp.code:8:8: error: variable 'foo' redeclared
694 .tmp.code:3:8: info: this is where 'foo' was first declared
696 ###### test: type_err_const1
704 ###### output: type_err_const1
705 .tmp.code:3:12: error: unhandled parse error: :
706 oceani: no program found.
709 ## Test erroneous command line args
711 To improve coverage, we want to test correct handling of strange command
712 line arguments. These tests won't use code, so the exiting test types
713 won't work. So we need to be able to explicitly give the command line,
714 and the expected output, and have that tested and the coverage assessed.
715 Rather than having to spell out the whole command name, just give "cmd",
716 and discard that. Requiring but discarding the command make an empty
717 command list possible.
720 @for t in $(oceani_special_tests); do \
721 echo -n "Test $$t ... ";\
722 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
723 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
724 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
725 if ! cmp -s .tmp.want .tmp.have; then \
726 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
728 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
733 oceani_special_tests += "cmd"
734 oceani_special_tests += "cmd,-zyx"
735 oceani_special_tests += "cmd,nofile"
736 oceani_special_tests += "cmd,/dev/null"
737 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
740 oceani: no input file given
742 ###### output: cmd,-zyx
743 ./oceani: invalid option -- 'z'
744 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
746 ###### output: cmd,nofile
747 oceani: cannot open nofile
749 ###### output: cmd,/dev/null
750 oceani: could not find any code in /dev/null
752 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
753 oceani: cannot find section toast:nothing