1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @rm -rf coverage; mkdir -p coverage
43 @for T in $(oceani_tests); do \
44 echo -n "Test $$T ... "; \
45 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
46 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
47 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
48 if ! cmp -s .tmp.want .tmp.have; then \
49 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
50 echo -n "passed ... "; \
51 if ! valgrind ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
52 > /dev/null 2> .tmp.valg; then \
53 echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
54 echo -n "valgrind passed ... "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo "Printing Failed"; diff -u .tmp.code1 .tmp.code2; exit1 ; fi ; \
60 echo "Printing passed"; \
61 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
62 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
67 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
68 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
69 @ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
70 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
71 if (ran < (ran + skip) *0.90) exit(1) }' \
72 coverage/oceani.mdc.gcov
75 coverage_oceani: oceani.c
76 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
78 ## Values and variables
80 The first test stores values in variables and performs various
84 oceani_tests += "valvar"
90 a := 23; b:=12 ; b1 := -b
91 print a, b, a+b, a-b, a*b, a/b, a%b
92 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
93 print +a, +b, +b1, -a, -b, -b1
95 print x and y, x or y, x and x, y or y, x and not x, x < y
97 c ::= "This is a string"
101 aconst :: string = "unchanging"
103 ###### output: valvar
105 23 12 35 11 276 1.91667 11
106 False False True True False False True
108 False True True False False False
109 This is a string field theory This is a string field theory
111 Next we change the value of variables
114 oceani_tests += "setvar"
121 a = (a + a) * (a + a)
125 ###### output: setvar
128 ## Conditions and Loops
130 Now we need to test if/else and some different loops
133 oceani_tests += cond_loop
135 ###### test: cond_loop
143 for b:=1; then b=b+b; while b < 100:
146 // Newtons method for square root of 2
152 current := guess * guess
153 use +(current - target) > 0.000000001
155 guess = (guess + (target / guess) ) / 2
158 print "error is ", target - guess * guess
160 for j:=0; then j = j+3 ; while j < 10:
161 if j != 0 and then 20 / j > 3:
162 print "20 /", j," =", 20 / j
164 print "I won't calculate 20 /", j
165 pi ::= 3.1415926535897
166 if 355/113 == pi or else +(pi - 355/113) < 0.001:
169 ###### output: cond_loop
176 error is -4.51095e-12
177 I won't calculate 20 / 0
180 I won't calculate 20 / 9
185 The demonstration code presented in the interpreted is suitable for the test suite.
186 Here I break it into two parts, keeping the array code separate.
189 oceani_tests += "sayhello,55,33"
190 oceani_tests += "sayhello,12,60"
192 ###### test: sayhello
195 print "Hello World, what lovely oceans you have!"
196 /* When a variable is defined in both branches of an 'if',
197 * and used afterwards, the variables are merged.
203 print "Is", A, "bigger than", B,"? ", bigger
204 /* If a variable is not used after the 'if', no
205 * merge happens, so types can be different
208 double:string = "yes"
209 print A, "is more than twice", B, "?", double
212 print "double", B, "is", double
223 print "GCD of", A, "and", B,"is", a
225 print a, "is not positive, cannot calculate GCD"
227 print b, "is not positive, cannot calculate GCD"
232 print "Fibonacci:", f1,f2,
241 /* Binary search... */
258 print "Yay, I found", target
260 print "Closest I found was", mid
262 ###### output: sayhello,55,33
263 Hello World, what lovely oceans you have!
264 Is 55 bigger than 33 ? yes
266 GCD of 55 and 33 is 11
267 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
268 Closest I found was 77.3438
270 ###### output: sayhello,12,60
271 Hello World, what lovely oceans you have!
272 Is 12 bigger than 60 ? no
274 GCD of 12 and 60 is 12
275 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
276 Closest I found was 77.3438
279 oceani_tests += "insert_sort"
280 ###### test: insert_sort
285 for i:=1; then i = i + 1; while i < size:
286 n := list[i-1] * list[i-1]
287 list[i] = (n / 100) % 10000
290 for i:=0; then i = i + 1; while i < size:
291 print "list[",i,"]=",list[i]
293 for i := 1; then i=i+1; while i < size:
294 for j:=i-1; then j=j-1; while j >= 0:
295 if list[j] > list[j+1]:
300 for i:=0; then i = i + 1; while i < size:
301 print "list[",i,"]=",list[i]
303 ###### output: insert_sort
419 We already have some array tests, but this is where we put other
420 ad-hoc things array related.
423 oceani_tests += arrays
431 bools[3] = strings[1] == "Hello"
432 bools[1] = strings[2] <= "there"
434 for i:=0; then i=i+1; while i<5:
438 ###### output: arrays
439 False True False False False
441 ## Test code with syntax errors
443 Syntax errors aren't handled well yet - the result is almost always a
444 single message about the first error. So this section will be fairly
445 thin until we add proper parsing recovery in the face of common errors.
447 A special case of syntax errors is token errors, when a token is only
448 accepted because the parser doesn't know quite enough to reject it.
449 There are handled better as they are quite local, so a single test
450 program can trigger most of the possible errors.
452 To handle erronous code, we need a different set of tests, as we need to
453 capture `stderr`. The same test code will be used for type errors too.
454 As error messages contain the line number, and we don't want changes to
455 this file to change the reported numbers, we copy the code into a
456 separate file first, then run from there.
459 @for t in $(oceani_failing_tests); do \
460 echo -n "Test $$t ... "; \
461 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
462 echo '``````' > .tmp.code; \
463 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
464 ./oceani .tmp.code > .tmp.have 2>&1; \
465 if ! cmp -s .tmp.want .tmp.have; then \
466 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
468 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
472 oceani_failing_tests := syn1
473 oceani_failing_tests += tokerr
478 if then else while do
481 .tmp.code:3:11: error: unhandled parse error: then
485 a := 1i // imaginary numbers aren't understood
486 b:[2i]number // array sizes are handled separately
487 c:[3.14159]Boolean // array sizes must be integers
488 d:[1_000_000_000_000]number // they mustn't be huge
489 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
492 This is a multiline string
493 With an unsupportable suffix
497 yy:[unknowable]number
499 zz:[zzsize]string // size must be constant, use ::=
501 // These numbers should be bad in all contexts: FIXME
504 ###### output: tokerr
505 .tmp.code:3:13: error: unsupported number suffix: 1i
506 .tmp.code:4:11: error: unsupported number suffix: 2i
507 .tmp.code:5:11: error: array size must be an integer: 3.14159
508 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
509 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
510 .tmp.code:9:17: error: unsupported string suffix: """
511 This is a multiline string
512 With an unsupportable suffix
514 .tmp.code:14:11: error: undefined type: unknown
515 .tmp.code:15:12: error: name undeclared: unknowable
516 .tmp.code:17:12: error: array size must be a constant: zzsize
517 .tmp.code:20:12: error: unrecognised number: 00123
519 ## Tests for type errors
521 Type error don't cause parsing to abort, so we can fit many in the
522 one test program. Some type errors are found during the parse, others
523 during type analysis which doesn't run if parsing failed. So we cannot
524 fit everything in one.
526 These programs were generated by looking for the
527 various places that `type_err()` are called.
530 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
532 ###### test: type_err1
535 print "hello" ++ 5, 5 ++ "hello"
540 if 3 * 4 and not True: print "Weird"
542 ###### output: type_err1
543 .tmp.code:3:25: error: expected string found number
544 .tmp.code:3:28: error: expected string found number
545 .tmp.code:6:8: error: Cannot assign to a constant: b
546 .tmp.code:5:8: info: name was defined as a constant here
547 .tmp.code:6:8: error: Cannot assign to a constant: b
548 .tmp.code:5:8: info: name was defined as a constant here
549 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
550 oceani: type error in program - not running.
552 ###### test: type_err2
562 ###### output: type_err2
563 .tmp.code:4:8: error: variable 'a' redeclared
564 .tmp.code:3:8: info: this is where 'a' was first declared
565 .tmp.code:5:8: error: variable 'a' redeclared
566 .tmp.code:3:8: info: this is where 'a' was first declared
567 .tmp.code:6:8: error: variable 'a' redeclared
568 .tmp.code:3:8: info: this is where 'a' was first declared
569 .tmp.code:7:8: error: variable 'a' redeclared
570 .tmp.code:3:8: info: this is where 'a' was first declared
571 .tmp.code:8:8: Variable declared with no type or value: c
573 ###### test: type_err3
578 c = "hello" ++ (True and False)
580 print 45 + ( "Hello" ++ "there")
590 case "Hello": print "Hello"
592 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
600 ###### output: type_err3
601 .tmp.code:4:12: error: expected number but variable 'c' is string
602 .tmp.code:3:8: info: this is where 'c' was set to string
603 .tmp.code:4:12: error: Arithmetic returns number but string expected
604 .tmp.code:3:8: info: variable 'c' was set as string here.
605 .tmp.code:5:24: error: Boolean operation found where string expected
606 .tmp.code:6:12: error: Comparison returns Boolean but string expected
607 .tmp.code:3:8: info: variable 'c' was set as string here.
608 .tmp.code:7:21: error: Concat returns string but number expected
609 .tmp.code:8:8: error: string cannot be indexed
610 .tmp.code:8:8: error: string cannot be indexed
611 .tmp.code:17:13: error: expected number found string
612 .tmp.code:13:16: error: expected number, found string
613 .tmp.code:20:8: error: cannot assign value of type [5]number
614 .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
615 .tmp.code:19:36: info: this is where 'a3' was set to [10]number
616 .tmp.code:21:8: error: cannot assign value of type [5]number
617 .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
618 .tmp.code:19:51: info: this is where 'a4' was set to [5]string
619 .tmp.code:22:8: error: cannot assign value of type [5]number
620 .tmp.code:23:16: error: expected number found string
621 .tmp.code:24:16: error: expected string found Boolean
622 .tmp.code:25:12: error: have number but need string
623 .tmp.code:3:8: info: variable 'c' was set as string here.
624 oceani: type error in program - not running.
626 ###### test: type_err4
631 ###### output: type_err4
632 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
633 .tmp.code:3:14: info: this is where 'b' was set to label
634 .tmp.code:3:16: error: expected label found number
635 .tmp.code:3:14: info: variable 'b' was set as label here.
636 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
637 .tmp.code:3:14: info: this is where 'b' was set to label
638 oceani: type error in program - not running.
640 ## Test erroneous command line args
642 To improve coverage, we want to test correct handling of strange command
643 line arguments. These tests won't use code, so the exiting test types
644 won't work. So we need to be able to explicitly give the command line,
645 and the expected output, and have that tested and the coverage assessed.
646 Rather than having to spell out the whole command name, just give "cmd",
647 and discard that. Requiring but discarding the command make an empty
648 command list possible.
651 @for t in $(oceani_special_tests); do \
652 echo -n "Test $$t ... ";\
653 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
654 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
655 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
656 if ! cmp -s .tmp.want .tmp.have; then \
657 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
659 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
664 oceani_special_tests += "cmd"
665 oceani_special_tests += "cmd,-zyx"
666 oceani_special_tests += "cmd,nofile"
667 oceani_special_tests += "cmd,/dev/null"
668 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
671 oceani: no input file given
673 ###### output: cmd,-zyx
674 ./oceani: invalid option -- 'z'
675 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
677 ###### output: cmd,nofile
678 oceani: cannot open nofile
680 ###### output: cmd,/dev/null
681 oceani: could not find any code in /dev/null
683 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
684 oceani: cannot find section toast:nothing