1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @rm -rf coverage; mkdir -p coverage
43 @for T in $(oceani_tests); do \
44 echo -n "Test $$T ... "; \
45 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
46 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
47 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
48 if ! cmp -s .tmp.want .tmp.have; then \
49 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
50 echo -n "passed ... "; \
51 if ! valgrind ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
52 > /dev/null 2> .tmp.valg; then \
53 echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
54 echo -n "valgrind passed ... "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo "Printing Failed"; diff -u .tmp.code1 .tmp.code2; exit1 ; fi ; \
60 echo "Printing passed"; \
61 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
62 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
67 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
68 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
69 @ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
70 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
71 if (ran < (ran + skip) *0.90) exit(1) }' \
72 coverage/oceani.mdc.gcov
75 coverage_oceani: oceani.c
76 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
78 ## Values and variables
80 The first test stores values in variables and performs various
84 oceani_tests += "valvar"
90 a := 23; b:=12 ; b1 := -b
91 print a, b, a+b, a-b, a*b, a/b, a%b
92 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
93 print +a, +b, +b1, -a, -b, -b1
95 print x and y, x or y, x and x, y or y, x and not x, x < y
97 c ::= "This is a string"
101 aconst :: string = "unchanging"
103 ###### output: valvar
105 23 12 35 11 276 1.91667 11
106 False False True True False False True
108 False True True False False False
109 This is a string field theory This is a string field theory
111 Next we change the value of variables
114 oceani_tests += "setvar"
121 a = (a + a) * (a + a)
125 ###### output: setvar
128 ## Conditions and Loops
130 Now we need to test if/else and some different loops
133 oceani_tests += cond_loop
135 ###### test: cond_loop
143 for b:=1; then b=b+b; while b < 100:
146 // Newtons method for square root of 2
152 current := guess * guess
153 use +(current - target) > 0.000000001
155 guess = (guess + (target / guess) ) / 2
158 print "error is ", target - guess * guess
160 for j:=0; then j = j+3 ; while j < 10:
161 if j != 0 and then 20 / j > 3:
162 print "20 /", j," =", 20 / j
164 print "I won't calculate 20 /", j
165 pi ::= 3.1415926535897
166 if 355/113 == pi or else +(pi - 355/113) < 0.001:
168 print "lower" if 355/113 < pi else "higher"
170 ###### output: cond_loop
177 error is -4.51095e-12
178 I won't calculate 20 / 0
181 I won't calculate 20 / 9
187 The demonstration code presented in the interpreted is suitable for the test suite.
188 Here I break it into two parts, keeping the array code separate.
191 oceani_tests += "sayhello,55,33"
192 oceani_tests += "sayhello,12,60"
194 ###### test: sayhello
197 print "Hello World, what lovely oceans you have!"
198 /* When a variable is defined in both branches of an 'if',
199 * and used afterwards, the variables are merged.
205 print "Is", A, "bigger than", B,"? ", bigger
206 /* If a variable is not used after the 'if', no
207 * merge happens, so types can be different
210 double:string = "yes"
211 print A, "is more than twice", B, "?", double
214 print "double", B, "is", double
225 print "GCD of", A, "and", B,"is", a
227 print a, "is not positive, cannot calculate GCD"
229 print b, "is not positive, cannot calculate GCD"
234 print "Fibonacci:", f1,f2,
243 /* Binary search... */
260 print "Yay, I found", target
262 print "Closest I found was", mid
264 ###### output: sayhello,55,33
265 Hello World, what lovely oceans you have!
266 Is 55 bigger than 33 ? yes
268 GCD of 55 and 33 is 11
269 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
270 Closest I found was 77.3438
272 ###### output: sayhello,12,60
273 Hello World, what lovely oceans you have!
274 Is 12 bigger than 60 ? no
276 GCD of 12 and 60 is 12
277 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
278 Closest I found was 77.3438
281 oceani_tests += "insert_sort"
282 ###### test: insert_sort
287 for i:=1; then i = i + 1; while i < size:
288 n := list[i-1] * list[i-1]
289 list[i] = (n / 100) % 10000
292 for i:=0; then i = i + 1; while i < size:
293 print "list[",i,"]=",list[i]
295 for i := 1; then i=i+1; while i < size:
296 for j:=i-1; then j=j-1; while j >= 0:
297 if list[j] > list[j+1]:
302 for i:=0; then i = i + 1; while i < size:
303 print "list[",i,"]=",list[i]
305 ###### output: insert_sort
421 We already have some array tests, but this is where we put other
422 ad-hoc things array related.
425 oceani_tests += arrays
433 bools[3] = strings[1] == "Hello"
434 bools[1] = strings[2] <= "there"
436 for i:=0; then i=i+1; while i<5:
440 ###### output: arrays
441 False True False False False
443 ## Test code with syntax errors
445 Syntax errors aren't handled well yet - the result is almost always a
446 single message about the first error. So this section will be fairly
447 thin until we add proper parsing recovery in the face of common errors.
449 A special case of syntax errors is token errors, when a token is only
450 accepted because the parser doesn't know quite enough to reject it.
451 There are handled better as they are quite local, so a single test
452 program can trigger most of the possible errors.
454 To handle erronous code, we need a different set of tests, as we need to
455 capture `stderr`. The same test code will be used for type errors too.
456 As error messages contain the line number, and we don't want changes to
457 this file to change the reported numbers, we copy the code into a
458 separate file first, then run from there.
461 @for t in $(oceani_failing_tests); do \
462 echo -n "Test $$t ... "; \
463 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
464 echo '``````' > .tmp.code; \
465 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
466 ./oceani .tmp.code > .tmp.have 2>&1; \
467 if ! cmp -s .tmp.want .tmp.have; then \
468 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
470 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
474 oceani_failing_tests := syn1
475 oceani_failing_tests += tokerr
480 if then else while do
483 .tmp.code:3:11: error: unhandled parse error: then
487 a := 1i // imaginary numbers aren't understood
488 b:[2i]number // array sizes are handled separately
489 c:[3.14159]Boolean // array sizes must be integers
490 d:[1_000_000_000_000]number // they mustn't be huge
491 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
494 This is a multiline string
495 With an unsupportable suffix
499 yy:[unknowable]number
501 zz:[zzsize]string // size must be constant, use ::=
503 // These numbers should be bad in all contexts: FIXME
506 ###### output: tokerr
507 .tmp.code:3:13: error: unsupported number suffix: 1i
508 .tmp.code:4:11: error: unsupported number suffix: 2i
509 .tmp.code:5:11: error: array size must be an integer: 3.14159
510 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
511 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
512 .tmp.code:9:17: error: unsupported string suffix: """
513 This is a multiline string
514 With an unsupportable suffix
516 .tmp.code:14:11: error: undefined type: unknown
517 .tmp.code:15:12: error: name undeclared: unknowable
518 .tmp.code:17:12: error: array size must be a constant: zzsize
519 .tmp.code:20:12: error: unrecognised number: 00123
521 ## Tests for type errors
523 Type error don't cause parsing to abort, so we can fit many in the
524 one test program. Some type errors are found during the parse, others
525 during type analysis which doesn't run if parsing failed. So we cannot
526 fit everything in one.
528 These programs were generated by looking for the
529 various places that `type_err()` are called.
532 oceani_failing_tests += type_err1 type_err2 type_err3 type_err4
534 ###### test: type_err1
537 print "hello" ++ 5, 5 ++ "hello"
542 if 3 * 4 and not True: print "Weird"
544 ###### output: type_err1
545 .tmp.code:3:25: error: expected string found number
546 .tmp.code:3:28: error: expected string found number
547 .tmp.code:6:8: error: Cannot assign to a constant: b
548 .tmp.code:5:8: info: name was defined as a constant here
549 .tmp.code:6:8: error: Cannot assign to a constant: b
550 .tmp.code:5:8: info: name was defined as a constant here
551 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
552 oceani: type error in program - not running.
554 ###### test: type_err2
564 ###### output: type_err2
565 .tmp.code:4:8: error: variable 'a' redeclared
566 .tmp.code:3:8: info: this is where 'a' was first declared
567 .tmp.code:5:8: error: variable 'a' redeclared
568 .tmp.code:3:8: info: this is where 'a' was first declared
569 .tmp.code:6:8: error: variable 'a' redeclared
570 .tmp.code:3:8: info: this is where 'a' was first declared
571 .tmp.code:7:8: error: variable 'a' redeclared
572 .tmp.code:3:8: info: this is where 'a' was first declared
573 .tmp.code:8:8: Variable declared with no type or value: c
575 ###### test: type_err3
580 c = "hello" ++ (True and False)
582 print 45 + ( "Hello" ++ "there")
592 case "Hello": print "Hello"
594 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
602 ###### output: type_err3
603 .tmp.code:4:12: error: expected number but variable 'c' is string
604 .tmp.code:3:8: info: this is where 'c' was set to string
605 .tmp.code:4:12: error: Arithmetic returns number but string expected
606 .tmp.code:3:8: info: variable 'c' was set as string here.
607 .tmp.code:5:24: error: Boolean operation found where string expected
608 .tmp.code:6:12: error: Comparison returns Boolean but string expected
609 .tmp.code:3:8: info: variable 'c' was set as string here.
610 .tmp.code:7:21: error: Concat returns string but number expected
611 .tmp.code:8:8: error: string cannot be indexed
612 .tmp.code:8:8: error: string cannot be indexed
613 .tmp.code:17:13: error: expected number found string
614 .tmp.code:13:16: error: expected number, found string
615 .tmp.code:20:8: error: cannot assign value of type [5]number
616 .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
617 .tmp.code:19:36: info: this is where 'a3' was set to [10]number
618 .tmp.code:21:8: error: cannot assign value of type [5]number
619 .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
620 .tmp.code:19:51: info: this is where 'a4' was set to [5]string
621 .tmp.code:22:8: error: cannot assign value of type [5]number
622 .tmp.code:23:16: error: expected number found string
623 .tmp.code:24:16: error: expected string found Boolean
624 .tmp.code:25:12: error: have number but need string
625 .tmp.code:3:8: info: variable 'c' was set as string here.
626 oceani: type error in program - not running.
628 ###### test: type_err4
633 ###### output: type_err4
634 .tmp.code:3:14: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
635 .tmp.code:3:14: info: this is where 'b' was set to label
636 .tmp.code:3:16: error: expected label found number
637 .tmp.code:3:14: info: variable 'b' was set as label here.
638 .tmp.code:4:17: error: expected *unknown*type* (labels not permitted) but variable 'b' is label
639 .tmp.code:3:14: info: this is where 'b' was set to label
640 oceani: type error in program - not running.
642 ## Test erroneous command line args
644 To improve coverage, we want to test correct handling of strange command
645 line arguments. These tests won't use code, so the exiting test types
646 won't work. So we need to be able to explicitly give the command line,
647 and the expected output, and have that tested and the coverage assessed.
648 Rather than having to spell out the whole command name, just give "cmd",
649 and discard that. Requiring but discarding the command make an empty
650 command list possible.
653 @for t in $(oceani_special_tests); do \
654 echo -n "Test $$t ... ";\
655 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
656 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
657 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
658 if ! cmp -s .tmp.want .tmp.have; then \
659 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
661 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
666 oceani_special_tests += "cmd"
667 oceani_special_tests += "cmd,-zyx"
668 oceani_special_tests += "cmd,nofile"
669 oceani_special_tests += "cmd,/dev/null"
670 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
673 oceani: no input file given
675 ###### output: cmd,-zyx
676 ./oceani: invalid option -- 'z'
677 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
679 ###### output: cmd,nofile
680 oceani: cannot open nofile
682 ###### output: cmd,/dev/null
683 oceani: could not find any code in /dev/null
685 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
686 oceani: cannot find section toast:nothing