1 # Ocean Interpreter test code
3 Regular testing is, of course, important for developing any software.
4 The Ocean interpreted is no exception. This document allows easy
7 - a collection of test program
8 - the expected output of these programs when run with various arguments
9 - some "Makefile" code to tie it all together.
11 Three different sorts of tests are run. As soon as any fail, the whole
14 1/ Each program is run and the output is compared against the expected
16 2/ Each program is then run under valgrind, and an error is reported
17 if valgrind detects an error, or if it reports and lost or unfreed
19 3/ Each program is parsed and printed, then the result is parsed and printed.
20 The two results must match.
21 4/ Each program is run with a version of `oceani` with test-coverage
22 recording enabled. Once all programs have successfully run and
23 all the coverage data is available, we check that all lines have
24 been tested at least once. A few exceptions are allowed such as
25 lines that call `abort()`. If any non-exceptional lines have not
26 been run, this final test fails.
27 Until the tests suite is (more) completed, we only throw and error
28 if fewer than 75% of the lines have been tested.
30 Each test has a name, which used to identify the section in this file, and optionally some
31 arguments separated from the name by commas. For each test, there is a section named
32 "output:" followed by the name-and-arguments.
34 ###### File: oceani-tests.mk
39 tests:: oceani_test_suite
40 oceani_test_suite: oceani coverage_oceani
41 @rm -rf coverage; mkdir -p coverage
43 @for T in $(oceani_tests); do \
44 echo -n "Test $$T ... "; \
45 i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
46 ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
47 ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
48 if ! cmp -s .tmp.want .tmp.have; then \
49 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
50 echo -n "passed ... "; \
51 if ! valgrind ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
52 > /dev/null 2> .tmp.valg; then \
53 echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
54 echo -n "valgrind passed ... "; \
55 echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
56 ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
57 ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
58 if ! cmp -s .tmp.code1 .tmp.code2; then \
59 echo "Printing Failed"; diff -u .tmp.code1 .tmp.code2; exit1 ; fi ; \
60 echo "Printing passed"; \
61 ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
62 ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
67 @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
68 @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
69 @ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
70 END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
71 if (ran < (ran + skip) *0.85) exit(1) }' \
72 coverage/oceani.mdc.gcov
75 coverage_oceani: oceani.c
76 $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
78 ## Values and variables
80 The first test stores values in variables and performs various
84 oceani_tests += "valvar"
91 print a, b, a+b, a-b, a*b, a/b, a%b
92 print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
94 c ::= "This is a string"
100 23 12 35 11 276 1.91667 11
101 False False True True False False True
102 This is a string field theory This is a string field theory
104 Next we change the value of variables
107 oceani_tests += "setvar"
114 a = (a + a) * (a + a)
118 ###### output: setvar
121 ## Conditions and Loops
123 Now we need to test if/else and some different loops
126 oceani_tests += cond_loop
128 ###### test: cond_loop
136 for b:=1; then b=b+b; while b < 100:
139 // Newtons method for square root of 2
145 current := guess * guess
146 use +(current - target) > 0.000000001
148 guess = (guess + (target / guess) ) / 2
151 print "error is ", target - guess * guess
153 ###### output: cond_loop
160 error is -4.51095e-12
164 The demonstration code presented in the interpreted is suitable for the test suite.
165 Here I break it into two parts, keeping the array code separate.
168 oceani_tests += "sayhello,55,33"
169 oceani_tests += "sayhello,12,60"
171 ###### test: sayhello
174 print "Hello World, what lovely oceans you have!"
175 /* When a variable is defined in both branches of an 'if',
176 * and used afterwards, the variables are merged.
182 print "Is", A, "bigger than", B,"? ", bigger
183 /* If a variable is not used after the 'if', no
184 * merge happens, so types can be different
187 double:string = "yes"
188 print A, "is more than twice", B, "?", double
191 print "double", B, "is", double
202 print "GCD of", A, "and", B,"is", a
204 print a, "is not positive, cannot calculate GCD"
206 print b, "is not positive, cannot calculate GCD"
211 print "Fibonacci:", f1,f2,
220 /* Binary search... */
237 print "Yay, I found", target
239 print "Closest I found was", mid
241 ###### output: sayhello,55,33
242 Hello World, what lovely oceans you have!
243 Is 55 bigger than 33 ? yes
245 GCD of 55 and 33 is 11
246 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
247 Closest I found was 77.3438
249 ###### output: sayhello,12,60
250 Hello World, what lovely oceans you have!
251 Is 12 bigger than 60 ? no
253 GCD of 12 and 60 is 12
254 Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
255 Closest I found was 77.3438
258 oceani_tests += "insert_sort"
259 ###### test: insert_sort
264 for i:=1; then i = i + 1; while i < size:
265 n := list[i-1] * list[i-1]
266 list[i] = (n / 100) % 10000
269 for i:=0; then i = i + 1; while i < size:
270 print "list[",i,"]=",list[i]
272 for i := 1; then i=i+1; while i < size:
273 for j:=i-1; then j=j-1; while j >= 0:
274 if list[j] > list[j+1]:
279 for i:=0; then i = i + 1; while i < size:
280 print "list[",i,"]=",list[i]
282 ###### output: insert_sort
396 ## Test code with syntax errors
398 Syntax errors aren't handled well yet - the result is almost always a
399 single message about the first error. So this section will be fairly
400 thin until we add proper parsing recovery in the face of common errors.
402 A special case of syntax errors is token errors, when a token is only
403 accepted because the parser doesn't know quite enough to reject it.
404 There are handled better as they are quite local, so a single test
405 program can trigger most of the possible errors.
407 To handle erronous code, we need a different set of tests, as we need to
408 capture `stderr`. The same test code will be used for type errors too.
409 As error messages contain the line number, and we don't want changes to
410 this file to change the reported numbers, we copy the code into a
411 separate file first, then run from there.
414 @for t in $(oceani_failing_tests); do \
415 echo -n "Test $$t ... "; \
416 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
417 echo '``````' > .tmp.code; \
418 ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
419 ./oceani .tmp.code > .tmp.have 2>&1; \
420 if ! cmp -s .tmp.want .tmp.have; then \
421 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
423 ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
427 oceani_failing_tests := syn1
428 oceani_failing_tests += tokerr
433 if then else while do
436 .tmp.code:3:11: error: unhandled parse error: then
440 a := 1i // imaginary numbers aren't understood
441 b:[2i]number // array sizes are handled separately
442 c:[3.14159]Boolean // array sizes must be integers
443 d:[1_000_000_000_000]number // they mustn't be huge
444 patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream
447 This is a multiline string
448 With an unsupportable suffix
452 yy:[unknowable]number
454 zz:[zzsize]string // size must be constant, use ::=
456 // These numbers should be bad in all contexts: FIXME
459 ###### output: tokerr
460 .tmp.code:3:13: error: unsupported number suffix: 1i
461 .tmp.code:4:11: error: unsupported number suffix: 2i
462 .tmp.code:5:11: error: array size must be an integer: 3.14159
463 .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
464 .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
465 .tmp.code:9:17: error: unsupported string suffix: """
466 This is a multiline string
467 With an unsupportable suffix
469 .tmp.code:14:11: error: undefined type: unknown
470 .tmp.code:15:12: error: name undeclared: unknowable
471 .tmp.code:17:12: error: array size must be a constant: zzsize
472 .tmp.code:20:12: error: unrecognised number: 00123
474 ## Tests for type errors
476 Type error don't cause parsing to abort, so we can fit many in the
477 one test program. Some type errors are found during the parse, others
478 during type analysis which doesn't run if parsing failed. So we cannot
479 fit everything in one.
481 These programs were generated by looking for the
482 various places that `type_err()` are called.
485 oceani_failing_tests += type_err1 type_err2 type_err3
487 ###### test: type_err1
490 print "hello" ++ 5, 5 ++ "hello"
495 if 3 * 4 and not True: print "Weird"
497 ###### output: type_err1
498 .tmp.code:3:25: error: expected string found number
499 .tmp.code:3:28: error: expected string found number
500 .tmp.code:6:8: error: Cannot assign to a constant: b
501 .tmp.code:5:8: info: name was defined as a constant here
502 .tmp.code:6:8: error: Cannot assign to a constant: b
503 .tmp.code:5:8: info: name was defined as a constant here
504 .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
505 oceani: type error in program - not running.
507 ###### test: type_err2
517 ###### output: type_err2
518 .tmp.code:4:8: error: variable 'a' redeclared
519 .tmp.code:3:8: info: this is where 'a' was first declared
520 .tmp.code:5:8: error: variable 'a' redeclared
521 .tmp.code:3:8: info: this is where 'a' was first declared
522 .tmp.code:6:8: error: variable 'a' redeclared
523 .tmp.code:3:8: info: this is where 'a' was first declared
524 .tmp.code:7:8: error: variable 'a' redeclared
525 .tmp.code:3:8: info: this is where 'a' was first declared
526 .tmp.code:8:8: Variable declared with no type or value: c
528 ###### test: type_err3
533 c = "hello" ++ (True and False)
535 print 45 + ( "Hello" ++ "there")
545 case "Hello": print "Hello"
547 a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
555 ###### output: type_err3
556 .tmp.code:4:12: error: expected number but variable 'c' is string
557 .tmp.code:3:8: info: this is where 'c' was set to string
558 .tmp.code:4:12: error: Arithmetic returns number but string expected
559 .tmp.code:3:8: info: variable 'c' was set as string here.
560 .tmp.code:5:24: error: Boolean operation found where string expected
561 .tmp.code:6:12: error: Comparison returns Boolean but string expected
562 .tmp.code:3:8: info: variable 'c' was set as string here.
563 .tmp.code:7:21: error: Concat returns string but number expected
564 .tmp.code:8:8: error: string cannot be indexed
565 .tmp.code:8:8: error: string cannot be indexed
566 .tmp.code:17:13: error: expected number found string
567 .tmp.code:13:16: error: expected number, found string
568 .tmp.code:20:8: error: cannot assign value of type [5]number
569 .tmp.code:21:13: error: expected [5]number but variable 'a3' is [10]number
570 .tmp.code:19:36: info: this is where 'a3' was set to [10]number
571 .tmp.code:21:8: error: cannot assign value of type [5]number
572 .tmp.code:22:13: error: expected [5]number but variable 'a4' is [5]string
573 .tmp.code:19:51: info: this is where 'a4' was set to [5]string
574 .tmp.code:22:8: error: cannot assign value of type [5]number
575 .tmp.code:23:16: error: expected number found string
576 .tmp.code:24:16: error: expected string found Boolean
577 .tmp.code:25:12: error: have number but need string
578 .tmp.code:3:8: info: variable 'c' was set as string here.
579 oceani: type error in program - not running.
582 ## Test erroneous command line args
584 To improve coverage, we want to test correct handling of strange command
585 line arguments. These tests won't use code, so the exiting test types
586 won't work. So we need to be able to explicitly give the command line,
587 and the expected output, and have that tested and the coverage assessed.
588 Rather than having to spell out the whole command name, just give "cmd",
589 and discard that. Requiring but discarding the command make an empty
590 command list possible.
593 @for t in $(oceani_special_tests); do \
594 echo -n "Test $$t ... ";\
595 i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
596 ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
597 ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
598 if ! cmp -s .tmp.want .tmp.have; then \
599 echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
601 ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
606 oceani_special_tests += "cmd"
607 oceani_special_tests += "cmd,-zyx"
608 oceani_special_tests += "cmd,nofile"
609 oceani_special_tests += "cmd,/dev/null"
610 oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"
613 oceani: no input file given
615 ###### output: cmd,-zyx
616 ./oceani: invalid option -- 'z'
617 Usage: oceani --trace --print --noexec --brackets--section=SectionName prog.ocn
619 ###### output: cmd,nofile
620 oceani: cannot open nofile
622 ###### output: cmd,/dev/null
623 oceani: could not find any code in /dev/null
625 ###### output: cmd,--section,toast:nothing,oceani-tests.mdc
626 oceani: cannot find section toast:nothing