[ocean] / csrc / oceani-tests.mdc

# Ocean Interpreter test code

Regular testing is, of course, important for developing any software.
The Ocean interpreted is no exception.  This document allows easy
testing by providing:

- a collection of test program
- the expected output of these programs when run with various arguments
- some "Makefile" code to tie it all together.

Three different sorts of tests are run.  As soon as any fail, the whole
test stops.

1/ Each program is run and the output is compared against the expected
    output
2/ Each program is then run under valgrind, and an error is reported
    if valgrind detects an error, or if it reports and lost or unfreed
    memory.
3/ Each program is parsed and printed, then the result is parsed and printed.
    The two results must match.
4/ Each program is run with a version of `oceani` with test-coverage
    recording enabled.  Once all programs have successfully run and
    all the coverage data is available, we check that all lines have
    been tested at least once. A few exceptions are allowed such as
    lines that call `abort()`.  If any non-exceptional lines have not
    been run, this final test fails.
    Until the tests suite is (more) completed, we only throw and error
    if fewer than 75% of the lines have been tested.

Each test has a name, which used to identify the section in this file, and optionally some
arguments separated from the name by commas.  For each test, there is a section named
"output:" followed by the name-and-arguments.

###### File: oceani-tests.mk

        oceani_tests :=
        ## test list

        tests:: oceani_test_suite
        oceani_test_suite: oceani coverage_oceani
                @echo -n Checking grammar ...
                @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
                    { echo "Grammar contains conflicts, please review" ; exit 1; }
                @echo ok
                @rm -rf coverage; mkdir -p coverage
                @cp *.gcno coverage
                @for T in $(oceani_tests); do \
                    echo -n "Test $$T.. "; \
                    i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
                    ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
                    ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
                    if ! cmp -s .tmp.want .tmp.have; then \
                       echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
                    echo -n "printing.. "; \
                    echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
                    ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
                    ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
                    if ! cmp -s .tmp.code1 .tmp.code2; then \
                       echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
                    echo -n "extra-newlines.. "; \
                    sed -e 'i\
                    ' .tmp.code1 > .tmp.code1a; \
                    echo '``````' > .tmp.code2a ;\
                    ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
                    if ! cmp -s .tmp.code1 .tmp.code2a; then \
                       echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
                    echo -n "exec-after-print.. "; \
                    ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
                    if ! cmp -s .tmp.want .tmp.have; then \
                       echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
                    echo " all passed"; \
                    ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
                    ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
                done

                ## test code

                @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
                @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
                @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
                @ awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
                    END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
                         if (ran < (ran + skip) *0.94) exit(1) }' \
                        coverage/oceani.mdc.gcov
                @rm -f .tmp*

                @for T in $(oceani_tests); do \
                    echo -n "Valgrind $$T.. "; \
                    i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
                    if ! valgrind --error-exitcode=1 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
                         > /dev/null 2>&1 ; then \
                       echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
                    if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
                       echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
                    if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
                       echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
                    echo " passed"; \
                done

        coverage_oceani: oceani.c
                $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)

## Values and variables

The first test stores values in variables and performs various
calculations on them.

###### test list
       oceani_tests += "valvar"

###### test: valvar

        program:
                a := 23; b:=12 ; b1 := -b
                print a, b, a+b, a-b, a*b, a/b, a%b
                print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
                print +a, +b, +b1, -a, -b, -b1
                x := True; y := False
                print x and y, x or y, x and x, y or y, x and not x, x < y

                c ::= "This is a string"
                d ::= " field theory"
                print c, d, c++d

                aconst :: string = "unchanging"

                // Check wrapping
                print
                  a + b
                  + (a*2)
                  + b1
                  + b

###### output: valvar

        23 12 35 11 276 1.91667 11
        False False True True False False True
        23 12 12 -23 -12 12
        False True True False False False
        This is a string  field theory This is a string field theory
        81

Next we change the value of variables

###### test list
       oceani_tests += "setvar"

###### test: setvar

        program:
                a := 4
                a = a * a
                a = (a + a) * (a + a)
                a = a * a * a
                print a, a/a

###### output: setvar
        1.07374e+09 1

Now some contants

###### test list
        oceani_tests += "consts"

###### test: consts
        const
                pi ::= 3.141 592 653
                four ::= 2 + 2 ; five ::= 10/2
        const pie ::= "I like Pie";
                cake ::= "The cake is"
                  ++ " a lie"

        program:
                print "Hello World, what lovely oceans you have!"
                print "are there", five, "?"
                print pi, pie, "but", cake

###### output: consts
        Hello World, what lovely oceans you have!
        are there 5 ?
        3.14159 I like Pie but The cake is a lie

Test merging of variables from multiple cases

###### test list
        oceani_tests += varmerge

###### test: varmerge

        program:
                for i:=0; then i=i+1; while i < 5:
                        switch i:
                                case 0: num:="zero"
                                case 1: num:="one"
                                case 2: num:="two"
                                case 3: num:="three"
                                else    num:="many"
                        print num,", ",
                print

###### output: varmerge
        zero , one , two , three , many , 

## Conditions and Loops

Now we need to test if/else and some different loops

###### test list
       oceani_tests += cond_loop

###### test: cond_loop

        program:
                a := 4
                if a < 5:
                        print "Success"
                else
                        print "Failure"
                for b:=1; then b=b+b; while b < 100:
                        print '', b,
                print
                // Newtons method for square root of 2
                target ::= 2
                guess := target
                for
                        count: number = 0
                while
                        current := guess * guess
                        use +(current - target) > 0.000000001
                do
                        guess = (guess + (target / guess) ) / 2
                        print count, guess
                        count = count + 1
                print "error is ", target - guess * guess

                for j:=0; then j = j+3 ; while j < 10:
                        if j != 0 and then 20 / j > 3:
                                print "20 /", j," =", 20 / j
                        else
                                print "I won't calculate 20 /", j
                pi ::= 3.1415926535897
                if 355/113 == pi or else +(pi - 355/113) < 0.001:
                        print "Close enough"
                print "lower" if 355/113 < pi else "higher"

                if pi > 3 then print "pi exceeds three"; else print "need more pie"
                if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
                for { i := 0; sum := 0 }
                then { i = i+1 }
                while i <= 10:
                        sum = sum + i
                else
                        pass
                        print "sum 1..10 is", sum

###### output: cond_loop
        Success
         1 2 4 8 16 32 64
        0 1.5
        1 1.41667
        2 1.41422
        3 1.41421
        error is  -4.51095e-12
        I won't calculate 20 / 0
        20 / 3  = 6.66667
        20 / 6  = 3.33333
        I won't calculate 20 / 9
        Close enough
        higher
        pi exceeds three
        pi sufficient
        sum 1..10 is 55

## Say Hello

The demonstration code presented in the interpreted is suitable for the test suite.
Here I break it into two parts, keeping the array code separate.

###### test list
        oceani_tests += "sayhello,55,33,hello,True"
        oceani_tests += "sayhello,12,60,there,False"

###### test: sayhello

        program As Bs astr bbools:
                A := $As; B := $Bs
                bbool := bbools == "True"
                print "Hello World, what lovely oceans you have!"
                /* When a variable is defined in both branches of an 'if',
                 * and used afterwards, the variables are merged.
                 */
                if A > B:
                        bigger := "yes"
                else
                        bigger := "no"
                print "Is", A, "bigger than", B,"? ", bigger
                /* If a variable is not used after the 'if', no
                 * merge happens, so types can be different
                 */
                if A > B * 2:
                        double:string = "yes"
                        print A, "is more than twice", B, "?", double
                else
                        double := B*2
                        print "double", B, "is", double

                a : number
                a = A;
                b:number = B
                if a > 0 and b > 0:
                        while a != b:
                                if a < b:
                                        b = b - a
                                else
                                        a = a - b
                        print "GCD of", A, "and", B,"is", a
                else if a <= 0:
                        print a, "is not positive, cannot calculate GCD"
                else
                        print b, "is not positive, cannot calculate GCD"

                for
                        togo := 10
                        f1 := 1; f2 := 1
                        print "Fibonacci:", f1,f2,
                then togo = togo - 1
                while togo > 0:
                        f3 := f1 + f2
                        print "", f3,
                        f1 = f2
                        f2 = f3
                print ""

                if bbool:
                        print astr ++ " was the str"
                else
                        print "I found the str over " ++ astr

                /* Binary search... */
                for
                        lo:= 0; hi := 100
                        target := 77
                while
                        mid := (lo + hi) / 2
                        if mid == target:
                                use Found
                        if mid < target:
                                lo = mid
                        else
                                hi = mid
                        if hi - lo < 1:
                                use GiveUp
                        use True
                do pass
                case Found:
                        print "Yay, I found", target
                case GiveUp:
                        print "Closest I found was", mid

###### output: sayhello,55,33,hello,True
        Hello World, what lovely oceans you have!
        Is 55 bigger than 33 ?  yes
        double 33 is 66
        GCD of 55 and 33 is 11
        Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
        hello was the str
        Closest I found was 77.3438

###### output: sayhello,12,60,there,False
        Hello World, what lovely oceans you have!
        Is 12 bigger than 60 ?  no
        double 60 is 120
        GCD of 12 and 60 is 12
        Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
        I found the str over there
        Closest I found was 77.3438

###### test list
        oceani_tests += "insert_sort"
###### test: insert_sort
        program:
                size::=55
                list:[size]number
                list[0] = 1_234
                for i:=1; then i = i + 1; while i < size:
                        n := list[i-1] * list[i-1]
                        list[i] = (n / 100) % 10000

                print "Before sort:"
                for i:=0; then i = i + 1; while i < size:
                        print "list[",i,"]=",list[i]

                for i := 1; then i=i+1; while i < size:
                        for j:=i-1; then j=j-1; while j >= 0:
                                if list[j] > list[j+1]:
                                        t:= list[j]
                                        list[j] = list[j+1]
                                        list[j+1] = t
                print "After sort:"
                for i:=0; then i = i + 1; while i < size:
                        print "list[",i,"]=",list[i]

###### output: insert_sort
        Before sort:
        list[ 0 ]= 1234
        list[ 1 ]= 5227
        list[ 2 ]= 3215
        list[ 3 ]= 3362
        list[ 4 ]= 3030
        list[ 5 ]= 1809
        list[ 6 ]= 2724
        list[ 7 ]= 4201
        list[ 8 ]= 6484
        list[ 9 ]= 422
        list[ 10 ]= 1780
        list[ 11 ]= 1684
        list[ 12 ]= 8358
        list[ 13 ]= 8561
        list[ 14 ]= 2907
        list[ 15 ]= 4506
        list[ 16 ]= 3040
        list[ 17 ]= 2416
        list[ 18 ]= 8370
        list[ 19 ]= 569
        list[ 20 ]= 3237
        list[ 21 ]= 4781
        list[ 22 ]= 8579
        list[ 23 ]= 5992
        list[ 24 ]= 9040
        list[ 25 ]= 7216
        list[ 26 ]= 706
        list[ 27 ]= 4984
        list[ 28 ]= 8402
        list[ 29 ]= 5936
        list[ 30 ]= 2360
        list[ 31 ]= 5696
        list[ 32 ]= 4444
        list[ 33 ]= 7491
        list[ 34 ]= 1150
        list[ 35 ]= 3225
        list[ 36 ]= 4006
        list[ 37 ]= 480
        list[ 38 ]= 2304
        list[ 39 ]= 3084
        list[ 40 ]= 5110
        list[ 41 ]= 1121
        list[ 42 ]= 2566
        list[ 43 ]= 5843
        list[ 44 ]= 1406
        list[ 45 ]= 9768
        list[ 46 ]= 4138
        list[ 47 ]= 1230
        list[ 48 ]= 5129
        list[ 49 ]= 3066
        list[ 50 ]= 4003
        list[ 51 ]= 240
        list[ 52 ]= 576
        list[ 53 ]= 3317
        list[ 54 ]= 24
        After sort:
        list[ 0 ]= 24
        list[ 1 ]= 240
        list[ 2 ]= 422
        list[ 3 ]= 480
        list[ 4 ]= 569
        list[ 5 ]= 576
        list[ 6 ]= 706
        list[ 7 ]= 1121
        list[ 8 ]= 1150
        list[ 9 ]= 1230
        list[ 10 ]= 1234
        list[ 11 ]= 1406
        list[ 12 ]= 1684
        list[ 13 ]= 1780
        list[ 14 ]= 1809
        list[ 15 ]= 2304
        list[ 16 ]= 2360
        list[ 17 ]= 2416
        list[ 18 ]= 2566
        list[ 19 ]= 2724
        list[ 20 ]= 2907
        list[ 21 ]= 3030
        list[ 22 ]= 3040
        list[ 23 ]= 3066
        list[ 24 ]= 3084
        list[ 25 ]= 3215
        list[ 26 ]= 3225
        list[ 27 ]= 3237
        list[ 28 ]= 3317
        list[ 29 ]= 3362
        list[ 30 ]= 4003
        list[ 31 ]= 4006
        list[ 32 ]= 4138
        list[ 33 ]= 4201
        list[ 34 ]= 4444
        list[ 35 ]= 4506
        list[ 36 ]= 4781
        list[ 37 ]= 4984
        list[ 38 ]= 5110
        list[ 39 ]= 5129
        list[ 40 ]= 5227
        list[ 41 ]= 5696
        list[ 42 ]= 5843
        list[ 43 ]= 5936
        list[ 44 ]= 5992
        list[ 45 ]= 6484
        list[ 46 ]= 7216
        list[ 47 ]= 7491
        list[ 48 ]= 8358
        list[ 49 ]= 8370
        list[ 50 ]= 8402
        list[ 51 ]= 8561
        list[ 52 ]= 8579
        list[ 53 ]= 9040
        list[ 54 ]= 9768

## Arrays

We already have some array tests, but this is where we put other
ad-hoc things array related.

###### test list
        oceani_tests += arrays

###### test: arrays

        program:
                bools:[5]Boolean
                strings:[4]string

                bools[3] = strings[1] == "Hello"
                bools[1] = strings[2] <= "there"

                for i:=0; then i=i+1; while i<5:
                        j ::= i
                        ra:[j]number
                        ra[i-1] = i*i
                        ra[6] = 42 // mustn't crash
                        print '', bools[i], ra[j-1],
                print

###### output: arrays
         False 0 True 1 False 4 False 9 False 16

## Structures

Time to test if structure declarations and accesses work correctly.

###### test list
        oceani_tests += structs

###### test: structs

        const three ::= 3
        struct foo
                size:[three]number
                name:string
                active:Boolean

        struct baz { a:number; b:Boolean; }

        program:
                info:[4]foo

                for i:=0; then i=i+1; while i < 4:
                        switch i
                        case 2: nm:= "peter"
                        case 0: nm:= "bob"
                        case 1: nm:= "jane"
                        else    nm:= "janine"

                        info[i].name = nm
                        info[i].size[0] = i*i
                        info[i].active = nm == "jane"

                for i:=0; then i=i+1; while i < 4:
                        print info[i].name, info[i].active, info[i].size[0]

###### output: structs

        bob False 0
        jane True 1
        peter False 4
        janine False 9

## Test code with syntax errors

Syntax errors aren't handled well yet - the result is almost always a
single message about the first error.  So this section will be fairly
thin until we add proper parsing recovery in the face of common errors.

A special case of syntax errors is token errors, when a token is only
accepted because the parser doesn't know quite enough to reject it.
There are handled better as they are quite local, so a single test
program can trigger most of the possible errors.

To handle erronous code, we need a different set of tests, as we need to
capture `stderr`. The same test code will be used for type errors too.
As error messages contain the line number, and we don't want changes to
this file to change the reported numbers, we copy the code into a
separate file first, then run from there.

###### test code
        @for t in $(oceani_failing_tests); do \
            echo -n "Test $$t ... "; \
            ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
            echo '``````' > .tmp.code; \
            ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
            ./oceani .tmp.code > .tmp.have 2>&1; \
            if ! cmp -s .tmp.want .tmp.have; then \
               echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
            echo  "passed"; \
            ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
        done || true

###### test list
        oceani_failing_tests := syn1
        oceani_failing_tests += tokerr

###### test: syn1

        program:
                if then else while do

###### output: syn1
        .tmp.code:3:11: Syntax error in statement: then

###### test: tokerr
        program:
                a := 1i  // imaginary numbers aren't understood
                b:[2i]number // array sizes are handled separately
                c:[3.14159]Boolean // array sizes must be integers
                d:[1_000_000_000_000]number // they mustn't be huge
                patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream

                multi := """
                This is a multiline string
                With an unsupportable suffix
                """Aa

                xx:unknown = 24
                yy:[unknowable]number
                zzsize := 4
                zz:[zzsize]string // size must be constant, use ::=

                // These numbers should be bad in all contexts: FIXME
                aa:[00123]number

###### output: tokerr
        .tmp.code:3:13: error: unsupported number suffix: 1i
        .tmp.code:4:11: error: unsupported number suffix: 2i
        .tmp.code:5:11: error: array size must be an integer: 3.14159
        .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
        .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
        .tmp.code:9:17: error: unsupported string suffix: """
                This is a multiline string
                With an unsupportable suffix
                """Aa
        .tmp.code:14:11: error: undefined type: unknown
        .tmp.code:15:12: error: name undeclared: unknowable
        .tmp.code:17:12: error: array size must be a constant: zzsize
        .tmp.code:20:12: error: unrecognised number: 00123

## Tests for type errors

Type error don't cause parsing to abort, so we can fit many in the
one test program.  Some type errors are found during the parse, others
during type analysis which doesn't run if parsing failed.  So we cannot
fit everything in one.

These programs were generated by looking for the
various places that `type_err()` are called.

###### test list
        oceani_failing_tests += type_err1 type_err2 type_err3 type_err4

###### test: type_err1

        program:
                print "hello" ++ 5, 5 ++ "hello"

                b ::= 3
                b = b + 1

                if 3 * 4 and not True: print "Weird"

###### output: type_err1
        .tmp.code:3:25: error: expected string found number
        .tmp.code:3:28: error: expected string found number
        .tmp.code:6:8: error: Cannot assign to a constant: b
        .tmp.code:5:8: info: name was defined as a constant here
        .tmp.code:6:8: error: Cannot assign to a constant: b
        .tmp.code:5:8: info: name was defined as a constant here
        .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
        oceani: type error in program - not running.

###### test: type_err2

        program:
                a := 1
                a := 2
                a ::= 3
                a:number = 4
                a ::number = 5
                c:

###### output: type_err2
        .tmp.code:4:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:5:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:6:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:7:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:8:8: Variable declared with no type or value: c

###### test: type_err3

        struct foo
                a: number
                b:string = "hello"

        program:
                c := "hello"
                c = c + 1
                c = "hello" ++ (True and False)
                c = 4 < 5
                print 45 + ( "Hello" ++ "there")
                c[5] = 1

                while
                        use 1
                        use True
                        use "Hello"
                do
                        print
                case 1: print "one"
                case "Hello": print "Hello"

                a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
                a1 = a2
                a1 = a3
                a1 = a4
                a1[2] = "hello"
                a4[1] = True
                c = a2[3]

                bar:foo
                foo.c = 43
                print c.foo

###### output: type_err3
        .tmp.code:8:12: error: expected number but variable 'c' is string
        .tmp.code:7:8: info: this is where 'c' was set to string
        .tmp.code:8:12: error: Arithmetic returns number but string expected
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:9:24: error: Boolean operation found where string expected
        .tmp.code:10:12: error: Comparison returns Boolean but string expected
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:11:21: error: Concat returns string but number expected
        .tmp.code:12:8: error: string cannot be indexed
        .tmp.code:12:8: error: string cannot be indexed
        .tmp.code:21:13: error: expected number found string
        .tmp.code:17:16: error: expected number, found string
        .tmp.code:24:8: error: cannot assign value of type [5]number
        .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
        .tmp.code:23:36: info: this is where 'a3' was set to [10]number
        .tmp.code:25:8: error: cannot assign value of type [5]number
        .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
        .tmp.code:23:51: info: this is where 'a4' was set to [5]string
        .tmp.code:26:8: error: cannot assign value of type [5]number
        .tmp.code:27:16: error: expected number found string
        .tmp.code:28:16: error: expected string found Boolean
        .tmp.code:29:12: error: have number but need string
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:32:8: error: variable used but not declared: foo
        .tmp.code:32:8: error: field reference attempted on none, not a struct
        .tmp.code:32:16: error: expected none found number
        .tmp.code:33:14: error: field reference attempted on string, not a struct
        oceani: type error in program - not running.

###### test: type_err4
        program:
                a:=1; b=2; c::=3
                print a, b, c

###### output: type_err4
        .tmp.code:3:14: error: variable used but not declared: b
        .tmp.code:3:16: error: expected none found number
        .tmp.code:3:14: info: variable 'b' was set as none here.
        oceani: type error in program - not running.

###### test list
        oceani_failing_tests += type_err_const type_err_const1

###### test: type_err_const
        const
                foo :: number = 45
                bar ::= "string" + 56
        const
                bar ::= "baz"
        program:
                foo := 4
                print foo, bar

###### output: type_err_const
        .tmp.code:4:16: error: expected number found string
        .tmp.code:6:8: error: name already declared: bar
        .tmp.code:4:8: info: this is where 'bar' was first declared
        .tmp.code:8:8: error: variable 'foo' redeclared
        .tmp.code:3:8: info: this is where 'foo' was first declared

###### test: type_err_const1
        const
                foo : number = 45
                bar := "string"
        program:
                foo := 4
                print foo, bar

###### output: type_err_const1
        .tmp.code:3:12: Syntax error in constant: :
        .tmp.code:4:12: Syntax error in constant: :

## Test erroneous command line args

To improve coverage, we want to test correct handling of strange command
line arguments.  These tests won't use code, so the exiting test types
won't work.  So we need to be able to explicitly give the command line,
and the expected output, and have that tested and the coverage assessed.
Rather than having to spell out the whole command name, just give "cmd",
and discard that.  Requiring but discarding the command make an empty
command list possible.

###### test code
        @for t in $(oceani_special_tests); do \
            echo -n "Test $$t ... ";\
            i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
            ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
            ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
            if ! cmp -s .tmp.want .tmp.have; then \
               echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
            echo  "passed"; \
            ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
        done || true

###### test list
        oceani_special_tests += "cmd"
        oceani_special_tests += "cmd,-zyx"
        oceani_special_tests += "cmd,nofile"
        oceani_special_tests += "cmd,/dev/null"
        oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"

###### output: cmd
        oceani: no input file given

###### output: cmd,-zyx
        ./oceani: invalid option -- 'z'
        Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn

###### output: cmd,nofile
        oceani: cannot open nofile

###### output: cmd,/dev/null
        oceani: could not find any code in /dev/null

###### output: cmd,--section,toast:nothing,oceani-tests.mdc
        oceani: cannot find section toast:nothing