[ocean] / csrc / oceani-tests.mdc

# Ocean Interpreter test code

Regular testing is, of course, important for developing any software.
The Ocean interpreted is no exception.  This document allows easy
testing by providing:

- a collection of test program
- the expected output of these programs when run with various arguments
- some "Makefile" code to tie it all together.

Three different sorts of tests are run.  As soon as any fail, the whole
test stops.

1/ Each program is run and the output is compared against the expected
    output
2/ Each program is then run under valgrind, and an error is reported
    if valgrind detects an error, or if it reports and lost or unfreed
    memory.
3/ Each program is parsed and printed, then the result is parsed and printed.
    The two results must match.
4/ Each program is run with a version of `oceani` with test-coverage
    recording enabled.  Once all programs have successfully run and
    all the coverage data is available, we check that all lines have
    been tested at least once. A few exceptions are allowed such as
    lines that call `abort()`.  If any non-exceptional lines have not
    been run, this final test fails.
    Until the tests suite is (more) completed, we only throw and error
    if fewer than 75% of the lines have been tested.

Each test has a name, which used to identify the section in this file, and optionally some
arguments separated from the name by commas.  For each test, there is a section named
"output:" followed by the name-and-arguments.

###### File: oceani-tests.mk

        oceani_tests :=
        ## test list
        oceani_valg_tests := $(oceani_tests)
        ## combine test lists

        tests:: oceani_test_suite
        oceani_test_suite: oceani coverage_oceani
                @echo -n Checking grammar ...
                @./parsergen --report --LALR --tag Parser oceani.mdc | grep " - no conflicts" > /dev/null || \
                    { echo "Grammar contains conflicts, please review" ; exit 1; }
                @echo ok
                @rm -rf coverage; mkdir -p coverage
                @cp *.gcno coverage
                @for T in $(oceani_tests); do \
                    echo -n "Test $$T.. "; \
                    i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
                    ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
                    ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
                    if ! cmp -s .tmp.want .tmp.have; then \
                       echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
                    echo -n "printing.. "; \
                    echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
                    ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
                    ./oceani --noexec --print .tmp.code1 >> .tmp.code2 || exit 1;\
                    if ! cmp -s .tmp.code1 .tmp.code2; then \
                       echo " Failed"; diff -u .tmp.code1 .tmp.code2; exit 1 ; fi ; \
                    echo -n "extra-newlines.. "; \
                    sed -e 'i\
                    ' .tmp.code1 > .tmp.code1a; \
                    echo '``````' > .tmp.code2a ;\
                    ./oceani --noexec --print .tmp.code1a >> .tmp.code2a || exit 1;\
                    if ! cmp -s .tmp.code1 .tmp.code2a; then \
                       echo " Failed"; diff -u .tmp.code1 .tmp.code2a; exit 1; fi ; \
                    echo -n "exec-after-print.. "; \
                    ./oceani .tmp.code1 $${1+"$$@"} > .tmp.have ; \
                    if ! cmp -s .tmp.want .tmp.have; then \
                       echo " FAILED"; diff -u .tmp.want .tmp.have; exit 1;fi; \
                    echo " all passed"; \
                    ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
                    ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
                done

                ## test code

                @for i in coverage/#*.gcda; do mv $$i coverage/$${i##*#}; done
                @gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
                @mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage || true
                @rm -f .tmp*

                @[ -n "$$SKIP_VALGRIND" ] || for T in $(oceani_valg_tests); do \
                    echo -n "Valgrind $$T.. "; \
                    i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
                    if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
                         > /dev/null 2>&1 ; [ $$? -eq 42 ] ; then \
                       echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
                    if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
                       echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
                    if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
                       echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
                    echo " passed"; \
                done
                ## valgrind test code
                @[ -n "$$SKIP_COVERAGE_CHECK" ] || awk '/NOTEST/ { next } /^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
                    END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
                         if (ran < (ran + skip) *0.99) exit(1) }' \
                        coverage/oceani.mdc.gcov

        coverage_oceani: oceani.c
                $(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)

## Values and variables

The first test stores values in variables and performs various
calculations on them.

###### test list
       oceani_tests += "valvar"

###### test: valvar

        func main(argv:[]string)
                a := 23; b:=12 ; b1 := -b
                print a, b, a+b, a-b, a*b, a/b, a%b
                print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
                print +a, +b, +b1, -a, -b, -b1
                x := True; y := False
                print x and y, x or y, x and x, y or y, x and not x, x < y

                c ::= "This is a string"
                d ::= " field theory"
                print c, d, c++d

                aconst :: string = "unchanging"

                // Check wrapping
                print
                  a + b
                  + (a*2)
                  + b1
                  + b

                z:Boolean= True
                w:Boolean = False
                if ?z:
                        print "w??z", w??z
                        print "z??w", z??w
                        print "z??False", z??False, "w??False", w??False
                if ?w:
                        print "Weird?"
                print $"-34.56", $"not-a-number"
                print $"4i"
                if True { pass; pass }

###### output: valvar

        23 12 35 11 276 1.916666667 11
        False False True True False False True
        23 12 12 -23 -12 12
        False True True False False False
        This is a string  field theory This is a string field theory
        81
        w??z True
        z??w True
        z??False True w??False False
        -34.56 0
        Unsupported suffix: 4i
        4

Next we change the value of variables

###### test list
       oceani_tests += "setvar"

###### test: setvar

        func main()
                a := 4
                a = a * a
                a = (a + a) * (a + a)
                a = a * a * a
                print a, a/a

###### output: setvar
        1073741824 1

Now some contants

###### test list
        oceani_tests += "consts"

###### test: consts
        const
                tau ::= pi * 2
                pi ::= 3.141 592 653
                four ::= 2 + 2 ; five ::= 10/2
        const pie ::= "I like Pie";
                cake ::= "The cake is"
                  ++ " a lie"

        func main()
                print "Hello World, what lovely oceans you have!"
                print "are there", five, "?"
                print pi, pie, "but", cake, "Tau is", tau

###### output: consts
        Hello World, what lovely oceans you have!
        are there 5 ?
        3.141592653 I like Pie but The cake is a lie Tau is 6.283185306

Test merging of variables from multiple cases

###### test list
        oceani_tests += varmerge

###### test: varmerge

        func main()
                for i:=0; then i=i+1; while i < 5:
                        switch i:
                                case 0: num:="zero"
                                case 1: scratch:=42; num:="one"
                                case 2: num:="two"
                                case 3: num:="three"
                                else    num:="many"
                        print num,", ",
                print

                for i:=0; then i=i+1; while i < 5:
                        switch { use i }
                                case 0: num:="zero" ;
                                case 1: num:="one" ;
                                case 2 { num:="two" } case 3 { num:="three" }
                                else  num:="many"
                        // re-declare a CondScope variable
                        num := i*i
                        print num,", ",
                print

###### output: varmerge
        zero , one , two , three , many , 
        0 , 1 , 4 , 9 , 16 , 

## Conditions and Loops

Now we need to test if/else and some different loops

###### test list
       oceani_tests += cond_loop

###### test: cond_loop

        func main()
                a := 4
                if a < 5:
                        print "Success"
                else
                        print "Failure"
                for b:=1; then b=b+b; while b < 100:
                        print '', b,
                print
                // Newtons method for square root of 2
                target ::= 2
                guess := target
                for
                        count: number = 0
                while
                        current := guess * guess
                        use +(current - target) > 0.000000001
                do
                        guess = (guess + (target / guess) ) / 2
                        print count, guess
                        count = count + 1
                print "error is ", target - guess * guess

                for j:=0; then j = j+3 ; while j < 10:
                        if j != 0 and then 20 / j > 3:
                                print "20 /", j," =", 20 / j
                        else
                                print "I won't calculate 20 /", j
                pi ::= 3.1415926535897
                if 355/113 == pi or else +(pi - 355/113) < 0.001:
                        print "Close enough"
                print "lower" if 355/113 < pi else "higher"
                print "higher" if 355/113 > pi else "lower"

                if pi > 3 then print "pi exceeds three"; else print "need more pie"
                if (pi < 3) { print "not enough pi" } else { print "pi sufficient" }
                for { i := 0; sum := 0 }
                then { i = i+1 }
                while i <= 10:
                        sum = sum + i
                else
                        pass
                        print "sum 1..10 is", sum
                if
                        PI1 := 22/7
                        use PI1 < pi
                then
                        print "Smaller"
                else
                        print 'larger'

###### output: cond_loop
        Success
         1 2 4 8 16 32 64
        0 1.5
        1 1.416666667
        2 1.414215686
        3 1.414213562
        error is  -4.510950445e-12
        I won't calculate 20 / 0
        20 / 3  = 6.666666667
        20 / 6  = 3.333333333
        I won't calculate 20 / 9
        Close enough
        higher
        higher
        pi exceeds three
        pi sufficient
        sum 1..10 is 55
        larger

## Say Hello

The demonstration code presented in the interpreted is suitable for the test suite.
Here I break it into two parts, keeping the array code separate.

###### test list
        oceani_tests += "sayhello,55,33,hello,True"
        oceani_tests += "sayhello,12,60,there,False"

###### test: sayhello

        func main(av:[]string)
                A := $av[1]; B := $av[2]
                astr := av[3]
                l := av[]
                bbool := av[l-1] == "True"
                print "Hello World, what lovely oceans you have!"
                /* When a variable is defined in both branches of an 'if',
                 * and used afterwards, the variables are merged.
                 */
                if A > B:
                        bigger := "yes"
                else
                        bigger := "no"
                print "Is", A, "bigger than", B,"? ", bigger
                /* If a variable is not used after the 'if', no
                 * merge happens, so types can be different
                 */
                if A > B * 2:
                        double:string = "yes"
                        print A, "is more than twice", B, "?", double
                else
                        double := B*2
                        print "double", B, "is", double

                a : number
                a = A;
                b:number = B
                print "still", bigger // check for regression in scoping
                if a > 0 and b > 0:
                        while a != b:
                                if a < b:
                                        b = b - a
                                else
                                        a = a - b
                        print "GCD of", A, "and", B,"is", a
                else if a <= 0:
                        print a, "is not positive, cannot calculate GCD"
                else
                        print b, "is not positive, cannot calculate GCD"

                for
                        togo := 10
                        f1 := 1; f2 := 1
                        print "Fibonacci:", f1,f2,
                then togo = togo - 1
                while togo > 0:
                        f3 := f1 + f2
                        print "", f3,
                        f1 = f2
                        f2 = f3
                print ""

                if bbool:
                        print astr ++ " was the str"
                else
                        print "I found the str over " ++ astr

                /* Binary search... */
                for
                        lo:= 0; hi := 100
                        target := 77
                while
                        mid := (lo + hi) / 2
                        if mid == target:
                                use .Found
                        if mid < target:
                                lo = mid
                        else
                                hi = mid
                        if hi - lo < 1:
                                lo = mid
                                use .GiveUp
                        use True
                do pass
                case .Found:
                        print "Yay, I found", target
                case .GiveUp:
                        print "Closest I found was", lo

###### output: sayhello,55,33,hello,True
        Hello World, what lovely oceans you have!
        Is 55 bigger than 33 ?  yes
        double 33 is 66
        still yes
        GCD of 55 and 33 is 11
        Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
        hello was the str
        Closest I found was 77.34375

###### output: sayhello,12,60,there,False
        Hello World, what lovely oceans you have!
        Is 12 bigger than 60 ?  no
        double 60 is 120
        still no
        GCD of 12 and 60 is 12
        Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
        I found the str over there
        Closest I found was 77.34375

###### test list
        oceani_tests += "insert_sort"
###### test: insert_sort
        func main()
                size::=55
                list:[size]number
                list[0] = 1_234
                for i:=1; then i = i + 1; while i < size:
                        n := list[i-1] * list[i-1]
                        list[i] = (n / 100) % 10000

                print "Before sort:"
                for i:=0; then i = i + 1; while i < size:
                        print "list[",i,"]=",list[i]

                for i := 1; then i=i+1; while i < size:
                        for j:=i-1; then j=j-1; while j >= 0:
                                if list[j] > list[j+1]:
                                        t:= list[j]
                                        list[j] = list[j+1]
                                        list[j+1] = t
                print "After sort:"
                for i:=0; then i = i + 1; while i < size:
                        print "list[",i,"]=",list[i]

###### output: insert_sort
        Before sort:
        list[ 0 ]= 1234
        list[ 1 ]= 5227
        list[ 2 ]= 3215
        list[ 3 ]= 3362
        list[ 4 ]= 3030
        list[ 5 ]= 1809
        list[ 6 ]= 2724
        list[ 7 ]= 4201
        list[ 8 ]= 6484
        list[ 9 ]= 422
        list[ 10 ]= 1780
        list[ 11 ]= 1684
        list[ 12 ]= 8358
        list[ 13 ]= 8561
        list[ 14 ]= 2907
        list[ 15 ]= 4506
        list[ 16 ]= 3040
        list[ 17 ]= 2416
        list[ 18 ]= 8370
        list[ 19 ]= 569
        list[ 20 ]= 3237
        list[ 21 ]= 4781
        list[ 22 ]= 8579
        list[ 23 ]= 5992
        list[ 24 ]= 9040
        list[ 25 ]= 7216
        list[ 26 ]= 706
        list[ 27 ]= 4984
        list[ 28 ]= 8402
        list[ 29 ]= 5936
        list[ 30 ]= 2360
        list[ 31 ]= 5696
        list[ 32 ]= 4444
        list[ 33 ]= 7491
        list[ 34 ]= 1150
        list[ 35 ]= 3225
        list[ 36 ]= 4006
        list[ 37 ]= 480
        list[ 38 ]= 2304
        list[ 39 ]= 3084
        list[ 40 ]= 5110
        list[ 41 ]= 1121
        list[ 42 ]= 2566
        list[ 43 ]= 5843
        list[ 44 ]= 1406
        list[ 45 ]= 9768
        list[ 46 ]= 4138
        list[ 47 ]= 1230
        list[ 48 ]= 5129
        list[ 49 ]= 3066
        list[ 50 ]= 4003
        list[ 51 ]= 240
        list[ 52 ]= 576
        list[ 53 ]= 3317
        list[ 54 ]= 24
        After sort:
        list[ 0 ]= 24
        list[ 1 ]= 240
        list[ 2 ]= 422
        list[ 3 ]= 480
        list[ 4 ]= 569
        list[ 5 ]= 576
        list[ 6 ]= 706
        list[ 7 ]= 1121
        list[ 8 ]= 1150
        list[ 9 ]= 1230
        list[ 10 ]= 1234
        list[ 11 ]= 1406
        list[ 12 ]= 1684
        list[ 13 ]= 1780
        list[ 14 ]= 1809
        list[ 15 ]= 2304
        list[ 16 ]= 2360
        list[ 17 ]= 2416
        list[ 18 ]= 2566
        list[ 19 ]= 2724
        list[ 20 ]= 2907
        list[ 21 ]= 3030
        list[ 22 ]= 3040
        list[ 23 ]= 3066
        list[ 24 ]= 3084
        list[ 25 ]= 3215
        list[ 26 ]= 3225
        list[ 27 ]= 3237
        list[ 28 ]= 3317
        list[ 29 ]= 3362
        list[ 30 ]= 4003
        list[ 31 ]= 4006
        list[ 32 ]= 4138
        list[ 33 ]= 4201
        list[ 34 ]= 4444
        list[ 35 ]= 4506
        list[ 36 ]= 4781
        list[ 37 ]= 4984
        list[ 38 ]= 5110
        list[ 39 ]= 5129
        list[ 40 ]= 5227
        list[ 41 ]= 5696
        list[ 42 ]= 5843
        list[ 43 ]= 5936
        list[ 44 ]= 5992
        list[ 45 ]= 6484
        list[ 46 ]= 7216
        list[ 47 ]= 7491
        list[ 48 ]= 8358
        list[ 49 ]= 8370
        list[ 50 ]= 8402
        list[ 51 ]= 8561
        list[ 52 ]= 8579
        list[ 53 ]= 9040
        list[ 54 ]= 9768

## Arrays

We already have some array tests, but this is where we put other
ad-hoc things array related.

###### test list
        oceani_tests += arrays

###### test: arrays

        func main()
                bools:[5]Boolean
                strings:[4]string

                bools[3] = strings[1] == "Hello"
                bools[1] = strings[2] <= "there"

                for i:=0; then i=i+1; while i<5:
                        j ::= i
                        ra:[j]number
                        ra[i-1] = i*i
                        ra[6] = 42 // mustn't crash
                        print '', bools[i], ra[j-1],
                print

###### output: arrays
         False 0 True 1 False 4 False 9 False 16

## Structures

Time to test if structure declarations and accesses work correctly.

###### test list
        oceani_tests += structs

###### test: structs

        const three ::= 3
        struct foo
                size:[three]number
                name:string = "Hello"
                thing:baz
                active:Boolean = True

        struct baz { a:number; b:Boolean; }
        struct bat {
                a:string
                b:Boolean
        }
        struct bat2 a:string; b:Boolean

        func main
        do
                info:[4]foo

                for i:=0; then i=i+1; while i < 4:
                        switch i
                        case 2: nm:= "peter"
                        case 0: nm:= "bob"
                        case 1: nm:= "jane"
                        else    nm:= "janine"

                        info[i].name = nm
                        info[i].size[0] = i*i
                        if nm != "jane":
                                info[i].active = False

                for i:=0; then i=i+1; while i < 4:
                        print info[i].name, info[i].active, info[i].size[0]
                info[0].thing.b = True
                x:bat
                x.a = "Hello"

###### output: structs

        bob False 0
        jane True 1
        peter False 4
        janine False 9

## Functions

Test functions.  They don't return anything, so we need to get them to print

###### test list
        oceani_tests += functions func_ret_type

###### test: functions

        func test1
                t: Boolean
        do
                if t:
                        print "true"

        func noarg
        do
                pass

        func twoarg
                a:number
                b:string
        do
                while a > 0:
                        print b
                        a = a - 1

        func test(n:number; s:string)
                if n >= 1:
                        print n,s,
                        test(n-1, "."++s)
                else
                        print "done"

        func to_polar
                x:number; y:number
        return
                rho:number
                theta:number
        do
                rho = x + y
                theta = x - y

        func main()
                for i:=0; then i = i + 1; while i < 5:
                        test(i, " ")
                angular := to_polar(32, 23)
                print angular.rho, angular.theta

        func test2(n:number; s:string;) : (ret:number)
                ret = n + $s

        func random
        return
                n:number
        do
                n = 4 // xkcd:221

###### output: functions
        done
        1  done
        2  1 . done
        3  2 . 1 .. done
        4  3 . 2 .. 1 ... done
        55 9

###### test: func_ret_type

        func double(n:number):number
                use n+n

        func answer
                prefix:string
                suffix:string
        return string
        do
                use prefix ++ suffix

        func noarg_returns
        return Boolean
        do
                use 22/7 == 3.14159

        func main()
                for j:=10; then j = j - 3; while j > -5:
                        print answer("dou","ble"), j, "is", double(j)

###### output: func_ret_type
        double 10 is 20
        double 7 is 14
        double 4 is 8
        double 1 is 2
        double -2 is -4

## References

A simple linked list example

###### test list
        oceani_tests += "linked_list,one,two,three,four"

###### test: linked_list

        struct linkage
                next: @node
        struct node
                list: linkage
                this: string

        func insert(list:@linkage; new:string)
                p:=list
                while ?p.next and then p.next.this < new:
                        p = p.next.list
                t:@node = @new()
                t.list.next = p.next
                t.this = new
                p.next = t;

        func printlist(list:@linkage)
                while ?list.next:
                        print list@.next.this
                        list = list@.next.list

        func freelist(list:@linkage)
                if list.next != @nil:
                        lp:@linkage = list.next.list
                        freelist(lp)
                        @free = list.next

        func main(argv:[]string)
                list : linkage

                insert(list, "@start");
                insert(list, "~end")
                for i:=1; then i=i+1; while i < argv[]:
                        insert(list, argv[i])
                insert(list, "Hello!")
                printlist(list)
                freelist(list)

###### output: linked_list,one,two,three,four
        @start
        Hello!
        four
        one
        three
        two
        ~end

## Test code with syntax errors

Syntax errors aren't handled well yet - the result is almost always a
single message about the first error.  So this section will be fairly
thin until we add proper parsing recovery in the face of common errors.

A special case of syntax errors is token errors, when a token is only
accepted because the parser doesn't know quite enough to reject it.
There are handled better as they are quite local, so a single test
program can trigger most of the possible errors.

To handle erronous code, we need a different set of tests, as we need to
capture `stderr`. The same test code will be used for type errors too.
As error messages contain the line number, and we don't want changes to
this file to change the reported numbers, we copy the code into a
separate file first, then run from there.

###### test code
        @for t in $(oceani_failing_tests); do \
            echo -n "Test $$t ... "; \
            ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
            echo '``````' > .tmp.code; \
            ./md2c oceani-tests.mdc "test: $$t" | grep -v '^#' >> .tmp.code; \
            ./oceani .tmp.code > .tmp.have 2>&1; \
            if ! cmp -s .tmp.want .tmp.have; then \
               echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
            echo  "passed"; \
            ./coverage_oceani --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1 ;\
        done || true

###### combine test lists
        oceani_valg_tests += $(oceani_failing_tests)

###### test list
        oceani_failing_tests := syn1
        oceani_failing_tests += tokerr

###### test: syn1

        func main()
                if then else while do

###### output: syn1
        .tmp.code:3:11: Syntax error in statement: then

###### test: tokerr
        func main()
                a := 1i  // imaginary numbers aren't understood
                b:[2i]number // array sizes are handled separately
                c:[3.14159]Boolean // array sizes must be integers
                d:[1_000_000_000_000]number // they mustn't be huge
                patn: string = "foo[ ,_]*bar"re // regexp strings are just a dream

                multi := """
                This is a multiline string
                With an unsupportable suffix
                """Aa

                xx:unknown = 24
                yy:[unknowable]number
                zzsize := 4
                zz:[zzsize]string // size must be constant, use ::=

                // These numbers should be bad in all contexts: FIXME
                aa:[00123]number

###### output: tokerr
        .tmp.code:3:13: error: unsupported number suffix: 1i
        .tmp.code:4:11: error: unsupported number suffix: 2i
        .tmp.code:5:11: error: array size must be an integer: 3.14159
        .tmp.code:6:11: error: array size is too large: 1_000_000_000_000
        .tmp.code:7:23: error: unsupported string suffix: "foo[ ,_]*bar"re
        .tmp.code:9:17: error: unsupported string suffix: """
                This is a multiline string
                With an unsupportable suffix
                """Aa
        .tmp.code:15:12: error: name undeclared: unknowable
        .tmp.code:17:12: error: array size must be a constant: zzsize
        .tmp.code:20:12: error: unrecognised number: 00123
        .tmp.code:14:11: error: type used but not declared: unknown

## Tests for type errors

Type error don't cause parsing to abort, so we can fit many in the
one test program.  Some type errors are found during the parse, others
during type analysis which doesn't run if parsing failed.  So we cannot
fit everything in one.

These programs were generated by looking for the
various places that `type_err()` are called.

###### test list
        oceani_failing_tests += type_err1 type_err2 type_err3 type_err4 type_err5 type_err6

###### test: type_err1

        func main()
                print "hello" ++ 5, 5 ++ "hello"

                b ::= 3
                b = b + 1

                if 3 * 4 and not True: print "Weird"
                d:number = .fred
                (d + b) = 12

###### output: type_err1
        .tmp.code:3:25: error: expected string found number
        .tmp.code:3:28: error: expected string found number
        .tmp.code:6:8: error: Cannot assign to a constant: b
        .tmp.code:5:8: info: name was defined as a constant here
        .tmp.code:8:11: error: Arithmetic returns number but Boolean expected
        .tmp.code:9:20: error: expected number found label
        .tmp.code:9:8: info: variable 'd' was set as number here.
        .tmp.code:10:8: error: cannot assign to an rval
        oceani: type error in program - not running.

###### test: type_err2

        func main()
                a := 1
                a := 2
                a ::= 3
                a:number = 4
                a ::number = 5
                c:
                d:number = 02

###### output: type_err2
        .tmp.code:4:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:5:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:6:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:7:8: error: variable 'a' redeclared
        .tmp.code:3:8: info: this is where 'a' was first declared
        .tmp.code:8:8: Variable declared with no type or value: c
        .tmp.code:9:19: error: unsupported number format: 02

###### test: type_err3

        struct foo
                a: number
                b:string = "hello"; d:Boolean

        func main()
                c := "hello"
                c = c + 1
                c = "hello" ++ (True and False)
                c = 4 < 5
                print 45 + ( "Hello" ++ "there")
                c[5] = 1

                while
                        use 1
                        use True
                        use "Hello"
                do
                        print
                case 1: print "one"
                case "Hello": print "Hello"

                a1:[5]number; a2:[5]number; a3:[10]number; a4:[5]string
                a1 = a2
                a1 = a3
                a1 = a4
                a1[2] = "hello"
                a4[1] = True
                c = a2[3]

                bar:foo
                foo.c = 43
                print c.foo
                print bar.c
                print bar.b + 42

                // trigger 'labels not permitted' error message
                while 1 if True else False:
                        print
                case 2: print "two"
                print "one" ++ a4[], c[]

                x:Boolean = $"42"

                five ::= 5
                four ::= 4
                x1:[five]number
                x2:[four]number
                x1 = x2

###### output: type_err3
        .tmp.code:8:12: error: expected number but variable 'c' is string
        .tmp.code:7:8: info: this is where 'c' was set to string
        .tmp.code:8:12: error: Arithmetic returns number but string expected
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:9:24: error: Boolean operation found where string expected
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:10:12: error: Comparison returns Boolean but string expected
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:11:21: error: Concat returns string but number expected
        .tmp.code:12:8: error: string cannot be indexed
        .tmp.code:12:8: error: string cannot be indexed
        .tmp.code:21:13: error: expected number found string
        .tmp.code:17:16: error: expected number, found string
        .tmp.code:24:8: error: cannot assign value of type [5]number
        .tmp.code:25:13: error: expected [5]number but variable 'a3' is [10]number
        .tmp.code:23:36: info: this is where 'a3' was set to [10]number
        .tmp.code:25:8: error: cannot assign value of type [5]number
        .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
        .tmp.code:26:13: error: expected [5]number but variable 'a4' is [5]string
        .tmp.code:23:51: info: this is where 'a4' was set to [5]string
        .tmp.code:26:8: error: cannot assign value of type [5]number
        .tmp.code:23:8: info: variable 'a1' was set as [5]number here.
        .tmp.code:27:16: error: expected number found string
        .tmp.code:28:16: error: expected string found Boolean
        .tmp.code:29:12: error: have number but need string
        .tmp.code:7:8: info: variable 'c' was set as string here.
        .tmp.code:32:8: error: variable used but not declared: foo
        .tmp.code:32:8: error: field reference on none is not supported
        .tmp.code:32:16: error: expected none found number
        .tmp.code:33:14: error: field reference on string is not supported
        .tmp.code:34:14: error: cannot find requested field in foo
        .tmp.code:35:17: error: have string but need number
        .tmp.code:38:29: error: expected number found Boolean
        .tmp.code:41:23: error: have number but need string
        .tmp.code:41:29: error: string cannot provide length
        .tmp.code:43:21: error: Can only convert string to number, not Boolean
        .tmp.code:43:8: info: variable 'x' was set as Boolean here.
        .tmp.code:49:13: error: expected [five]number but variable 'x2' is [four]number
        .tmp.code:48:8: info: this is where 'x2' was set to [four]number
        .tmp.code:49:8: error: cannot assign value of type [five]number
        .tmp.code:47:8: info: variable 'x1' was set as [five]number here.
        oceani: type error in program - not running.

###### test: type_err4
        func main()
                a:=1; b=2; c::=3
                print a, b, c

###### output: type_err4
        .tmp.code:3:14: error: variable used but not declared: b
        .tmp.code:3:16: error: expected none found number
        .tmp.code:3:14: info: variable 'b' was set as none here.
        oceani: type error in program - not running.

###### test: type_err5
        struct foo
                bar:baz
                a:number
        struct baz
                bat:foo
                b:string
        struct foo
                c:number
                x:[5]:string

###### output: type_err5
        .tmp.code:8:7: error: type already declared: foo
        .tmp.code:2:7: info: this is location of declartion: foo
        .tmp.code:10:13: Syntax error in struct field: :
        .tmp.code:5:7: error: type has recursive definition: baz
        .tmp.code:2:7: error: type has recursive definition: foo

###### test: type_err6

        func main()
                a:= "hello"
                if ?a:
                        print "no"
                print a ?? "there"

###### output: type_err6
        .tmp.code:4:12: error: '?' requires a testable value, not string
        .tmp.code:6:14: error: "??" requires a testable value, not string
        oceani: type error in program - not running.


###### test list
        oceani_failing_tests += type_err_const type_err_const1 type_err_const2 missing_program bad_main

###### test: type_err_const
        const
                foo :: number = 45
                bar ::= "string" + 56
        const
                bar ::= "baz"
        func main()
                foo := 4
                print foo, bar

        // trigger duplicate-main error
        func main()
                foo := 6
                print bar, foo

###### output: type_err_const
        .tmp.code:6:8: error: name already declared: bar
        .tmp.code:4:8: info: this is where 'bar' was first declared
        .tmp.code:8:8: error: variable 'foo' redeclared
        .tmp.code:3:8: info: this is where 'foo' was first declared
        .tmp.code:12:5: error: function 'main' redeclared
        .tmp.code:7:5: info: this is where 'main' was first declared
        .tmp.code:13:8: error: variable 'foo' redeclared
        .tmp.code:3:8: info: this is where 'foo' was first declared
        .tmp.code:4:16: error: expected number found string

###### test: type_err_const1
        const
                foo : number = 45
                bar := "string"
        func main()
                foo := 4
                print foo, bar

###### output: type_err_const1
        .tmp.code:3:12: Syntax error in constant: :
        .tmp.code:4:12: Syntax error in constant: :

###### test: type_err_const2
        const
                four ::= two + two
                two ::= four / 2
        
###### output: type_err_const2
        .tmp.code:3:8: error: const four cannot be resolved.
        .tmp.code:4:8: error: const two cannot be resolved.

###### test: missing_program
        const
                foo::="bar"

###### output: missing_program
        oceani: no main function found.

###### test: bad_main
        func main(foo:string)
                print foo

###### output: bad_main
        .tmp.code:2:10: error: expected []string but variable 'foo' is string
        .tmp.code:2:10: info: this is where 'foo' was set to string
        oceani: main has wrong type.

Test for type errors with functions

###### test list
        oceani_failing_tests += func_err_args func_err_redeclare

###### test: func_err_args

        func test1(a:number; b:string; c:[3]Boolean)
                print a, b, c[1]

        func test2(a:number; b:string; c:[3]Boolean)
                print a, b, c[1]

        func test3()
                # use undefined names
                print a, z

        func main()
                truth:[3]Boolean
                truth[1] = True
                test1(1,"hello")
                test1("hello",1)
                test1(1, "two", truth)
                test1(1, 2, truth)
                test1(1, "lo", truth, 4)
                print test(), test1(1,2,3)
                if test1 == test2:
                        pass

        func test4(a:number):string
                use a * a

        func test5(a:number):string
                print a

        struct foo
                a: number
                b:string = "hello"

        func test6(a:number):foo
                b:foo
                b.a = a
                use b

        func test7(a:@number)
                test7(45)
                test7("45")

###### output: func_err_args
        .tmp.code:40:14: error: cannot pass rval when reference expected
        .tmp.code:41:14: error: expected @number found string
        .tmp.code:34:5: error: function cannot return value of type foo
        .tmp.code:28:8: error: expected string, found none
        .tmp.code:25:8: error: expected string, found number
        .tmp.code:15:14: error: insufficient arguments to function.
        .tmp.code:16:14: error: expected number found string
        .tmp.code:16:22: error: expected string found number
        .tmp.code:16:14: error: insufficient arguments to function.
        .tmp.code:18:17: error: expected string found number
        .tmp.code:19:14: error: too many arguments to function.
        .tmp.code:20:14: error: attempt to call a non-function.
        .tmp.code:20:32: error: expected string found number
        .tmp.code:20:28: error: insufficient arguments to function.
        .tmp.code:21:20: error: expected "func test1" but variable 'test2' is "func test2"
        .tmp.code:5:5: info: this is where 'test2' was set to "func test2"
        .tmp.code:10:14: error: variable used but not declared: a
        .tmp.code:10:17: error: variable used but not declared: z
        oceani: type error in program - not running.

###### test: func_err_redeclare

        func test1(a:number; b:string; c:[3]Boolean)
                print a, b, c[1]

        func test1
        do
                pass

        func test1
                b:Boolean
        do
                pass

###### output: func_err_redeclare
        .tmp.code:5:5: error: function 'test1' redeclared
        .tmp.code:2:5: info: this is where 'test1' was first declared
        .tmp.code:9:5: error: function 'test1' redeclared
        .tmp.code:2:5: info: this is where 'test1' was first declared

Test for errors with references

###### test list
        oceani_failing_tests += ref_err1 ref_err2

###### test: ref_err1
        func main()
                ref:@number
                @foo = ref
                ref = @old()
                if ref == @null:
                        print "null"

###### output: ref_err1
        .tmp.code:4:9: error: only "@free" makes sense here: foo
        .tmp.code:5:15: error: Only reference function is "@new()": old
        .tmp.code:6:19: error: Only reference value is "@nil": null

###### test: ref_err2
        func main()
                ref:@number
                ref2:@string
                num:number = @new()
                print num@
                if num == @nil or ref == ref2 or ref == 2 or ref.foo:
                        @free = num
                ref = 1

###### output: ref_err2
        .tmp.code:5:22: error: @new() can only be used with references, not number
        .tmp.code:5:8: info: variable 'num' was set as number here.
        .tmp.code:6:14: error: Cannot dereference number
        .tmp.code:7:19: error: @nil can only be used with reference, not number
        .tmp.code:7:33: error: expected @number but variable 'ref2' is @string
        .tmp.code:4:8: info: this is where 'ref2' was set to @string
        .tmp.code:7:48: error: expected @number found number
        .tmp.code:7:53: error: field reference on number is not supported
        .tmp.code:7:56: error: have none but need Boolean
        .tmp.code:8:17: error: @free can only be assigned a reference, not number
        .tmp.code:8:17: error: @free can only be assigned a reference, not number
        .tmp.code:9:8: error: Cannot assign an rval to a reference.
        oceani: type error in program - not running.

## Test erroneous command line args

To improve coverage, we want to test correct handling of strange command
line arguments.  These tests won't use code, so the exiting test types
won't work.  So we need to be able to explicitly give the command line,
and the expected output, and have that tested and the coverage assessed.
Rather than having to spell out the whole command name, just give "cmd",
and discard that.  Requiring but discarding the command make an empty
command list possible.

###### test code
        @for t in $(oceani_special_tests); do \
            echo -n "Test $$t ... ";\
            i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
            ./md2c oceani-tests.mdc "output: $$t" | grep -v '^#' > .tmp.want; \
            ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
            if ! cmp -s .tmp.want .tmp.have; then \
               echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
            echo  "passed"; \
            ./coverage_oceani $${1+"$$@"} > /dev/null 2>&1 ;\
        done || true
###### valgrind test code
        @[ -n "$$SKIP_VALGRIND" ] || for t in $(oceani_special_tests); do\
            echo -n "Valgrind $$t.. "; \
            i="$$IFS"; IFS=,; set $$t; IFS="$$i"; shift ;\
            if valgrind --error-exitcode=42 --log-file=.tmp.valg ./oceani $${1+"$$@"} > .tmp.have 2>&1 ;\
               [ $$? -eq 42 ]; then \
                       echo "FAILED"; cat .tmp.valg; exit 1; fi ; \
            if grep 'LEAK SUMMARY' .tmp.valg > /dev/null; then \
               echo "valgrind found LEAKS"; cat .tmp.valg ; exit 1 ; fi; \
            if grep 'in use at exit [1-9]' .tmp.valg > /dev/null; then \
               echo "valgrind found memory in use at exit"; cat .tmp.valg ; exit 1 ; fi; \
            echo " passed"; \
        done

###### test list
        oceani_special_tests += "cmd"
        oceani_special_tests += "cmd,-zyx"
        oceani_special_tests += "cmd,nofile"
        oceani_special_tests += "cmd,/dev/null"
        oceani_special_tests += "cmd,--section,toast:nothing,oceani-tests.mdc"

###### output: cmd
        oceani: no input file given

###### output: cmd,-zyx
        ./oceani: invalid option -- 'z'
        Usage: oceani --trace --print --noexec --brackets --section=SectionName prog.ocn

###### output: cmd,nofile
        oceani: cannot open nofile

###### output: cmd,/dev/null
        oceani: could not find any code in /dev/null

###### output: cmd,--section,toast:nothing,oceani-tests.mdc
        oceani: cannot find section toast:nothing