From: NeilBrown <neil@brown.name>
Date: Sun, 5 May 2019 01:56:59 +0000 (+1000)
Subject: oceani-tests: add test suite.
X-Git-Tag: JamisonCreek-3~25
X-Git-Url: https://ocean-lang.org/code/?p=ocean;a=commitdiff_plain;h=17dd47fa4e2d6125d0faf4dede94af38c1ff4609

oceani-tests: add test suite.

This is the beginning of a test suite.  It guards
against errors, memory leaks, etc.

Signed-off-by: NeilBrown <neil@brown.name>
---

diff --git a/csrc/.gitignore b/csrc/.gitignore
index d954089..f8a607b 100644
--- a/csrc/.gitignore
+++ b/csrc/.gitignore
@@ -7,5 +7,9 @@ parsergen
 scanner
 calc
 oceani
+coverage_oceani
+coverage/
 itest*
 *.cgm
+*.gcno
+.tmp.*
diff --git a/csrc/oceani-tests.mdc b/csrc/oceani-tests.mdc
new file mode 100644
index 0000000..dc5a706
--- /dev/null
+++ b/csrc/oceani-tests.mdc
@@ -0,0 +1,393 @@
+# Ocean Interpreter test code
+
+Regular testing is, of course, important for developing any software.
+The Ocean interpreted is no exception.  This document allows easy
+testing by providing:
+
+- a collection of test program
+- the expected output of these programs when run with various arguments
+- some "Makefile" code to tie it all together.
+
+Three different sorts of tests are run.  As soon as any fail, the whole
+test stops.
+
+1/ Each program is run and the output is compared against the expected
+    output
+2/ Each program is then run under valgrind, and an error is reported
+    if valgrind detects an error, or if it reports and lost or unfreed
+    memory.
+3/ Each program is parsed and printed, then the result is parsed and printed.
+    The two results must match.
+4/ Each program is run with a version of `oceani` with test-coverage
+    recording enabled.  Once all programs have successfully run and
+    all the coverage data is available, we check that all lines have
+    been tested at least once. A few exceptions are allowed such as
+    lines that call `abort()`.  If any non-exceptional lines have not
+    been run, this final test fails.
+    Until the tests suite is (more) completed, we only throw and error
+    if fewer than 75% of the lines have been tested.
+
+Each test has a name, which used to identify the section in this file, and optionally some
+arguments separated from the name by commas.  For each test, there is a section named
+"output:" followed by the name-and-arguments.
+
+###### File: oceani-tests.mk
+
+	oceani_tests :=
+
+	tests:: oceani_test_suite
+	oceani_test_suite: oceani coverage_oceani
+		@rm -rf coverage; mkdir -p coverage
+		@cp *.gcno coverage
+		@for T in $(oceani_tests); do \
+		    echo -n "Test $$T ... "; \
+		    i="$$IFS"; IFS=,; set $$T; IFS="$$i"; t=$$1; shift; \
+		    ./md2c oceani-tests.mdc "output: $$T" | grep -v '^#' > .tmp.want; \
+		    ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > .tmp.have; \
+		    if ! cmp -s .tmp.want .tmp.have; then \
+		       echo "FAILED"; diff -u .tmp.want .tmp.have ; exit 1; fi ;\
+		    echo -n "passed ... "; \
+		    if ! valgrind ./oceani --section "test: $$t" oceani-tests.mdc $${1+"$$@"} \
+		         > /dev/null 2> .tmp.valg; then \
+		       echo "valgrind FAILED"; cat .tmp.valg; exit 1; fi ; \
+		    echo -n "valgrind passed ... "; \
+		    echo '``````' > .tmp.code1; echo '``````' > .tmp.code2 ;\
+		    ./oceani --noexec --print --section "test: $$t" oceani-tests.mdc >> .tmp.code1; \
+		    ./oceani --noexec --print .tmp.code1 >> .tmp.code2 ;\
+		    if ! cmp -s .tmp.code1 .tmp.code2; then \
+		       echo "Printing Failed"; diff -u .tmp.code1 .tmp.code2; exit1 ; fi ; \
+		    echo "Printing passed"; \
+		    ./coverage_oceani --print --section "test: $$t" oceani-tests.mdc $${1+"$$@"} > /dev/null ; \
+		    ./coverage_oceani -tpbn --section "test: $$t" oceani-tests.mdc > /dev/null 2>&1; \
+		done
+		@gcov -o coverage oceani.mdc > /dev/null 2> /dev/null
+		@mv *.gcov coverage ; [ -f .gcov ] && mv .gcov coverage
+		@ awk '/^ *[1-9]/ {ran+=1} /^ *###/ {skip+=1} \
+		    END {printf "coverage: %6.2f%%\n", ran * 100 / (ran + skip); \
+		         if (ran < (ran + skip) *0.75) exit(1) }' \
+		        coverage/oceani.mdc.gcov
+		@rm -f .tmp*
+
+	coverage_oceani: oceani.c
+		$(CC) $(CFLAGS) --coverage -fprofile-dir=coverage -o coverage_oceani oceani.c $(LDLIBS)
+
+	## test list
+
+## Values and variables
+
+The first test stores values in variables and performs various
+calculations on them.
+
+###### test list
+       oceani_tests += "valvar"
+
+
+###### test: valvar
+
+	program:
+		a := 23; b:=12
+		print a, b, a+b, a-b, a*b, a/b, a%b
+		print a<b, a<=b, a>b, a>=b, a<a, a==b, a==a
+
+		c ::= "This is a string"
+		d ::= " field theory"
+		print c, d, c++d
+
+###### output: valvar
+
+	23 12 35 11 276 1.91667 11
+	False False True True False False True
+	This is a string  field theory This is a string field theory
+
+Next we change the value of variables
+
+###### test list
+       oceani_tests += "setvar"
+
+###### test: setvar
+
+	program:
+		a := 4
+		a = a * a
+		a = (a + a) * (a + a)
+		a = a * a * a
+		print a, a/a
+
+###### output: setvar
+	1.07374e+09 1
+
+## Conditions and Loops
+
+Now we need to test if/else and some different loops
+
+###### test list
+       oceani_tests += cond_loop
+
+###### test: cond_loop
+
+	program:
+		a := 4
+		if a < 5:
+			print "Success"
+		else:
+			print "Failure"
+		for b:=1; then b=b+b; while b < 100:
+			print '', b,
+		print
+		// Newtons method for square root of 2
+		target ::= 2
+		guess := target
+		for:
+			count: number = 0
+		while:
+			current := guess * guess
+			use +(current - target) > 0.000000001
+		do:
+			guess = (guess + (target / guess) ) / 2
+			print count, guess
+			count = count + 1
+		print "error is ", target - guess * guess
+
+###### output: cond_loop
+	Success
+	 1 2 4 8 16 32 64
+	0 1.5
+	1 1.41667
+	2 1.41422
+	3 1.41421
+	error is  -4.51095e-12
+
+## Say Hello
+
+The demonstration code presented in the interpreted is suitable for the test suite.
+Here I break it into two parts, keeping the array code separate.
+
+###### test list
+	oceani_tests += "sayhello,55,33"
+	oceani_tests += "sayhello,12,60"
+
+###### test: sayhello
+
+	program A B:
+		print "Hello World, what lovely oceans you have!"
+		/* When a variable is defined in both branches of an 'if',
+		 * and used afterwards, the variables are merged.
+		 */
+		if A > B:
+			bigger := "yes"
+		else:
+			bigger := "no"
+		print "Is", A, "bigger than", B,"? ", bigger
+		/* If a variable is not used after the 'if', no
+		 * merge happens, so types can be different
+		 */
+		if A > B * 2:
+			double:string = "yes"
+			print A, "is more than twice", B, "?", double
+		else:
+			double := B*2
+			print "double", B, "is", double
+
+		a : number
+		a = A;
+		b:number = B
+		if a > 0 and b > 0:
+			while a != b:
+				if a < b:
+					b = b - a
+				else:
+					a = a - b
+			print "GCD of", A, "and", B,"is", a
+		else if a <= 0:
+			print a, "is not positive, cannot calculate GCD"
+		else:
+			print b, "is not positive, cannot calculate GCD"
+
+		for:
+			togo := 10
+			f1 := 1; f2 := 1
+			print "Fibonacci:", f1,f2,
+		then togo = togo - 1
+		while togo > 0:
+			f3 := f1 + f2
+			print "", f3,
+			f1 = f2
+			f2 = f3
+		print ""
+
+		/* Binary search... */
+		for:
+			lo:= 0; hi := 100
+			target := 77
+		while:
+			mid := (lo + hi) / 2
+			if mid == target:
+				use Found
+			if mid < target:
+				lo = mid
+			else:
+				hi = mid
+			if hi - lo < 1:
+				use GiveUp
+			use True
+		do: pass
+		case Found:
+			print "Yay, I found", target
+		case GiveUp:
+			print "Closest I found was", mid
+
+###### output: sayhello,55,33
+	Hello World, what lovely oceans you have!
+	Is 55 bigger than 33 ?  yes
+	double 33 is 66
+	GCD of 55 and 33 is 11
+	Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
+	Closest I found was 77.3438
+
+###### output: sayhello,12,60
+	Hello World, what lovely oceans you have!
+	Is 12 bigger than 60 ?  no
+	double 60 is 120
+	GCD of 12 and 60 is 12
+	Fibonacci: 1 1 2 3 5 8 13 21 34 55 89 144
+	Closest I found was 77.3438
+
+###### test list
+	oceani_tests += "insert_sort"
+###### test: insert_sort
+	program:
+		size::=55
+		list:[size]number
+		list[0] = 1234
+		for i:=1; then i = i + 1; while i < size:
+			n := list[i-1] * list[i-1]
+			list[i] = (n / 100) % 10000
+
+		print "Before sort:"
+		for i:=0; then i = i + 1; while i < size:
+			print "list[",i,"]=",list[i]
+
+		for i := 1; then i=i+1; while i < size:
+			for j:=i-1; then j=j-1; while j >= 0:
+				if list[j] > list[j+1]:
+					t:= list[j]
+					list[j] = list[j+1]
+					list[j+1] = t
+		print "After sort:"
+		for i:=0; then i = i + 1; while i < size:
+			print "list[",i,"]=",list[i]
+
+###### output: insert_sort
+	Before sort:
+	list[ 0 ]= 1234
+	list[ 1 ]= 5227
+	list[ 2 ]= 3215
+	list[ 3 ]= 3362
+	list[ 4 ]= 3030
+	list[ 5 ]= 1809
+	list[ 6 ]= 2724
+	list[ 7 ]= 4201
+	list[ 8 ]= 6484
+	list[ 9 ]= 422
+	list[ 10 ]= 1780
+	list[ 11 ]= 1684
+	list[ 12 ]= 8358
+	list[ 13 ]= 8561
+	list[ 14 ]= 2907
+	list[ 15 ]= 4506
+	list[ 16 ]= 3040
+	list[ 17 ]= 2416
+	list[ 18 ]= 8370
+	list[ 19 ]= 569
+	list[ 20 ]= 3237
+	list[ 21 ]= 4781
+	list[ 22 ]= 8579
+	list[ 23 ]= 5992
+	list[ 24 ]= 9040
+	list[ 25 ]= 7216
+	list[ 26 ]= 706
+	list[ 27 ]= 4984
+	list[ 28 ]= 8402
+	list[ 29 ]= 5936
+	list[ 30 ]= 2360
+	list[ 31 ]= 5696
+	list[ 32 ]= 4444
+	list[ 33 ]= 7491
+	list[ 34 ]= 1150
+	list[ 35 ]= 3225
+	list[ 36 ]= 4006
+	list[ 37 ]= 480
+	list[ 38 ]= 2304
+	list[ 39 ]= 3084
+	list[ 40 ]= 5110
+	list[ 41 ]= 1121
+	list[ 42 ]= 2566
+	list[ 43 ]= 5843
+	list[ 44 ]= 1406
+	list[ 45 ]= 9768
+	list[ 46 ]= 4138
+	list[ 47 ]= 1230
+	list[ 48 ]= 5129
+	list[ 49 ]= 3066
+	list[ 50 ]= 4003
+	list[ 51 ]= 240
+	list[ 52 ]= 576
+	list[ 53 ]= 3317
+	list[ 54 ]= 24
+	After sort:
+	list[ 0 ]= 24
+	list[ 1 ]= 240
+	list[ 2 ]= 422
+	list[ 3 ]= 480
+	list[ 4 ]= 569
+	list[ 5 ]= 576
+	list[ 6 ]= 706
+	list[ 7 ]= 1121
+	list[ 8 ]= 1150
+	list[ 9 ]= 1230
+	list[ 10 ]= 1234
+	list[ 11 ]= 1406
+	list[ 12 ]= 1684
+	list[ 13 ]= 1780
+	list[ 14 ]= 1809
+	list[ 15 ]= 2304
+	list[ 16 ]= 2360
+	list[ 17 ]= 2416
+	list[ 18 ]= 2566
+	list[ 19 ]= 2724
+	list[ 20 ]= 2907
+	list[ 21 ]= 3030
+	list[ 22 ]= 3040
+	list[ 23 ]= 3066
+	list[ 24 ]= 3084
+	list[ 25 ]= 3215
+	list[ 26 ]= 3225
+	list[ 27 ]= 3237
+	list[ 28 ]= 3317
+	list[ 29 ]= 3362
+	list[ 30 ]= 4003
+	list[ 31 ]= 4006
+	list[ 32 ]= 4138
+	list[ 33 ]= 4201
+	list[ 34 ]= 4444
+	list[ 35 ]= 4506
+	list[ 36 ]= 4781
+	list[ 37 ]= 4984
+	list[ 38 ]= 5110
+	list[ 39 ]= 5129
+	list[ 40 ]= 5227
+	list[ 41 ]= 5696
+	list[ 42 ]= 5843
+	list[ 43 ]= 5936
+	list[ 44 ]= 5992
+	list[ 45 ]= 6484
+	list[ 46 ]= 7216
+	list[ 47 ]= 7491
+	list[ 48 ]= 8358
+	list[ 49 ]= 8370
+	list[ 50 ]= 8402
+	list[ 51 ]= 8561
+	list[ 52 ]= 8579
+	list[ 53 ]= 9040
+	list[ 54 ]= 9768
+