Added build target to run solvetests

3 files changed, 64 insertions, 29 deletions

diff --git a/README.md b/README.md
index fa5b4a3..34d5f72 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,8 @@ output, the results compared with the .out files) and test/last.err
 Tests are always run in debug mode: this means that optimizations are
 disabled and some extra logging is enabled.
 
+More comprehensive tests (integration tests) can be performed with *tools*.
+
 ## Running "tools"
 
 In the tools folder there are some small programs that test various
@@ -104,6 +106,18 @@ file in the tools/results folder.
 
 To build and run a tool in debug mode, use `./build -d tool`.
 
+### The `solvetest` tools
+
+The tools denoted with `solvetest` can be used to test that the various
+solvers produce the correct solutions. They can be run individually as
+all other tools, or all together with
+
+```
+$ ./build solvetest   # Use -d for debug mode (very slow for some solvers)
+```
+
+If one of the solvetests fails, subsequent tests are going to be skipped.
+
 ## Command-line interface
 
 The `shell` folder contains the code for a rudimentary shell that can
diff --git a/build b/build
index 8c9e72f..9700d4a 100755
--- a/build
+++ b/build
@@ -177,6 +177,7 @@ build_help() {
         echo "shell       Build a basic nissy shell (./run)"
         echo "web         Build the WebAssembly / JavaScript module for nissy"
         echo "cpp FILES   Build and run the given FILES including cpp/nissy.h"
+        echo "solvetest   Build nissy and run the tests for solvers (tools)"
         echo ""
         echo "help        Show this help message"
         echo "config      Show build configuration and exit"
@@ -371,6 +372,39 @@ build_webtest() {
         rm -f runtest.js runtest.wasm
 }
 
+run_single_tool() {
+        results="tools/results"
+        last="$results/last.out"
+        date="$(date +'%Y-%m-%d-%H-%M-%S')"
+        file="$results/$toolname-$date.txt"
+        failed="tools/failed"
+
+        $CC $CFLAGS $WFLAGS $MFLAGS $(odflags) -o runtool "$t"/*.c nissy.o \
+                || exit 1
+
+        (
+                date +'%Y-%m-%d %H:%M'
+                echo ""
+                echo "=========== Running tool  ==========="
+                echo "tool name: $toolname"
+                echo ""
+                echo "======== nissy build command ========"
+                echo "$CC $CFLAGS $WFLAGS $MFLAGS $(odflags)"
+                echo ""
+                echo "======== tool build command  ========"
+                echo "$CC $CFLAGS $WFLAGS $MFLAGS $(odflags)"
+                echo ""
+                echo "============ tool output ============"
+                ./runtool $@ || touch "$failed"
+        ) | tee "$file" "$last"
+
+        rm -f runtool
+        if [ -f "$failed" ]; then
+                rm "$failed"
+                exit 1
+        fi
+}
+
 build_tool() {
         pattern="$1"
 
@@ -394,31 +428,17 @@ build_tool() {
         fi
 
         build_nissy || exit 1
+        run_single_tool
+}
 
-        results="tools/results"
-        last="$results/last.out"
-        date="$(date +'%Y-%m-%d-%H-%M-%S')"
-        file="$results/$toolname-$date.txt"
-
-        $CC $CFLAGS $WFLAGS $MFLAGS $(odflags) -o runtool "$t"/*.c nissy.o \
-                || exit 1
-
-        (
-                date +'%Y-%m-%d %H:%M'
-                echo ""
-                echo "=========== Running tool  ==========="
-                echo "tool name: $toolname"
-                echo ""
-                echo "======== nissy build command ========"
-                echo "$CC $CFLAGS $WFLAGS $MFLAGS $(odflags)"
-                echo ""
-                echo "======== tool build command  ========"
-                echo "$CC $CFLAGS $WFLAGS $MFLAGS $(odflags)"
-                echo ""
-                echo "============ tool output ============"
-                ./runtool $@
-        ) | tee "$file" "$last"
-        rm -f runtool
+build_solvetest() {
+        build_nissy || exit 1
+        for t in tools/*; do
+                if [ -d "$t" ] && (echo "$t" | grep -q "solvetest"); then
+                        toolname="$(basename "$t" .c)"
+                        run_single_tool || exit 1
+                fi
+        done
 }
 
 if [ "$1" = "-d" ]; then
@@ -435,7 +455,7 @@ fi
 
 case "$target" in
 help|config|clean|\
-nissy|lib|sharedlib|shell|python|cpp|web|test|webtest|tool)
+nissy|lib|sharedlib|shell|python|cpp|web|test|webtest|tool|solvetest)
         mkdir -p tables tools/results
         (build_"$target" $@) || exit 1
         exit 0
diff --git a/tools/solvetest.h b/tools/solvetest.h
index 4c72ca5..75f7aab 100644
--- a/tools/solvetest.h
+++ b/tools/solvetest.h
@@ -95,23 +95,24 @@ void run(void) {
         for (i = 0; s[i].scramble[0]; i++) {
                 printf("\n%d. %s\n", i, s[i].scramble);
 
-                /* Multiple solutions */
                 if (nissy_applymoves(NISSY_SOLVED_CUBE, s[i].scramble, cube)
-                     == -1) {
+                     == NISSY_ERROR_INVALID_MOVES) {
                         printf("Invalid scramble\n");
-                        continue;
+                        exit(1);
                 }
+
                 nissy_solve(cube, solver,
                     NISSFLAG, MINMOVES, MAXMOVES, MAXSOLUTIONS, OPTIMAL,
                     0, size, buf, SOL_BUFFER_LEN, sol, stats,
                     NULL, NULL);
+
                 if (check_all(sol, s[i].solutions)) {
                         printf("All solutions are correct\n");
                 } else {
                         printf("Error!\n");
                         printf("Found solution(s):\n%s", sol);
                         printf("Valid solution(s):\n%s", s[i].solutions);
-                        return;
+                        exit(1);
                 }
         }