Better error handling and reporting in run-testcase.py

Testcases now report whether runs failed or succeeded.

diff --git a/test/run-testcase.py b/test/run-testcase.py
index 4908cffc0452f306d336a88f15d7177553dbb194..80a05a7a998d6e7a157c832ce39495043ba61f4c 100755 (executable)
--- a/test/run-testcase.py
+++ b/test/run-testcase.py
@@ -371,15 +371,11 @@ def checkRun(test_proc, expected_output):
     if (len(actual_array) > len(expected_array)):
         raise ExtraOutput(actual_array[len(expected_array)].strip())
 
-# Compile the run's target program with all the requested plug-ins
-# then run the resulting executable and check that its output is as
-# expected.
-# Returns True if the run passes.
-def doRun(run):
+# Same as doRun but takes a temporary working directory (to place
+# compiled objects) as an input.
+def doRunInTempDir(run, tmp_dir):
     print "  Run:", run.name
 
-    tmp_dir = tempfile.mkdtemp(prefix='test-out-')
-
     # Compile all the plug-ins for this test.
     plugin_libs = []
     for i in range(len(run.plugin_list)):
@@ -394,19 +390,39 @@ def doRun(run):
     test_executable = compileTestcase(tmp_dir, run.target_source,
                                       run.hooks_source, plugin_libs)
 
-    if test_executable is not None:
-        test_proc = subprocess.Popen([test_executable], stdout = subprocess.PIPE)
-        try:
-            checkRun(test_proc, run.output)
-        except TestProgramException as e:
-            print e.getMessage()
-            return False
-    else:
+    if test_executable is None:
         return False
 
+    # Open the compiled test program...
+    try:
+        test_proc = subprocess.Popen([test_executable],
+                                     stdout = subprocess.PIPE)
+    except OSError as e:
+        sys.stderr.write('Fatal error opening test program: {0:s}\n'
+                         .format(e.strerror))
+        sys.exit(1)
+
+    # ... and run it.
+    try:
+        checkRun(test_proc, run.output)
+    except TestProgramException as e:
+        print e.getMessage()
+        return False
+
+    return True
+
+# Compile the run's target program with all the requested plug-ins
+# then run the resulting executable and check that its output is as
+# expected.
+# Returns True if the run passes.
+def doRun(run):
+    tmp_dir = tempfile.mkdtemp(prefix='test-out-')
+    result = doRunInTempDir(run, tmp_dir)
+
     # Delete temp directory
     shutil.rmtree(path = tmp_dir, ignore_errors = True)    
-    return True
+
+    return result
 
 def usage():
     sys.stderr.write(
@@ -488,5 +504,13 @@ if __name__ == '__main__':
         sys.exit(1)
 
     print "Testcase:", dh.name
+    passed = 0
     for run in dh.run_list:
-        doRun(run)
+        if doRun(run):
+            passed += 1
+
+    if passed == len(dh.run_list):
+        print "All runs passed!"
+    else:
+        print ("Testcase failed.  {0:d} out of {0:d} runs passed."
+               .format(passed, len(dh.run_list)))