run-testcase.py now checks the output of the test program.

diff --git a/test/run-testcase.py b/test/run-testcase.py
index c26aedc9a6d0db8d8ac22cb1457d4c241a17fa13..4908cffc0452f306d336a88f15d7177553dbb194 100755 (executable)
--- a/test/run-testcase.py
+++ b/test/run-testcase.py
@@ -71,6 +71,40 @@ class BadId(XMLDataException):
         return ("Invalid reference: No " + self.id_name + " with id "
                 + self.val)
 
+# For good measure, some things that can go wrong when we run the test
+# program.
+class TestProgramException(Exception):
+    def getMessage(self):
+        return "Unexpected result from test program"
+
+class ErrorReturnCode(TestProgramException):
+    def getMessage(self):
+        return "Test program returned non-zero exit code"
+
+class MissingOutput(TestProgramException):
+    def __init__(self, expected):
+        self.expected = expected
+
+    def getMessage(self):
+        return "Test program exited early.  Expected output:\n" + self.expected
+
+class WrongOutput(TestProgramException):
+    def __init__(self, expected, actual):
+        self.expected = expected
+        self.actual = actual
+
+    def getMessage(self):
+        return ("Test program printed wrong output.  Expected:\n" + self.expected
+                + "\nbut got:\n" + self.actual)
+
+class ExtraOutput(TestProgramException):
+    def __init__(self, extra):
+        self.extra = extra
+
+    def getMessage(self):
+        return ("Test program printed unexpected additional output:\n"
+                + self.extra)
+
 # A plug-in includes a source file and id.
 class Plugin:
     def __init__(self, plugin_id, source):
@@ -313,6 +347,30 @@ def compileTestcase(working_dir, target_source, hooks_source, plugin_libs):
 
     return executable        
 
+# Run the given process (which is the test program) and check that its
+# output matches expected output.  If the run fails or if the output
+# does not match, checkRun raises a TestProgramException.
+def checkRun(test_proc, expected_output):
+    (actual_output, _) = test_proc.communicate()
+
+    if (test_proc.returncode != 0):
+        raise ErrorReturnCode()
+
+    actual_array = actual_output.strip().splitlines()
+    expected_array = expected_output.strip().splitlines()
+
+    for i in range(len(expected_array)):
+        if (i >= len(actual_array)):
+            raise MissingOutput(expected_array[i].strip())
+
+        if (expected_array[i].strip() != actual_array[i].strip()):
+            raise WrongOutput(expected_array[i].strip(),
+                              actual_array[i].strip())
+
+    assert len(actual_array) >= len(expected_array)
+    if (len(actual_array) > len(expected_array)):
+        raise ExtraOutput(actual_array[len(expected_array)].strip())
+
 # Compile the run's target program with all the requested plug-ins
 # then run the resulting executable and check that its output is as
 # expected.
@@ -337,11 +395,18 @@ def doRun(run):
                                       run.hooks_source, plugin_libs)
 
     if test_executable is not None:
-        test_proc = subprocess.Popen([test_executable])
-        test_proc.wait()
+        test_proc = subprocess.Popen([test_executable], stdout = subprocess.PIPE)
+        try:
+            checkRun(test_proc, run.output)
+        except TestProgramException as e:
+            print e.getMessage()
+            return False
+    else:
+        return False
 
     # Delete temp directory
     shutil.rmtree(path = tmp_dir, ignore_errors = True)    
+    return True
 
 def usage():
     sys.stderr.write(

diff --git a/test/test-driver.c b/test/test-driver.c
index a45479775c8003d21855cc9416af7d3fcf780031..f3adde8a1a38dce2cc4696b57b954e0ea595c670 100644 (file)
--- a/test/test-driver.c
+++ b/test/test-driver.c
@@ -3,19 +3,8 @@
 
 #include "test-driver.h"
 
-void check_printf(const char *fmt, ...)
-{
-  va_list args;
-
-  va_start(args, fmt);
-  vfprintf(stderr, fmt, args);
-  va_end(args);
-}
-
 int main()
 {
-  printf("In test driver!\n");
-
   run_test();
 
   return 0;