From: Justin Seyster <jseyster@cs.sunysb.edu>
Date: Mon, 22 Mar 2010 23:48:59 +0000 (-0400)
Subject: Added Python test case framework.
X-Git-Tag: release-v1.0~120
X-Git-Url: https://git.fsl.cs.stonybrook.edu/?a=commitdiff_plain;h=c7778398490d6a7b2b2d9905f23d5057a4f7d42c;p=interaspect.git

Added Python test case framework.
---

diff --git a/test/run-testcase.py b/test/run-testcase.py
new file mode 100755
index 0000000..e0966f9
--- /dev/null
+++ b/test/run-testcase.py
@@ -0,0 +1,613 @@
+#!/usr/bin/env python
+
+import errno
+import getopt
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+from xml.sax import handler
+from xml.sax import make_parser
+from xml.sax.handler import feature_namespaces
+
+gcc_path = 'gcc'
+gcc_interaspect_lib = None
+gcc_interaspect_src = None
+
+# A whole host of specialized exceptions for things that can go
+# horribly wrong with the input XML test case.
+class XMLDataException(Exception):
+    def getMessage(self):
+        return "Fatal error reading test case description";
+
+class MissingAttr(XMLDataException):
+    def __init__(self, element, attr):
+        self.element = element
+        self.attr = attr
+
+    def getMessage(self):
+        return ("Element <" + self.element 
+                + "> missing required attribute: " + self.attr);
+
+class MisplacedElement(XMLDataException):
+    def __init__(self, element, parent):
+        self.element = element
+        self.parent = parent
+
+    def getMessage(self):
+        return "Element <" + self.element + "> outside of <" + self.parent + ">"
+
+class BadName(XMLDataException):
+    def __init__(self, attr, val):
+        self.attr = attr
+        self.val = val
+
+    def getMessage(self):
+        return ("Value of " + self.attr + " not a legal C function name: "
+                + self.val)
+
+class BadId(XMLDataException):
+    def __init__(self, id_name, val):
+        self.id_name = id_name
+        self.val = val
+
+    def getMessage(self):
+        return ("Invalid reference: No " + self.id_name + " with id "
+                + self.val)
+
+class DuplicateHook(XMLDataException):
+    def __init__(self, run_name, hook_name):
+        self.run_name = run_name
+        self.hook_name = hook_name
+
+    def getMessage(self):
+        return (self.run_name + ": Run contains two hooks with same name, "
+                + self.hook_name + ", but different arguments")
+
+# A plug-in includes a source file and the descriptions of each hook
+# that the plug-in might add a call for.
+class Plugin:
+    def __init__(self, plugin_id, source):
+        self.plugin_id = plugin_id
+        self.source = source
+        self.hooks = {}
+
+# A hook description comprises the C function name for the hook and
+# the types for all its arguments.
+class Hook:
+    def __init__(self, name):
+        self.name = name
+        self.arg_list = []
+
+class Arg:
+    def __init__(self, arg_id, arg_type):
+        self.arg_id = arg_id
+        self.arg_type = arg_type
+
+# A run has a target source file along with a list of plug-ins that
+# should be used when compiling the target and the hooks necessary for
+# the target to compile with those plug-ins.  It also has the full
+# output that the target is supposed to produce (because of the
+# plug-in) when it runs.
+class Run:
+    def __init__(self, name, target_source):
+        self.name = name;
+        self.target_source = target_source;
+        self.plugin_list = []
+        self.hooks = {}
+        self.output = []
+
+# A Value is one entry in the output list for a Run object.
+class Value:
+    def __init__(self, val_type, val):
+        self.val_type = val_type
+        self.val = val
+
+# Called for two hooks with the same name, return True if they have
+# _exactly_ the same arguments.
+def hooksMatch(hook1, hook2):
+    assert hook1.name == hook2.name
+
+    if len(hook1.arg_list) != len(hook2.arg_list):
+        return False
+
+    for i in range(len(hook1.arg_list)):
+        arg1 = hook1.arg_list[i]
+        arg2 = hook2.arg_list[i]
+        if arg1.arg_id != arg2.arg_id or arg1.arg_type != arg2.arg_type:
+            return False
+
+    return True
+
+class TestcaseHandler(handler.ContentHandler):
+    plugins = {}
+    current_plugin = None
+    current_hook = None
+
+    run_list = []
+    current_run = None
+    current_call_hook = None
+
+    current_value_arg_id = None
+    current_value_cdata = ""
+
+    def isAllowedInC(self, token):
+        if not re.match('[a-zA-Z_][0-9a-zA-Z_]*', token):
+            return False
+        elif token == 'if':
+            return False
+        elif token == 'else':
+            return False
+        elif token == 'for':
+            return False
+        elif token == 'do':
+            return False
+        elif token == 'while':
+            return False
+        elif token == 'void':
+            return False
+        elif token == 'unsigned':
+            return False
+        elif token == 'short':
+            return False
+        elif token == 'long':
+            return False
+        elif token == 'int':
+            return False
+        elif token == 'float':
+            return False
+        elif token == 'double':
+            return False
+        else:
+            return True
+
+    def startTestcase(self, attrs):
+        self.name = attrs.get('name')
+        if self.name is None:
+            raise MissingAttr("testcase", "name")
+
+    def startPlugin(self, attrs):
+        plugin_id = attrs.get('id')
+        if plugin_id is None:
+            raise MissingAttr("plugin", "id")
+
+        source = attrs.get('source')
+        if source is None:
+            raise MissingAttr("plugin", "source")
+
+        self.current_plugin = Plugin(plugin_id, source)
+
+    def endPlugin(self):
+        self.plugins[self.current_plugin.plugin_id] = (self.current_plugin)
+        self.current_plugin = None
+
+    def startHook(self, attrs):
+        if self.current_hook is not None:
+            raise MisplacedElement("hook", "plugin");
+
+        name = attrs.get('name')
+        if name is None:
+            raise MissingAttr("hook", "name")
+        elif not self.isAllowedInC(name):
+            raise BadName("name", name)
+
+        self.current_hook = Hook(name)
+
+    def endHook(self):
+        if self.current_plugin is None:
+            raise MisplacedElement("hook", "plugin")
+
+        self.current_plugin.hooks[self.current_hook.name] = self.current_hook
+        self.current_hook = None
+
+    def startArg(self, attrs):
+        if self.current_hook is None:
+            raise MisplacedElement("arg", "hook")
+
+        arg_id = attrs.get('id')
+        if arg_id is None:
+            raise MissingAttr("arg", "id")
+
+        arg_type = attrs.get('type')
+        if arg_type is None:
+            raise MissingAttr("arg", "type")
+
+        arg = Arg(arg_id, arg_type)
+        self.current_hook.arg_list.append(arg)
+
+    def startRun(self, attrs):
+        if self.current_plugin is not None:
+            raise MisplacedElement("run", "testcase")
+
+        run_name = attrs.get('name')
+        if run_name is None:
+            raise MissingAttr("run", "name")
+
+        target_source = attrs.get('target')
+        if target_source is None:
+            raise MissingAttr("run", "target")
+
+        self.current_run = Run(run_name, target_source)
+
+    def endRun(self):
+        self.run_list.append(self.current_run)
+        self.current_run = None
+
+    def startUsing(self, attrs):
+        if self.current_run is None:
+            raise MisplacedElement("using", "run")
+
+        plugin_id = attrs.get('plugin')
+        if plugin_id is None:
+            raise MissingAttr("using", "plugin")
+
+        try:
+            plugin = self.plugins[plugin_id]
+        except KeyError as e:
+            raise BadId("plugin", plugin_id)
+
+        # Add all of this plug-in's hooks to the run.
+        for name, hook in plugin.hooks.iteritems():
+            dup_hook = None
+            try:
+                dup_hook = self.current_run.hooks[name]
+            except KeyError as e:
+                pass
+
+            if dup_hook is None:
+                # We don't have any hooks with this name.
+                self.current_run.hooks[name] = hook
+            else:
+                # There is already a hook with this name.  Make sure
+                # it matches.
+                if not hooksMatch(hook, dup_hook):
+                    raise DuplicateHook(self.current_run.name, name)
+
+        # And the plug-in itself!
+        self.current_run.plugin_list.append(plugin)
+
+    def startCall(self, attrs):
+        if self.current_run is None or self.current_call_hook is not None:
+            raise MisplacedElement("call", "run")
+
+        hook_name = attrs.get('name')
+        if hook_name is None:
+            raise MissingAttr("call", "name")
+
+        try:
+            hook = self.current_run.hooks[hook_name]
+        except KeyError as e:
+            raise BadId("hook", hook_name)
+
+        self.current_call_hook = hook
+
+        output_val = Value('Hook', hook_name)
+        self.current_run.output.append(output_val)
+
+    def endCall(self):
+        self.current_call_hook = None
+
+    def startValue(self, attrs):
+        if (self.current_call_hook is None
+            or self.current_value_arg_id is not None):
+            raise MisplacedElement("value", "call")
+
+        arg_id = attrs.get('id')
+        if arg_id is None:
+            raise MissingAttr("value", "id")
+
+        # We can't examine the cdata until the endValue event.  Stash
+        # the argument name until then.
+        self.current_value_arg_id = arg_id
+        self.current_value_cdata = ""
+
+    def endValue(self):
+        assert self.current_call_hook is not None
+        assert self.current_value_arg_id is not None
+
+        # Find the arg with the given arg id.  These arguments are not
+        # stored in an associative array because their order is
+        # important.  These lists are way too small to justify the
+        # overhead of a separate index.
+        arg = None
+        for arg_it in self.current_call_hook.arg_list:
+            if self.current_value_arg_id == arg_it.arg_id:
+                arg = arg_it
+        if arg is None:
+            raise BadId("arg", self.current_value_arg_id)
+
+        new_value = Value(arg.arg_type, self.current_value_cdata)
+        self.current_run.output.append(new_value)
+
+        self.current_value_arg_id = None
+
+    # Parsing with SAX is simple but tedious.  There's a start
+    # function for each tag and an end function for some tags.  They
+    # just stash the data they find into a relevant data structure
+    # (after validating that it makes sense).
+    def startElement(self, name, attrs):
+        if name == 'testcase':
+            self.startTestcase(attrs)
+        elif name == 'plugin':
+            self.startPlugin(attrs)
+        elif name == 'hook':
+            self.startHook(attrs)
+        elif name == 'arg':
+            self.startArg(attrs)
+        elif name == 'run':
+            self.startRun(attrs)
+        elif name == 'using':
+            self.startUsing(attrs)
+        elif name == 'call':
+            self.startCall(attrs)
+        elif name == 'value':
+            self.startValue(attrs)
+
+    def endElement(self, name):
+        if name == 'plugin':
+            self.endPlugin()
+        elif name == 'hook':
+            self.endHook()
+        elif name == 'run':
+            self.endRun()
+        elif name == 'call':
+            self.endCall()
+        elif name == 'value':
+            self.endValue()
+
+    def characters(self, chars):
+        if self.current_value_arg_id is not None:
+            self.current_value_cdata += chars
+
+def getCheckFormat(c_type):
+    c_type = c_type.strip()
+    if re.match(r".*\bchar\s+\*", c_type):
+        return r"string: %s\n"
+    if re.match(r".*\*$", c_type):
+        return r"pointer: %p\n"
+    elif c_type == 'int':
+        return r"int: %d\n"
+    else:
+        return None
+
+# Print the C prototype and function body for the given plug-in hook
+# to the given stream.
+def printHook(stream, hook):
+    params = ['{0:s} arg{1:d}'.format(hook.arg_list[i].arg_type, i) for i in
+              range(len(hook.arg_list))]
+    param_text = ', '.join(params)
+
+    stream.write('void {0:s}({1:s})\n'.format(hook.name, param_text))
+    stream.write('{\n');
+    stream.write('  check_printf("hook: {0:s}\\n");\n'.format(hook.name))
+    for i in range(len(hook.arg_list)):
+        arg = hook.arg_list[i]
+        check_format = getCheckFormat(arg.arg_type)
+        if check_format is not None:
+            stream.write('  check_printf("{0:s}", arg{1:d});\n'.format(check_format, i))
+    stream.write('}\n');
+
+# Run GCC with the given arguments.  On failure, print an appropriate
+# error and return False.
+# If the compile fails because of an error in the C file, runGCC also
+# prints compile_fail_msg.
+def runGCC(args, compile_fail_msg):
+    args = [gcc_path] + args
+    try:
+        gcc_proc = subprocess.Popen(args, stderr = subprocess.PIPE)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            print "Fatal -- Bad path to GCC:", gcc_path
+            return False
+        else:
+            print "Fatal error launching GCC:"
+            print e.strerror
+            return False
+    (stdoutdata, stderrdata) = gcc_proc.communicate();
+    gcc_proc.wait();
+
+    if gcc_proc.returncode != 0:
+        print compile_fail_msg
+        print stderrdata
+        return False
+
+    return True
+
+# Compile and link an InterAspect plug-in!  Several compilation files
+# get created in the given working directory.  It is the caller's
+# responsibility to delete these files.  No files will be left over in
+# other directories, though.
+#
+# The compiled plug-in file will be named plugin_base_name.so.1.0.0.
+#
+# On success, compilePlugin returns the path to the final plug-in .so
+# file (which will be in working_dir).  On failure, the return value
+# is None.
+def compilePlugin(working_dir, plugin_id, plugin_base_name, plugin_source):
+    plugin_lib_name = '{0:s}/{1:s}.so.1.0.0'.format(working_dir,
+                                                    plugin_base_name)
+
+
+    include_flag = '-I{0:s}/src'.format(gcc_interaspect_src)
+    cmd_args = ['-Wall', '-Werror', include_flag, '-fPIC', '-shared',
+                '-Wl,-soname,{0:s}.so.1'.format(plugin_base_name), '-o',
+                plugin_lib_name, plugin_source, gcc_interaspect_lib]
+    result = runGCC(cmd_args,
+      'Fatal -- Failed to compile plugin: "{0:s}"'.format(plugin_id))
+    if not result:
+        return None
+
+    return plugin_lib_name
+
+# Compile the testcase instrumentation target, along with
+# auto-generated hook functions.  Several compilation files get
+# created in the given working directory.  It is the caller's
+# responsibility to delete these files.  No files will be left over in
+# other directories, though.
+#
+# working_dir: A temporary directory to store intermediate files and
+# the resulting executable.
+#
+# target_source: The C source file for the target program (the program
+# that we intend to instrument with the test plug-ins).
+#
+# plugin_libs: A list of .so plug-in files that will be used to
+# compile the target program.
+#
+# On success, compileTestcase returns the path to the final test
+# executable (which will be in working_dir).  On failure, the return
+# value is None.
+def compileTestcase(working_dir, target_source, plugin_libs, hooks):
+    # Create a C file with the necessary plug-in hooks and compile it.
+    hook_file_name = working_dir + '/hooks.c'
+    hook_file = open(hook_file_name, 'w');
+    hook_file.write('#include "test-driver.h"\n')
+    for name, hook in hooks.iteritems():
+        printHook(hook_file, hook)
+    hook_file.close()
+
+    test_include = '-I{0:s}/test'.format(gcc_interaspect_src)
+    hook_o_file = working_dir + '/hooks.o'
+    cmd_args = ['-Wall', '-Werror', test_include, '-c', '-o', hook_o_file,
+                hook_file_name]
+    result = runGCC(cmd_args, "Fatal -- Failed to compile plug-in hooks:")
+    if not result:
+        return None
+
+    # Compile the test driver, which has the main function.
+    main_c_file = gcc_interaspect_src + 'test/test-driver.c'
+    main_o_file = working_dir + '/test-driver.o'
+    cmd_args = ['-Wall', '-Werror', '-c', '-o', main_o_file, main_c_file]
+    result = runGCC(cmd_args, "Fatal -- Failed to compile test driver:")
+    if not result:
+        return None
+
+    # Compile the target itself.
+    target_o_file = working_dir + '/target.o'
+    cmd_args = ['-fplugin={0:s}'.format(lib) for lib in plugin_libs]
+    cmd_args += ['-Wall', '-Werror', '-c', '-o', target_o_file, target_source]
+    result = runGCC(cmd_args, "Fatal -- Failed to compile target source:")
+    if not result:
+        return None
+
+    # Link the final binary
+    executable = working_dir + '/test-executable'
+    cmd_args = ['-o', executable, hook_o_file, main_o_file, target_o_file]
+    result = runGCC(cmd_args, "Fatal -- Failed to link test case:")
+    if not result:
+        return None
+
+    return executable        
+
+# Compile the run's target program with all the requested plug-ins
+# then run the resulting executable and check that its output is as
+# expected.
+# Returns True if the run passes.
+def doRun(run):
+    print "  Run:", run.name
+
+    tmp_dir = tempfile.mkdtemp(prefix='test-out-')
+
+    # Compile all the plug-ins for this test.
+    plugin_libs = []
+    for i in range(len(run.plugin_list)):
+        plugin = run.plugin_list[i]
+        plugin_base_name = 'test_plugin_{0:d}'.format(i + 1)
+        plugin_lib_name = compilePlugin(tmp_dir, plugin.plugin_id,
+                                        plugin_base_name, plugin.source)
+        if plugin_lib_name is None:
+            return False
+        plugin_libs.append(plugin_lib_name)
+
+    test_executable = compileTestcase(tmp_dir, run.target_source, plugin_libs,
+                                      run.hooks)
+
+    if test_executable is not None:
+        test_proc = subprocess.Popen([test_executable])
+        test_proc.wait()
+
+    # Delete temp directory
+    shutil.rmtree(path = tmp_dir, ignore_errors = True)    
+
+def usage():
+    sys.stderr.write(
+"""Usage:
+  run-testcase.py [options] testcase.xml
+
+Options
+  --with-gcc: Path to gcc to compile the tests with
+  --with-ia-lib-dir: Directory with InterAspect library file
+  --with-ia-src-dir: Directory with InterAspect source
+""")
+    # End function usage()
+
+# Return True if filename exists and is readable.  If it is not, print
+# an appropriate error message and return False.
+def checkFileReadable(filename):
+    try:
+        filehandle = open(filename, 'r')
+        filehandle.close()
+    except IOError as e:
+        sys.stderr.write('{0:s}: {1:s}\n'.format(filename, e.strerror))
+        return False
+
+    return True
+
+if __name__ == '__main__':
+    # Deal with command line arguments
+    try:
+        long_switches = ["with-gcc=", "with-ia-lib-dir=", "with-ia-src-dir="]
+        opts, args = getopt.gnu_getopt(sys.argv[1:], "", long_switches)
+    except getopt.GetoptError:
+        usage()
+        sys.exit(1)
+
+    for opt, arg in opts:
+        if (opt == "--with-gcc"):
+            gcc_path = arg
+        elif (opt == "--with-ia-lib-dir"):
+            gcc_interaspect_lib = arg + '/libinteraspect.a'
+        elif (opt == "--with-ia-src-dir"):
+            gcc_interaspect_src = arg
+        else:
+            assert(1)
+
+    # Validate args
+    if gcc_interaspect_lib is None:
+        sys.stderr.write("Must specify --with-ia-lib-dir\n")
+        sys.exit(1)
+    elif not checkFileReadable(gcc_interaspect_lib):
+        sys.exit(1)
+    elif gcc_interaspect_src is None:
+        sys.stderr.write("Must specify --with-ia-src-dir\n")
+        sys.exit(1)
+    elif not checkFileReadable('{0:s}/src/aop.h'.format(gcc_interaspect_src)):
+        sys.exit(1)
+
+    # There should be exactly one non-option arg: the test case
+    if len(args) != 1:
+        usage()
+        sys.exit(1)
+    test_description = args[0]
+
+    parser = make_parser()
+    parser.setFeature(feature_namespaces, 0)
+
+    dh = TestcaseHandler()
+    parser.setContentHandler(dh)
+
+    try:
+        parser.parse(test_description)
+    except IOError as e:
+        sys.stderr.write('{0:s}: {1:s}\n'.format(test_description, e.strerror))
+        sys.exit(1)
+    except XMLDataException as e:
+        print e.getMessage()
+        sys.exit(1)
+
+    print "Testcase:", dh.name
+    for run in dh.run_list:
+        doRun(run)
diff --git a/test/test-driver.c b/test/test-driver.c
new file mode 100644
index 0000000..a454797
--- /dev/null
+++ b/test/test-driver.c
@@ -0,0 +1,22 @@
+#include <stdarg.h>
+#include <stdio.h>
+
+#include "test-driver.h"
+
+void check_printf(const char *fmt, ...)
+{
+  va_list args;
+
+  va_start(args, fmt);
+  vfprintf(stderr, fmt, args);
+  va_end(args);
+}
+
+int main()
+{
+  printf("In test driver!\n");
+
+  run_test();
+
+  return 0;
+}