diff options
-rw-r--r-- | testtools/testrunner.py | 171 |
1 files changed, 171 insertions, 0 deletions
diff --git a/testtools/testrunner.py b/testtools/testrunner.py new file mode 100644 index 0000000..9d3aee8 --- /dev/null +++ b/testtools/testrunner.py @@ -0,0 +1,171 @@ +#!/bin/python +# +# Testcase runner for NetSurf projects +# +# Usage: testrunner <builddir> <testdir> <prefix> [<executable extension>] +# +# Operates upon INDEX files described in the README. +# Locates and executes testcases, feeding data files to programs +# as appropriate. +# Logs testcase output to file. +# Aborts test sequence on detection of error. +# + +import os +import sys +import Queue +import threading +import subprocess + +global builddir +global directory +global prefix +global exeext + +if len(sys.argv) < 4: + print("Usage: testrunner.pl <builddir> <testdir> <prefix> [<exeext>]") + sys.exit(1) + +# Get directories +builddir = sys.argv[1] +directory = sys.argv[2] +prefix = sys.argv[3] + +# Get EXE extension (if any) +exeext = None +if len(sys.argv) > 4: + exeext = sys.argv[4] + +# Open log file and /dev/null +LOG = open(os.path.join(builddir, "testlog"), "w") + + +def run_test(test_exe, test_data): + io_q = Queue.Queue() + + def output_collector(identifier, stream): + for line in stream: + io_q.put((identifier, line.rstrip())) + + if not stream.closed: + stream.close() + + # Run the test + proc = subprocess.Popen([test_exe, test_data], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Threads for sticking output into queue + tout = threading.Thread(target=output_collector, name='stdout-watcher', + args=('STDOUT', proc.stdout)) + terr = threading.Thread(target=output_collector, name='stderr-watcher', + args=('STDERR', proc.stderr)) + + tout.start() + terr.start() + + # Wait for test to finish + ret = proc.wait() + + # And join the output collector threads + tout.join() + terr.join() + + # If last line of STDOUT is "PASS", then the test passed + last_line = None + + # The STDERR output is just collected up and put at the end of LOG + error_lines = [] + + while not io_q.empty(): + identifier, line = io_q.get() + if identifier == "STDOUT": + LOG.write("%s\n" % line) + last_line = line + else: + error_lines += line + + # Check for test process returning error + if ret != 0: + LOG.write(" FAIL: Exit status %i\n" % ret) + last_line = "FAIL" + + # Dump stderr to log + for error_line in error_lines: + LOG.write("%s\n" % line) + + if not last_line == "PASS": + last_line = "FAIL" + + # Print rest result + print(last_line) + return + +# Open testcase index +with open(os.path.join(directory, "INDEX"), "r") as TINDEX: + # Parse testcase index, looking for testcases + for line in TINDEX: + # Skip comments + if line[0] == '#' or line.isspace(): + continue + + # Found one; decompose + decomposed = filter(None, line.split('\t')) + + # Strip whitespace + test = decomposed[0].strip() + desc = decomposed[1].strip() + data = None + if len(decomposed) > 2: + data = decomposed[2].strip() + + # Append prefix & EXE extension to binary name + test = prefix + test + if exeext: + test += exeext + + print("Test: " + desc) + + if data: + # Testcase has external data files + + # Open datafile index + with open(os.path.join(directory, "data/" + data + "/INDEX"), "r") as DINDEX: + # Parse datafile index, looking for datafiles + for dline in DINDEX: + if dline[0] == '#' or dline.isspace(): + continue + + # Found one; decompose + ddecomposed = filter(None, dline.split('\t')) + + # Strip whitespace + dtest = ddecomposed[0].strip() + ddesc = ddecomposed[1].strip() + + LOG.write("Running %s %s\n" % ( + os.path.join(builddir, test), + os.path.join(directory, "data/" + data + "/" + dtest))) + + # Make message fit on an 80 column terminal + msg = " ==> " + test + " [" + data + "/" + dtest + "]" + msg += "." * (80 - len(msg) - 8) + + sys.stdout.write(msg) + + # Run testcase + run_test(os.path.join(builddir, test), + os.path.join(directory, "data/" + data + "/" + dtest)) + else: + # Testcase has no external data files + LOG.write("Running %s\n" % (os.path.join(builddir, test))) + + # Make message fit on an 80 column terminal + msg = " ==> " + test + msg += "." * (80 - len(msg) - 8) + + sys.stdout.write(msg) + + # Run testcase + run_test(os.path.join(builddir, test)) + + print("") |