tests,ext: Add a new testing library proposal

The new test library is split into two parts: The framework which resides
in ext/, and the gem5 helping components in /tests/gem5.

Change-Id: Ib4f3ae8d7eb96a7306335a3e739b7e8041aa99b9
Signed-off-by: Sean Wilson <spwilson2@wisc.edu>
Reviewed-on: https://gem5-review.googlesource.com/4421
Reviewed-by: Giacomo Travaglini <giacomo.travaglini@arm.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

ext/testlib/__init__.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+
+from .state import *
+from .runner import *
+from .test import *
+from .suite import *
+from .loader import *
+from .fixture import *
+from .config import *
+from main import main
+
+#TODO Remove this awkward bootstrap
+#FIXME
+from gem5 import *
+
+#TODO Remove this as an export, users should getcwd from os
+from os import getcwd

ext/testlib/config.py

@@ -0,0 +1,687 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Global configuration module which exposes two types of configuration
+variables:
+
+1. config
+2. constants (Also attached to the config variable as an attribute)
+
+The main motivation for this module is to have a centralized location for
+defaults and configuration by command line and files for the test framework.
+
+A secondary goal is to reduce programming errors by providing common constant
+strings and values as python attributes to simplify detection of typos.
+A simple typo in a string can take a lot of debugging to uncover the issue,
+attribute errors are easier to notice and most autocompletion systems detect
+them.
+
+The config variable is initialzed by calling :func:`initialize_config`.
+Before this point only ``constants`` will be availaible. This is to ensure
+that library function writers never accidentally get stale config attributes.
+
+Program arguments/flag arguments are available from the config as attributes.
+If an attribute was not set by the command line or the optional config file,
+then it will fallback to the `_defaults` value, if still the value is not
+found an AttributeError will be raised.
+
+:func define_defaults:
+    Provided by the config if the attribute is not found in the config or
+    commandline. For instance, if we are using the list command fixtures might
+    not be able to count on the build_dir being provided since we aren't going
+    to build anything.
+
+:var constants:
+    Values not directly exposed by the config, but are attached to the object
+    for centralized access. I.E. you can reach them with
+    :code:`config.constants.attribute`. These should be used for setting
+    common string names used across the test framework.
+    :code:`_defaults.build_dir = None` Once this module has been imported
+    constants should not be modified and their base attributes are frozen.
+'''
+import abc
+import argparse
+import copy
+import os
+import re
+
+from ConfigParser import ConfigParser
+from pickle import HIGHEST_PROTOCOL as highest_pickle_protocol
+
+from helper import absdirpath, AttrDict, FrozenAttrDict
+
+class UninitialzedAttributeException(Exception):
+    '''
+    Signals that an attribute in the config file was not initialized.
+    '''
+    pass
+
+class UninitializedConfigException(Exception):
+    '''
+    Signals that the config was not initialized before trying to access an
+    attribute.
+    '''
+    pass
+
+class TagRegex(object):
+    def __init__(self, include, regex):
+        self.include = include
+        self.regex = re.compile(regex)
+
+    def __str__(self):
+        type_ = 'Include' if self.include else 'Remove'
+        return '%10s: %s' % (type_, self.regex.pattern)
+
+class _Config(object):
+    _initialized = False
+
+    __shared_dict = {}
+
+    constants = AttrDict()
+    _defaults = AttrDict()
+    _config = {}
+
+    _cli_args = {}
+    _post_processors = {}
+
+    def __init__(self):
+        # This object will act as if it were a singleton.
+        self.__dict__ = self.__shared_dict
+
+    def _init(self, parser):
+        self._parse_commandline_args(parser)
+        self._run_post_processors()
+        self._initialized = True
+
+    def _init_with_dicts(self, config, defaults):
+        self._config = config
+        self._defaults = defaults
+        self._initialized = True
+
+    def _add_post_processor(self, attr, post_processor):
+        '''
+        :param attr: Attribute to pass to and recieve from the
+        :func:`post_processor`.
+
+        :param post_processor: A callback functions called in a chain to
+            perform additional setup for a config argument. Should return a
+            tuple containing the new value for the config attr.
+        '''
+        if attr not in self._post_processors:
+            self._post_processors[attr] = []
+        self._post_processors[attr].append(post_processor)
+
+    def _set(self, name, value):
+        self._config[name] = value
+
+    def _parse_commandline_args(self, parser):
+        args = parser.parse_args()
+
+        self._config_file_args = {}
+
+        for attr in dir(args):
+            # Ignore non-argument attributes.
+            if not attr.startswith('_'):
+                self._config_file_args[attr] = getattr(args, attr)
+        self._config.update(self._config_file_args)
+
+    def _run_post_processors(self):
+        for attr, callbacks in self._post_processors.items():
+            newval = self._lookup_val(attr)
+            for callback in callbacks:
+                newval = callback(newval)
+            if newval is not None:
+                newval = newval[0]
+            self._set(attr, newval)
+
+
+    def _lookup_val(self, attr):
+        '''
+        Get the attribute from the config or fallback to defaults.
+
+        :returns: If the value is not stored return None. Otherwise a tuple
+            containing the value.
+        '''
+        if attr in self._config:
+            return (self._config[attr],)
+        elif hasattr(self._defaults, attr):
+            return (getattr(self._defaults, attr),)
+
+    def __getattr__(self, attr):
+        if attr in dir(super(_Config, self)):
+            return getattr(super(_Config, self), attr)
+        elif not self._initialized:
+            raise UninitializedConfigException(
+                'Cannot directly access elements from the config before it is'
+                ' initialized')
+        else:
+            val = self._lookup_val(attr)
+            if val is not None:
+                return val[0]
+            else:
+                raise UninitialzedAttributeException(
+                    '%s was not initialzed in the config.' % attr)
+
+    def get_tags(self):
+        d = {typ: set(self.__getattr__(typ))
+            for typ in self.constants.supported_tags}
+        if any(map(lambda vals: bool(vals), d.values())):
+            return d
+        else:
+            return {}
+
+def define_defaults(defaults):
+    '''
+    Defaults are provided by the config if the attribute is not found in the
+    config or commandline. For instance, if we are using the list command
+    fixtures might not be able to count on the build_dir being provided since
+    we aren't going to build anything.
+    '''
+    defaults.base_dir = os.path.abspath(os.path.join(absdirpath(__file__),
+                                                      os.pardir,
+                                                      os.pardir))
+    defaults.result_path = os.path.join(os.getcwd(), '.testing-results')
+    defaults.list_only_failed = False
+
+def define_constants(constants):
+    '''
+    'constants' are values not directly exposed by the config, but are attached
+    to the object for centralized access. These should be used for setting
+    common string names used across the test framework. A simple typo in
+    a string can take a lot of debugging to uncover the issue, attribute errors
+    are easier to notice and most autocompletion systems detect them.
+    '''
+    constants.system_out_name = 'system-out'
+    constants.system_err_name = 'system-err'
+
+    constants.isa_tag_type = 'isa'
+    constants.x86_tag = 'X86'
+    constants.sparc_tag = 'SPARC'
+    constants.alpha_tag = 'ALPHA'
+    constants.riscv_tag = 'RISCV'
+    constants.arm_tag = 'ARM'
+    constants.mips_tag = 'MIPS'
+    constants.power_tag = 'POWER'
+    constants.null_tag = 'NULL'
+
+    constants.variant_tag_type = 'variant'
+    constants.opt_tag = 'opt'
+    constants.debug_tag = 'debug'
+    constants.fast_tag = 'fast'
+
+    constants.length_tag_type = 'length'
+    constants.quick_tag = 'quick'
+    constants.long_tag = 'long'
+
+    constants.supported_tags = {
+        constants.isa_tag_type : (
+            constants.x86_tag,
+            constants.sparc_tag,
+            constants.alpha_tag,
+            constants.riscv_tag,
+            constants.arm_tag,
+            constants.mips_tag,
+            constants.power_tag,
+            constants.null_tag,
+            ),
+        constants.variant_tag_type: (
+            constants.opt_tag,
+            constants.debug_tag,
+            constants.fast_tag,
+        ),
+        constants.length_tag_type: (
+            constants.quick_tag,
+            constants.long_tag,
+        ),
+    }
+
+    constants.supported_isas = constants.supported_tags['isa']
+    constants.supported_variants = constants.supported_tags['variant']
+    constants.supported_lengths = constants.supported_tags['length']
+
+    constants.tempdir_fixture_name = 'tempdir'
+    constants.gem5_simulation_stderr = 'simerr'
+    constants.gem5_simulation_stdout = 'simout'
+    constants.gem5_simulation_stats = 'stats.txt'
+    constants.gem5_simulation_config_ini = 'config.ini'
+    constants.gem5_simulation_config_json = 'config.json'
+    constants.gem5_returncode_fixture_name = 'gem5-returncode'
+    constants.gem5_binary_fixture_name = 'gem5'
+    constants.xml_filename = 'results.xml'
+    constants.pickle_filename = 'results.pickle'
+    constants.pickle_protocol = highest_pickle_protocol
+
+    # The root directory which all test names will be based off of.
+    constants.testing_base = absdirpath(os.path.join(absdirpath(__file__),
+                                                     os.pardir))
+
+def define_post_processors(config):
+    '''
+    post_processors are used to do final configuration of variables. This is
+    useful if there is a dynamically set default, or some function that needs
+    to be applied after parsing in order to set a configration value.
+
+    Post processors must accept a single argument that will either be a tuple
+    containing the already set config value or ``None`` if the config value
+    has not been set to anything. They must return the modified value in the
+    same format.
+    '''
+
+    def set_default_build_dir(build_dir):
+        '''
+        Post-processor to set the default build_dir based on the base_dir.
+
+        .. seealso :func:`~_Config._add_post_processor`
+        '''
+        if not build_dir or build_dir[0] is None:
+            base_dir = config._lookup_val('base_dir')[0]
+            build_dir = (os.path.join(base_dir, 'build'),)
+        return build_dir
+
+    def fix_verbosity_hack(verbose):
+        return (verbose[0].val,)
+
+    def threads_as_int(threads):
+        if threads is not None:
+            return (int(threads[0]),)
+
+    def test_threads_as_int(test_threads):
+        if test_threads is not None:
+            return (int(test_threads[0]),)
+
+    def default_isa(isa):
+        if not isa[0]:
+            return [constants.supported_tags[constants.isa_tag_type]]
+        else:
+            return isa
+
+    def default_variant(variant):
+        if not variant[0]:
+            # Default variant is only opt. No need to run tests with multiple
+            # different compilation targets
+            return [[constants.opt_tag]]
+        else:
+            return variant
+
+    def default_length(length):
+        if not length[0]:
+            return [[constants.quick_tag]]
+        else:
+            return length
+
+    def compile_tag_regex(positional_tags):
+        if not positional_tags:
+            return positional_tags
+        else:
+            new_positional_tags_list = []
+            positional_tags = positional_tags[0]
+
+            for flag, regex in positional_tags:
+                if flag == 'exclude_tags':
+                    tag_regex = TagRegex(False, regex)
+                elif flag  == 'include_tags':
+                    tag_regex = TagRegex(True, regex)
+                else:
+                    raise ValueError('Unsupported flag.')
+                new_positional_tags_list.append(tag_regex)
+
+            return (new_positional_tags_list,)
+
+    config._add_post_processor('build_dir', set_default_build_dir)
+    config._add_post_processor('verbose', fix_verbosity_hack)
+    config._add_post_processor('isa', default_isa)
+    config._add_post_processor('variant', default_variant)
+    config._add_post_processor('length', default_length)
+    config._add_post_processor('threads', threads_as_int)
+    config._add_post_processor('test_threads', test_threads_as_int)
+    config._add_post_processor(StorePositionalTagsAction.position_kword,
+                               compile_tag_regex)
+class Argument(object):
+    '''
+    Class represents a cli argument/flag for a argparse parser.
+
+    :attr name: The long name of this object that will be stored in the arg
+        output by the final parser.
+    '''
+    def __init__(self, *flags, **kwargs):
+        self.flags = flags
+        self.kwargs = kwargs
+
+        if len(flags) == 0:
+            raise ValueError("Need at least one argument.")
+        elif 'dest' in kwargs:
+            self.name = kwargs['dest']
+        elif len(flags) > 1 or flags[0].startswith('-'):
+            for flag in flags:
+                if not flag.startswith('-'):
+                    raise ValueError("invalid option string %s: must start"
+                    "with a character '-'" % flag)
+
+                if flag.startswith('--'):
+                    if not hasattr(self, 'name'):
+                        self.name = flag.lstrip('-')
+
+        if not hasattr(self, 'name'):
+            self.name = flags[0].lstrip('-')
+        self.name = self.name.replace('-', '_')
+
+    def add_to(self, parser):
+        '''Add this argument to the given parser.'''
+        parser.add_argument(*self.flags, **self.kwargs)
+
+    def copy(self):
+        '''Copy this argument so you might modify any of its kwargs.'''
+        return copy.deepcopy(self)
+
+
+class _StickyInt:
+    '''
+    A class that is used to cheat the verbosity count incrementer by
+    pretending to be an int. This makes the int stay on the heap and eat other
+    real numbers when they are added to it.
+
+    We use this so we can allow the verbose flag to be provided before or after
+    the subcommand. This likely has no utility outside of this use case.
+    '''
+    def __init__(self, val=0):
+        self.val = val
+        self.type = int
+    def __add__(self, other):
+        self.val += other
+        return self
+
+common_args = NotImplemented
+
+class StorePositionAction(argparse.Action):
+    '''Base class for classes wishing to create namespaces where
+    arguments are stored in the order provided via the command line.
+    '''
+    position_kword = 'positional'
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not self.position_kword in namespace:
+            setattr(namespace, self.position_kword, [])
+        previous = getattr(namespace, self.position_kword)
+        previous.append((self.dest, values))
+        setattr(namespace, self.position_kword, previous)
+
+class StorePositionalTagsAction(StorePositionAction):
+    position_kword = 'tag_filters'
+
+def define_common_args(config):
+    '''
+    Common args are arguments which are likely to be simular between different
+    subcommands, so they are available to all by placing their definitions
+    here.
+    '''
+    global common_args
+
+    # A list of common arguments/flags used across cli parsers.
+    common_args = [
+        Argument(
+            'directory',
+            nargs='?',
+            default=os.getcwd(),
+            help='Directory to start searching for tests in'),
+        Argument(
+            '--exclude-tags',
+            action=StorePositionalTagsAction,
+            help='A tag comparison used to select tests.'),
+        Argument(
+            '--include-tags',
+            action=StorePositionalTagsAction,
+            help='A tag comparison used to select tests.'),
+        Argument(
+            '--isa',
+            action='append',
+            default=[],
+            help="Only tests that are valid with one of these ISAs. "
+                 "Comma separated."),
+        Argument(
+            '--variant',
+            action='append',
+            default=[],
+            help="Only tests that are valid with one of these binary variants"
+                 "(e.g., opt, debug). Comma separated."),
+        Argument(
+            '--length',
+            action='append',
+            default=[],
+            help="Only tests that are one of these lengths. Comma separated."),
+        Argument(
+            '--uid',
+            action='store',
+            default=None,
+            help='UID of a specific test item to run.'),
+        Argument(
+            '--build-dir',
+            action='store',
+            help='Build directory for SCons'),
+        Argument(
+            '--base-dir',
+            action='store',
+            default=config._defaults.base_dir,
+            help='Directory to change to in order to exec scons.'),
+        Argument(
+            '-j', '--threads',
+            action='store',
+            default=1,
+            help='Number of threads to run SCons with.'),
+        Argument(
+            '-t', '--test-threads',
+            action='store',
+            default=1,
+            help='Number of threads to spawn to run concurrent tests with.'),
+        Argument(
+            '-v',
+            action='count',
+            dest='verbose',
+            default=_StickyInt(),
+            help='Increase verbosity'),
+        Argument(
+            '--config-path',
+            action='store',
+            default=os.getcwd(),
+            help='Path to read a testing.ini config in'
+        ),
+        Argument(
+            '--skip-build',
+            action='store_true',
+            default=False,
+            help='Skip the building component of SCons targets.'
+        ),
+        Argument(
+            '--result-path',
+            action='store',
+            help='The path to store results in.'
+        ),
+    ]
+
+    # NOTE: There is a limitation which arises due to this format. If you have
+    # multiple arguments with the same name only the final one in the list
+    # will be saved.
+    #
+    # e.g. if you have a -v argument which increments verbosity level and
+    # a separate --verbose flag which 'store's verbosity level. the final
+    # one in the list will be saved.
+    common_args = AttrDict({arg.name:arg for arg in common_args})
+
+
+class ArgParser(object):
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, parser):
+        # Copy public methods of the parser.
+        for attr in dir(parser):
+            if not attr.startswith('_'):
+                setattr(self, attr, getattr(parser, attr))
+        self.parser = parser
+        self.add_argument = self.parser.add_argument
+
+        # Argument will be added to all parsers and subparsers.
+        common_args.verbose.add_to(parser)
+
+
+class CommandParser(ArgParser):
+    '''
+    Main parser which parses command strings and uses those to direct to
+    a subparser.
+    '''
+    def __init__(self):
+        parser = argparse.ArgumentParser()
+        super(CommandParser, self).__init__(parser)
+        self.subparser = self.add_subparsers(dest='command')
+
+
+class RunParser(ArgParser):
+    '''
+    Parser for the \'run\' command.
+    '''
+    def __init__(self, subparser):
+        parser = subparser.add_parser(
+            'run',
+            help='''Run Tests.'''
+        )
+
+        super(RunParser, self).__init__(parser)
+
+        common_args.uid.add_to(parser)
+        common_args.skip_build.add_to(parser)
+        common_args.directory.add_to(parser)
+        common_args.build_dir.add_to(parser)
+        common_args.base_dir.add_to(parser)
+        common_args.threads.add_to(parser)
+        common_args.test_threads.add_to(parser)
+        common_args.isa.add_to(parser)
+        common_args.variant.add_to(parser)
+        common_args.length.add_to(parser)
+        common_args.include_tags.add_to(parser)
+        common_args.exclude_tags.add_to(parser)
+
+
+class ListParser(ArgParser):
+    '''
+    Parser for the \'list\' command.
+    '''
+    def __init__(self, subparser):
+        parser = subparser.add_parser(
+            'list',
+            help='''List and query test metadata.'''
+        )
+        super(ListParser, self).__init__(parser)
+
+        Argument(
+            '--suites',
+            action='store_true',
+            default=False,
+            help='List all test suites.'
+        ).add_to(parser)
+        Argument(
+            '--tests',
+            action='store_true',
+            default=False,
+            help='List all test cases.'
+        ).add_to(parser)
+        Argument(
+            '--fixtures',
+            action='store_true',
+            default=False,
+            help='List all fixtures.'
+        ).add_to(parser)
+        Argument(
+            '--all-tags',
+            action='store_true',
+            default=False,
+            help='List all tags.'
+        ).add_to(parser)
+        Argument(
+            '-q',
+            dest='quiet',
+            action='store_true',
+            default=False,
+            help='Quiet output (machine readable).'
+        ).add_to(parser)
+
+        common_args.directory.add_to(parser)
+        common_args.isa.add_to(parser)
+        common_args.variant.add_to(parser)
+        common_args.length.add_to(parser)
+        common_args.include_tags.add_to(parser)
+        common_args.exclude_tags.add_to(parser)
+
+
+class RerunParser(ArgParser):
+    def __init__(self, subparser):
+        parser = subparser.add_parser(
+            'rerun',
+            help='''Rerun failed tests.'''
+        )
+        super(RerunParser, self).__init__(parser)
+
+        common_args.skip_build.add_to(parser)
+        common_args.directory.add_to(parser)
+        common_args.build_dir.add_to(parser)
+        common_args.base_dir.add_to(parser)
+        common_args.threads.add_to(parser)
+        common_args.test_threads.add_to(parser)
+        common_args.isa.add_to(parser)
+        common_args.variant.add_to(parser)
+        common_args.length.add_to(parser)
+
+config = _Config()
+define_constants(config.constants)
+
+# Constants are directly exposed and available once this module is created.
+# All constants MUST be defined before this point.
+config.constants = FrozenAttrDict(config.constants.__dict__)
+constants = config.constants
+
+'''
+This config object is the singleton config object available throughout the
+framework.
+'''
+def initialize_config():
+    '''
+    Parse the commandline arguments and setup the config varibles.
+    '''
+    global config
+
+    # Setup constants and defaults
+    define_defaults(config._defaults)
+    define_post_processors(config)
+    define_common_args(config)
+
+    # Setup parser and subcommands
+    baseparser = CommandParser()
+    runparser = RunParser(baseparser.subparser)
+    listparser = ListParser(baseparser.subparser)
+    rerunparser = RerunParser(baseparser.subparser)
+
+    # Initialize the config by parsing args and running callbacks.
+    config._init(baseparser)

ext/testlib/fixture.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import copy
+import traceback
+
+import helper
+import log
+
+global_fixtures = []
+
+class SkipException(Exception):
+    def __init__(self, fixture, testitem):
+        self.fixture = fixture
+        self.testitem = testitem
+
+        self.msg = 'Fixture "%s" raised SkipException for "%s".' % (
+               fixture.name, testitem.name
+        )
+        super(SkipException, self).__init__(self.msg)
+
+
+class Fixture(object):
+    '''
+    Base Class for a test Fixture.
+
+    Fixtures are items which possibly require setup and/or tearing down after
+    a TestCase, TestSuite, or the Library has completed.
+
+    Fixtures are the prefered method of carrying incremental results or
+    variables between TestCases in TestSuites. (Rather than using globals.)
+    Using fixtures rather than globals ensures that state will be maintained
+    when executing tests in parallel.
+
+    .. note:: In order for Fixtures to be enumerated by the test system this
+        class' :code:`__new__` method must be called.
+    '''
+    collector = helper.InstanceCollector()
+
+    def __new__(klass, *args, **kwargs):
+        obj = super(Fixture, klass).__new__(klass, *args, **kwargs)
+        Fixture.collector.collect(obj)
+        return obj
+
+    def __init__(self, name=None, **kwargs):
+        if name is None:
+            name = self.__class__.__name__
+        self.name = name
+
+    def skip(self, testitem):
+        raise SkipException(self.name, testitem.metadata)
+
+    def schedule_finalized(self, schedule):
+        '''
+        This method is called once the schedule of for tests is known.
+        To enable tests to use the same fixture defintion for each execution
+        fixtures must return a copy of themselves in this method.
+
+        :returns: a copy of this fixture which will be setup/torndown
+          when the test item this object is tied to is about to execute.
+        '''
+        return self.copy()
+
+    def init(self, *args, **kwargs):
+        pass
+
+    def setup(self, testitem):
+        pass
+
+    def teardown(self, testitem):
+        pass
+
+    def copy(self):
+        return copy.deepcopy(self)
+
+
+def globalfixture(fixture):
+    '''
+    Store the given fixture as a global fixture. Its setup() method
+    will be called before the first test is executed.
+    '''
+    global_fixtures.append(fixture)
+    return fixture

ext/testlib/handlers.py

@@ -0,0 +1,437 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Handlers for the testlib Log.
+
+
+'''
+from __future__ import print_function
+
+import multiprocessing
+import os
+import Queue
+import sys
+import threading
+import time
+import traceback
+
+import helper
+import log
+import result
+import state
+import test
+import terminal
+
+from config import config, constants
+
+
+class _TestStreamManager(object):
+    def __init__(self):
+        self._writers = {}
+
+    def open_writer(self, test_result):
+        if test_result in self._writers:
+            raise ValueError('Cannot have multiple writters on a single test.')
+        self._writers[test_result] = _TestStreams(test_result.stdout,
+                test_result.stderr)
+
+    def get_writer(self, test_result):
+        if test_result not in self._writers:
+            self.open_writer(test_result)
+        return self._writers[test_result]
+
+    def close_writer(self, test_result):
+        if test_result in self._writers:
+            writer = self._writers.pop(test_result)
+            writer.close()
+
+    def close(self):
+        for writer in self._writers.values():
+            writer.close()
+        self._writers.clear()
+
+class _TestStreams(object):
+    def __init__(self, stdout, stderr):
+        helper.mkdir_p(os.path.dirname(stdout))
+        helper.mkdir_p(os.path.dirname(stderr))
+        self.stdout = open(stdout, 'w')
+        self.stderr = open(stderr, 'w')
+
+    def close(self):
+        self.stdout.close()
+        self.stderr.close()
+
+class ResultHandler(log.Handler):
+    '''
+    Log handler which listens for test results and output saving data as
+    it is reported.
+
+    When the handler is closed it writes out test results in the python pickle
+    format.
+    '''
+    def __init__(self, schedule, directory):
+        '''
+        :param schedule: The entire schedule as a :class:`LoadedLibrary`
+            object.
+
+        :param directory: Directory to save test stdout/stderr and aggregate
+            results to.
+        '''
+        self.directory = directory
+        self.internal_results = result.InternalLibraryResults(schedule,
+                directory)
+        self.test_stream_manager = _TestStreamManager()
+        self._closed = False
+
+        self.mapping = {
+            log.LibraryStatus.type_id: self.handle_library_status,
+
+            log.SuiteResult.type_id: self.handle_suite_result,
+            log.TestResult.type_id: self.handle_test_result,
+
+            log.TestStderr.type_id: self.handle_stderr,
+            log.TestStdout.type_id: self.handle_stdout,
+        }
+
+    def handle(self, record):
+        if not self._closed:
+            self.mapping.get(record.type_id, lambda _:None)(record)
+
+    def handle_library_status(self, record):
+        if record['status'] in (state.Status.Complete, state.Status.Avoided):
+            self.test_stream_manager.close()
+
+    def handle_suite_result(self, record):
+        suite_result = self.internal_results.get_suite_result(
+                    record['metadata'].uid)
+        suite_result.result = record['result']
+
+    def handle_test_result(self, record):
+        test_result = self._get_test_result(record)
+        test_result.result = record['result']
+
+    def handle_stderr(self, record):
+        self.test_stream_manager.get_writer(
+            self._get_test_result(record)
+        ).stderr.write(record['buffer'])
+
+    def handle_stdout(self, record):
+        self.test_stream_manager.get_writer(
+            self._get_test_result(record)
+        ).stdout.write(record['buffer'])
+
+    def _get_test_result(self, test_record):
+        return self.internal_results.get_test_result(
+                    test_record['metadata'].uid,
+                    test_record['metadata'].suite_uid)
+
+    def _save(self):
+        #FIXME Hardcoded path name
+        result.InternalSavedResults.save(
+            self.internal_results,
+            os.path.join(self.directory, constants.pickle_filename))
+        result.JUnitSavedResults.save(
+            self.internal_results,
+            os.path.join(self.directory, constants.xml_filename))
+
+    def close(self):
+        if self._closed:
+            return
+        self._closed = True
+        self._save()
+
+
+#TODO Change from a handler to an internal post processor so it can be used
+# to reprint results
+class SummaryHandler(log.Handler):
+    '''
+    A log handler which listens to the log for test results
+    and reports the aggregate results when closed.
+    '''
+    color = terminal.get_termcap()
+    reset = color.Normal
+    colormap = {
+            state.Result.Errored: color.Red,
+            state.Result.Failed: color.Red,
+            state.Result.Passed: color.Green,
+            state.Result.Skipped: color.Cyan,
+    }
+    sep_fmtkey = 'separator'
+    sep_fmtstr = '{%s}' % sep_fmtkey
+
+    def __init__(self):
+        self.mapping = {
+            log.TestResult.type_id: self.handle_testresult,
+            log.LibraryStatus.type_id: self.handle_library_status,
+        }
+        self._timer = helper.Timer()
+        self.results = []
+
+    def handle_library_status(self, record):
+        if record['status'] == state.Status.Building:
+            self._timer.restart()
+
+    def handle_testresult(self, record):
+        result = record['result'].value
+        if result in (state.Result.Skipped, state.Result.Failed,
+                state.Result.Passed, state.Result.Errored):
+            self.results.append(result)
+
+    def handle(self, record):
+        self.mapping.get(record.type_id, lambda _:None)(record)
+
+    def close(self):
+        print(self._display_summary())
+
+    def _display_summary(self):
+        most_severe_outcome = None
+        outcome_fmt = ' {count} {outcome}'
+        strings = []
+
+        outcome_count = [0] * len(state.Result.enums)
+        for result in self.results:
+            outcome_count[result] += 1
+
+        # Iterate over enums so they are in order of severity
+        for outcome in state.Result.enums:
+            outcome = getattr(state.Result, outcome)
+            count  = outcome_count[outcome]
+            if count:
+                strings.append(outcome_fmt.format(count=count,
+                        outcome=state.Result.enums[outcome]))
+                most_severe_outcome = outcome
+        string = ','.join(strings)
+        if most_severe_outcome is None:
+            string = ' No testing done'
+            most_severe_outcome = state.Result.Passed
+        else:
+            string = ' Results:' + string + ' in {:.2} seconds '.format(
+                    self._timer.active_time())
+        string += ' '
+        return terminal.insert_separator(
+                string,
+                color=self.colormap[most_severe_outcome] + self.color.Bold)
+
+class TerminalHandler(log.Handler):
+    color = terminal.get_termcap()
+    verbosity_mapping = {
+        log.LogLevel.Warn: color.Yellow,
+        log.LogLevel.Error: color.Red,
+    }
+    default = color.Normal
+
+    def __init__(self, verbosity=log.LogLevel.Info, machine_only=False):
+        self.stream = verbosity >= log.LogLevel.Trace
+        self.verbosity = verbosity
+        self.machine_only = machine_only
+        self.mapping = {
+            log.TestResult.type_id: self.handle_testresult,
+            log.SuiteStatus.type_id: self.handle_suitestatus,
+            log.TestStatus.type_id: self.handle_teststatus,
+            log.TestStderr.type_id: self.handle_stderr,
+            log.TestStdout.type_id: self.handle_stdout,
+            log.TestMessage.type_id: self.handle_testmessage,
+            log.LibraryMessage.type_id: self.handle_librarymessage,
+        }
+
+    def _display_outcome(self, name, outcome, reason=None):
+        print(self.color.Bold
+                 + SummaryHandler.colormap[outcome]
+                 + name
+                 + ' '
+                 + state.Result.enums[outcome]
+                 + SummaryHandler.reset)
+
+        if reason is not None:
+            log.test_log.info('')
+            log.test_log.info('Reason:')
+            log.test_log.info(reason)
+            log.test_log.info(terminal.separator('-'))
+
+    def handle_teststatus(self, record):
+        if record['status'] == state.Status.Running:
+            log.test_log.debug('Starting Test Case: %s' %\
+                    record['metadata'].name)
+
+    def handle_testresult(self, record):
+        self._display_outcome(
+            'Test: %s'  % record['metadata'].name,
+            record['result'].value)
+
+    def handle_suitestatus(self, record):
+        if record['status'] == state.Status.Running:
+              log.test_log.debug('Starting Test Suite: %s ' %\
+                    record['metadata'].name)
+
+    def handle_stderr(self, record):
+        if self.stream:
+            print(record.data['buffer'], file=sys.stderr, end='')
+
+    def handle_stdout(self, record):
+        if self.stream:
+            print(record.data['buffer'], file=sys.stdout, end='')
+
+    def handle_testmessage(self, record):
+        if self.stream:
+            print(self._colorize(record['message'], record['level']))
+
+    def handle_librarymessage(self, record):
+        if not self.machine_only or record.data.get('machine_readable', False):
+            print(self._colorize(record['message'], record['level'],
+                    record['bold']))
+
+    def _colorize(self, message, level, bold=False):
+        return '%s%s%s%s' % (
+                self.color.Bold if bold else '',
+                self.verbosity_mapping.get(level, ''),
+                message,
+                self.default)
+
+    def handle(self, record):
+        if record.data.get('level', self.verbosity) > self.verbosity:
+            return
+        self.mapping.get(record.type_id, lambda _:None)(record)
+
+    def set_verbosity(self, verbosity):
+        self.verbosity = verbosity
+
+
+class PrintHandler(log.Handler):
+    def __init__(self):
+        pass
+
+    def handle(self, record):
+        print(str(record).rstrip())
+
+    def close(self):
+        pass
+
+
+class MultiprocessingHandlerWrapper(log.Handler):
+    '''
+    A handler class which forwards log records to subhandlers, enabling
+    logging across multiprocessing python processes.
+
+    The 'parent' side of the handler should execute either
+    :func:`async_process` or :func:`process` to forward
+    log records to subhandlers.
+    '''
+    def __init__(self, *subhandlers):
+        # Create thread to spin handing recipt of messages
+        # Create queue to push onto
+        self.queue = multiprocessing.Queue()
+        self.queue.cancel_join_thread()
+        self._shutdown = threading.Event()
+
+        # subhandlers should be accessed with the _handler_lock
+        self._handler_lock = threading.Lock()
+        self._subhandlers = subhandlers
+
+    def add_handler(self, handler):
+        self._handler_lock.acquire()
+        self._subhandlers = (handler, ) + self._subhandlers
+        self._handler_lock.release()
+
+    def _with_handlers(self, callback):
+        exception = None
+        self._handler_lock.acquire()
+        for handler in self._subhandlers:
+            # Prevent deadlock when using this handler by delaying
+            # exception raise until we get a chance to unlock.
+            try:
+                callback(handler)
+            except Exception as e:
+                exception = e
+                break
+        self._handler_lock.release()
+
+        if exception is not None:
+            raise exception
+
+    def async_process(self):
+        self.thread = threading.Thread(target=self.process)
+        self.thread.daemon = True
+        self.thread.start()
+
+    def process(self):
+        while not self._shutdown.is_set():
+            try:
+                item = self.queue.get(timeout=0.1)
+                self._handle(item)
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except EOFError:
+                return
+            except Queue.Empty:
+                continue
+
+    def _drain(self):
+        while True:
+            try:
+                item = self.queue.get(block=False)
+                self._handle(item)
+            except (KeyboardInterrupt, SystemExit):
+                raise
+            except EOFError:
+                return
+            except Queue.Empty:
+                return
+
+    def _handle(self, record):
+        self._with_handlers(lambda handler: handler.handle(record))
+
+    def handle(self, record):
+        self.queue.put(record)
+
+    def _close(self):
+        if hasattr(self, 'thread'):
+            self.thread.join()
+        _wrap(self._drain)
+        self._with_handlers(lambda handler: _wrap(handler.close))
+
+        # NOTE Python2 has an known bug which causes IOErrors to be raised
+        # if this shutdown doesn't go cleanly on both ends.
+        # This sleep adds some time for the sender threads on this process to
+        # finish pickling the object and complete shutdown after the queue is
+        # closed.
+        time.sleep(.2)
+        self.queue.close()
+        time.sleep(.2)
+
+    def close(self):
+        if not self._shutdown.is_set():
+            self._shutdown.set()
+            self._close()
+
+
+def _wrap(callback, *args, **kwargs):
+    try:
+        callback(*args, **kwargs)
+    except:
+        traceback.print_exc()

ext/testlib/helper.py

@@ -0,0 +1,460 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Helper classes for writing tests with this test library.
+'''
+from collections import MutableSet, OrderedDict
+
+import difflib
+import errno
+import os
+import Queue
+import re
+import shutil
+import stat
+import subprocess
+import tempfile
+import threading
+import time
+import traceback
+
+#TODO Tear out duplicate logic from the sandbox IOManager
+def log_call(logger, command, *popenargs, **kwargs):
+    '''
+    Calls the given process and automatically logs the command and output.
+
+    If stdout or stderr are provided output will also be piped into those
+    streams as well.
+
+    :params stdout: Iterable of items to write to as we read from the
+        subprocess.
+
+    :params stderr: Iterable of items to write to as we read from the
+        subprocess.
+    '''
+    if isinstance(command, str):
+        cmdstr = command
+    else:
+        cmdstr = ' '.join(command)
+
+    logger_callback = logger.trace
+    logger.trace('Logging call to command: %s' % cmdstr)
+
+    stdout_redirect = kwargs.get('stdout', tuple())
+    stderr_redirect = kwargs.get('stderr', tuple())
+
+    if hasattr(stdout_redirect, 'write'):
+        stdout_redirect = (stdout_redirect,)
+    if hasattr(stderr_redirect, 'write'):
+        stderr_redirect = (stderr_redirect,)
+
+    kwargs['stdout'] = subprocess.PIPE
+    kwargs['stderr'] = subprocess.PIPE
+    p = subprocess.Popen(command, *popenargs, **kwargs)
+
+    def log_output(log_callback, pipe, redirects=tuple()):
+        # Read iteractively, don't allow input to fill the pipe.
+        for line in iter(pipe.readline, ''):
+            for r in redirects:
+                r.write(line)
+            log_callback(line.rstrip())
+
+    stdout_thread = threading.Thread(target=log_output,
+                           args=(logger_callback, p.stdout, stdout_redirect))
+    stdout_thread.setDaemon(True)
+    stderr_thread = threading.Thread(target=log_output,
+                           args=(logger_callback, p.stderr, stderr_redirect))
+    stderr_thread.setDaemon(True)
+
+    stdout_thread.start()
+    stderr_thread.start()
+
+    retval = p.wait()
+    stdout_thread.join()
+    stderr_thread.join()
+    # Return the return exit code of the process.
+    if retval != 0:
+        raise subprocess.CalledProcessError(retval, cmdstr)
+
+# lru_cache stuff (Introduced in python 3.2+)
+# Renamed and modified to cacheresult
+class _HashedSeq(list):
+    '''
+    This class guarantees that hash() will be called no more than once per
+    element. This is important because the cacheresult() will hash the key
+    multiple times on a cache miss.
+
+    .. note:: From cpython 3.7
+    '''
+
+    __slots__ = 'hashvalue'
+
+    def __init__(self, tup, hash=hash):
+        self[:] = tup
+        self.hashvalue = hash(tup)
+
+    def __hash__(self):
+        return self.hashvalue
+
+def _make_key(args, kwds, typed,
+             kwd_mark = (object(),),
+             fasttypes = {int, str, frozenset, type(None)},
+             tuple=tuple, type=type, len=len):
+    '''
+    Make a cache key from optionally typed positional and keyword arguments.
+    The key is constructed in a way that is flat as possible rather than as
+    a nested structure that would take more memory.  If there is only a single
+    argument and its data type is known to cache its hash value, then that
+    argument is returned without a wrapper. This saves space and improves
+    lookup speed.
+
+    .. note:: From cpython 3.7
+    '''
+    key = args
+    if kwds:
+        key += kwd_mark
+        for item in kwds.items():
+            key += item
+    if typed:
+        key += tuple(type(v) for v in args)
+        if kwds:
+            key += tuple(type(v) for v in kwds.values())
+    elif len(key) == 1 and type(key[0]) in fasttypes:
+        return key[0]
+    return _HashedSeq(key)
+
+
+def cacheresult(function, typed=False):
+    '''
+    :param typed: If typed is True, arguments of different types will be
+        cached separately. I.e. f(3.0) and f(3) will be treated as distinct
+        calls with distinct results.
+
+    .. note:: From cpython 3.7
+    '''
+    sentinel = object()          # unique object used to signal cache misses
+    make_key = _make_key         # build a key from the function arguments
+    cache = {}
+    def wrapper(*args, **kwds):
+        # Simple caching without ordering or size limit
+        key = _make_key(args, kwds, typed)
+        result = cache.get(key, sentinel)
+        if result is not sentinel:
+            return result
+        result = function(*args, **kwds)
+        cache[key] = result
+        return result
+    return wrapper
+
+class OrderedSet(MutableSet):
+    '''
+    Maintain ordering of insertion in items to the set with quick iteration.
+
+    http://code.activestate.com/recipes/576694/
+    '''
+
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.map = {}                   # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def update(self, keys):
+        for key in keys:
+            self.add(key)
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)
+
+def absdirpath(path):
+    '''
+    Return the directory component of the absolute path of the given path.
+    '''
+    return os.path.dirname(os.path.abspath(path))
+
+joinpath = os.path.join
+
+def mkdir_p(path):
+    '''
+    Same thing as mkdir -p
+
+    https://stackoverflow.com/a/600612
+    '''
+    try:
+        os.makedirs(path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise
+
+
+class FrozenSetException(Exception):
+    '''Signals one tried to set a value in a 'frozen' object.'''
+    pass
+
+
+class AttrDict(object):
+    '''Object which exposes its own internal dictionary through attributes.'''
+    def __init__(self, dict_={}):
+        self.update(dict_)
+
+    def __getattr__(self, attr):
+        dict_ = self.__dict__
+        if attr in dict_:
+            return dict_[attr]
+        raise AttributeError('Could not find %s attribute' % attr)
+
+    def __setattr__(self, attr, val):
+        self.__dict__[attr] = val
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def __getitem__(self, item):
+        return self.__dict__[item]
+
+    def update(self, items):
+        self.__dict__.update(items)
+
+
+class FrozenAttrDict(AttrDict):
+    '''An AttrDict whose attributes cannot be modified directly.'''
+    __initialized = False
+    def __init__(self, dict_={}):
+        super(FrozenAttrDict, self).__init__(dict_)
+        self.__initialized = True
+
+    def __setattr__(self, attr, val):
+        if self.__initialized:
+            raise FrozenSetException(
+                        'Cannot modify an attribute in a FozenAttrDict')
+        else:
+            super(FrozenAttrDict, self).__setattr__(attr, val)
+
+    def update(self, items):
+        if self.__initialized:
+            raise FrozenSetException(
+                        'Cannot modify an attribute in a FozenAttrDict')
+        else:
+            super(FrozenAttrDict, self).update(items)
+
+
+class InstanceCollector(object):
+    '''
+    A class used to simplify collecting of Classes.
+
+    >> instance_list = collector.create()
+    >> # Create a bunch of classes which call collector.collect(self)
+    >> # instance_list contains all instances created since
+    >> # collector.create was called
+    >> collector.remove(instance_list)
+    '''
+    def __init__(self):
+        self.collectors = []
+
+    def create(self):
+        collection = []
+        self.collectors.append(collection)
+        return collection
+
+    def remove(self, collector):
+        self.collectors.remove(collector)
+
+    def collect(self, instance):
+        for col in self.collectors:
+            col.append(instance)
+
+
+def append_dictlist(dict_, key, value):
+    '''
+    Append the `value` to a list associated with `key` in `dict_`.
+    If `key` doesn't exist, create a new list in the `dict_` with value in it.
+    '''
+    list_ = dict_.get(key, [])
+    list_.append(value)
+    dict_[key] = list_
+
+
+class ExceptionThread(threading.Thread):
+    '''
+    Wrapper around a python :class:`Thread` which will raise an
+    exception on join if the child threw an unhandled exception.
+    '''
+    def __init__(self, *args, **kwargs):
+        threading.Thread.__init__(self, *args, **kwargs)
+        self._eq = Queue.Queue()
+
+    def run(self, *args, **kwargs):
+        try:
+            threading.Thread.run(self, *args, **kwargs)
+            self._eq.put(None)
+        except:
+            tb = traceback.format_exc()
+            self._eq.put(tb)
+
+    def join(self, *args, **kwargs):
+        threading.Thread.join(*args, **kwargs)
+        exception = self._eq.get()
+        if exception:
+            raise Exception(exception)
+
+
+def _filter_file(fname, filters):
+    with open(fname, "r") as file_:
+        for line in file_:
+            for regex in filters:
+                if re.match(regex, line):
+                    break
+            else:
+                yield line
+
+
+def _copy_file_keep_perms(source, target):
+    '''Copy a file keeping the original permisions of the target.'''
+    st = os.stat(target)
+    shutil.copy2(source, target)
+    os.chown(target, st[stat.ST_UID], st[stat.ST_GID])
+
+
+def _filter_file_inplace(fname, filters):
+    '''
+    Filter the given file writing filtered lines out to a temporary file, then
+    copy that tempfile back into the original file.
+    '''
+    reenter = False
+    (_, tfname) = tempfile.mkstemp(text=True)
+    with open(tfname, 'w') as tempfile_:
+        for line in _filter_file(fname, filters):
+            tempfile_.write(line)
+
+    # Now filtered output is into tempfile_
+    _copy_file_keep_perms(tfname, fname)
+
+
+def diff_out_file(ref_file, out_file, logger, ignore_regexes=tuple()):
+    '''Diff two files returning the diff as a string.'''
+
+    if not os.path.exists(ref_file):
+        raise OSError("%s doesn't exist in reference directory"\
+                                     % ref_file)
+    if not os.path.exists(out_file):
+        raise OSError("%s doesn't exist in output directory" % out_file)
+
+    _filter_file_inplace(out_file, ignore_regexes)
+    _filter_file_inplace(ref_file, ignore_regexes)
+
+    #try :
+    (_, tfname) = tempfile.mkstemp(text=True)
+    with open(tfname, 'r+') as tempfile_:
+        try:
+            log_call(logger, ['diff', out_file, ref_file], stdout=tempfile_)
+        except OSError:
+            # Likely signals that diff does not exist on this system. fallback
+            # to difflib
+            with open(out_file, 'r') as outf, open(ref_file, 'r') as reff:
+                diff = difflib.unified_diff(iter(reff.readline, ''),
+                                            iter(outf.readline, ''),
+                                            fromfile=ref_file,
+                                            tofile=out_file)
+                return ''.join(diff)
+        except subprocess.CalledProcessError:
+            tempfile_.seek(0)
+            return ''.join(tempfile_.readlines())
+        else:
+            return None
+
+class Timer():
+    def __init__(self):
+        self.restart()
+
+    def restart(self):
+        self._start = self.timestamp()
+        self._stop = None
+
+    def stop(self):
+        self._stop = self.timestamp()
+        return self._stop - self._start
+
+    def runtime(self):
+        return self._stop - self._start
+
+    def active_time(self):
+        return self.timestamp() - self._start
+
+    @staticmethod
+    def timestamp():
+        return time.time()

ext/testlib/loader.py

@@ -0,0 +1,302 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Contains the :class:`Loader` which is responsible for discovering and loading
+tests.
+
+Loading typically follows the following stages.
+
+1. Recurse down a given directory looking for tests which match a given regex.
+
+    The default regex used will match any python file (ending in .py) that has
+    a name starting or ending in test(s). If there are any additional
+    components of the name they must be connected with '-' or '_'. Lastly,
+    file names that begin with '.' will be ignored.
+
+    The following names would match:
+
+    - `tests.py`
+    - `test.py`
+    - `test-this.py`
+    - `tests-that.py`
+    - `these-test.py`
+
+    These would not match:
+
+    - `.test.py`    - 'hidden' files are ignored.
+    - `test`        - Must end in '.py'
+    - `test-.py`    - Needs a character after the hypen.
+    - `testthis.py` - Needs a hypen or underscore to separate 'test' and 'this'
+
+
+2. With all files discovered execute each file gathering its test items we
+   care about collecting. (`TestCase`, `TestSuite` and `Fixture` objects.)
+
+As a final note, :class:`TestCase` instances which are not put into
+a :class:`TestSuite` by the test writer will be placed into
+a :class:`TestSuite` named after the module.
+
+.. seealso:: :func:`load_file`
+'''
+
+import os
+import re
+import sys
+import traceback
+
+import config
+import log
+import suite as suite_mod
+import test as test_mod
+import fixture as fixture_mod
+import wrappers
+import uid
+
+class DuplicateTestItemException(Exception):
+    '''
+    Exception indicates multiple test items with the same UID
+    were discovered.
+    '''
+    pass
+
+
+# Match filenames that either begin or end with 'test' or tests and use
+# - or _ to separate additional name components.
+default_filepath_regex = re.compile(
+            r'(((.+[_])?tests?)|(tests?([-_].+)?))\.py$')
+
+def default_filepath_filter(filepath):
+    '''The default filter applied to filepaths to marks as test sources.'''
+    filepath = os.path.basename(filepath)
+    if default_filepath_regex.match(filepath):
+        # Make sure doesn't start with .
+        return not filepath.startswith('.')
+    return False
+
+def path_as_modulename(filepath):
+    '''Return the given filepath as a module name.'''
+    # Remove the file extention (.py)
+    return os.path.splitext(os.path.basename(filepath))[0]
+
+def path_as_suitename(filepath):
+    return os.path.split(os.path.dirname(os.path.abspath((filepath))))[-1]
+
+def _assert_files_in_same_dir(files):
+    if __debug__:
+        if files:
+            directory = os.path.dirname(files[0])
+            for f in files:
+                assert os.path.dirname(f) == directory
+
+class Loader(object):
+    '''
+    Class for discovering tests.
+
+    Discovered :class:`TestCase` and :class:`TestSuite` objects are wrapped by
+    :class:`LoadedTest` and :class:`LoadedSuite` objects respectively.
+    These objects provided additional methods and metadata about the loaded
+    objects and are the internal representation used by testlib.
+
+    To simply discover and load all tests using the default filter create an
+    instance and `load_root`.
+
+    >>> import os
+    >>> tl = Loader()
+    >>> tl.load_root(os.getcwd())
+
+    .. note:: If tests are not contained in a TestSuite, they will
+        automatically be placed into one for the module.
+
+    .. warn:: This class is extremely thread-unsafe.
+       It modifies the sys path and global config.
+       Use with care.
+    '''
+    def __init__(self):
+        self.suites = []
+        self.suite_uids = {}
+        self.filepath_filter = default_filepath_filter
+
+        # filepath -> Successful | Failed to load
+        self._files = {}
+
+    @property
+    def schedule(self):
+        return wrappers.LoadedLibrary(self.suites, fixture_mod.global_fixtures)
+
+    def load_schedule_for_suites(self, *uids):
+        files = {uid.UID.uid_to_path(id_) for id_ in uids}
+        for file_ in files:
+            self.load_file(file_)
+
+        return wrappers.LoadedLibrary(
+                [self.suite_uids[id_] for id_ in uids],
+                fixture_mod.global_fixtures)
+
+    def _verify_no_duplicate_suites(self, new_suites):
+        new_suite_uids = self.suite_uids.copy()
+        for suite in new_suites:
+            if suite.uid in new_suite_uids:
+                raise DuplicateTestItemException(
+                        "More than one suite with UID '%s' was defined" %\
+                                suite.uid)
+            new_suite_uids[suite.uid] = suite
+
+    def _verify_no_duplicate_tests_in_suites(self, new_suites):
+        for suite in new_suites:
+            test_uids = set()
+            for test in suite:
+                if test.uid in test_uids:
+                     raise DuplicateTestItemException(
+                            "More than one test with UID '%s' was defined"
+                            " in suite '%s'"
+                            % (test.uid, suite.uid))
+                test_uids.add(test.uid)
+
+    def load_root(self, root):
+        '''
+        Load files from the given root directory which match
+        `self.filepath_filter`.
+        '''
+        if __debug__:
+            self._loaded_a_file = True
+
+        for directory in self._discover_files(root):
+            if directory:
+                _assert_files_in_same_dir(directory)
+                for f in directory:
+                    self.load_file(f)
+
+    def load_dir(self, directory):
+        for dir_ in self._discover_files(directory):
+            _assert_files_in_same_dir(dir_)
+            for f in dir_:
+                self.load_file(f)
+
+    def load_file(self, path):
+        path = os.path.abspath(path)
+
+        if path in self._files:
+            if not self._files[path]:
+                raise Exception('Attempted to load a file which already'
+                        ' failed to load')
+            else:
+                log.test_log.debug('Tried to reload: %s' % path)
+                return
+
+        # Create a custom dictionary for the loaded module.
+        newdict = {
+            '__builtins__':__builtins__,
+            '__name__': path_as_modulename(path),
+            '__file__': path,
+        }
+
+        # Add the file's containing directory to the system path. So it can do
+        # relative imports naturally.
+        old_path = sys.path[:]
+        sys.path.insert(0, os.path.dirname(path))
+        cwd = os.getcwd()
+        os.chdir(os.path.dirname(path))
+        config.config.file_under_load = path
+
+        new_tests = test_mod.TestCase.collector.create()
+        new_suites = suite_mod.TestSuite.collector.create()
+        new_fixtures = fixture_mod.Fixture.collector.create()
+
+        def cleanup():
+            config.config.file_under_load = None
+            sys.path[:] = old_path
+            os.chdir(cwd)
+            test_mod.TestCase.collector.remove(new_tests)
+            suite_mod.TestSuite.collector.remove(new_suites)
+            fixture_mod.Fixture.collector.remove(new_fixtures)
+
+        try:
+            execfile(path, newdict, newdict)
+        except Exception as e:
+            log.test_log.debug(traceback.format_exc())
+            log.test_log.warn(
+                              'Exception thrown while loading "%s"\n'
+                              'Ignoring all tests in this file.'
+                               % (path))
+            cleanup()
+            return
+
+        # Create a module test suite for those not contained in a suite.
+        orphan_tests = set(new_tests)
+        for suite in new_suites:
+            for test in suite:
+                # Remove the test if it wasn't already removed.
+                # (Suites may contain copies of tests.)
+                if test in orphan_tests:
+                    orphan_tests.remove(test)
+        if orphan_tests:
+            orphan_tests = sorted(orphan_tests, key=new_tests.index)
+            # FIXME Use the config based default to group all uncollected
+            # tests.
+            # NOTE: This is automatically collected (we still have the
+            # collector active.)
+            suite_mod.TestSuite(tests=orphan_tests,
+                    name=path_as_suitename(path))
+
+        try:
+            loaded_suites = [wrappers.LoadedSuite(suite, path)
+                    for suite in new_suites]
+
+            self._verify_no_duplicate_suites(loaded_suites)
+            self._verify_no_duplicate_tests_in_suites(loaded_suites)
+        except Exception as e:
+            log.test_log.warn('%s\n'
+                    'Exception thrown while loading "%s"\n'
+                    'Ignoring all tests in this file.'
+                    % (traceback.format_exc(), path))
+        else:
+            log.test_log.info('Discovered %d tests and %d suites in %s'
+                    '' % (len(new_tests), len(loaded_suites), path))
+
+            self.suites.extend(loaded_suites)
+            self.suite_uids.update({suite.uid: suite
+                    for suite in loaded_suites})
+        cleanup()
+
+    def _discover_files(self, root):
+        '''
+        Recurse down from the given root directory returning a list of
+        directories which contain a list of files matching
+        `self.filepath_filter`.
+        '''
+        # Will probably want to order this traversal.
+        for root, dirnames, filenames in os.walk(root):
+            dirnames.sort()
+            if filenames:
+                filenames.sort()
+                filepaths = [os.path.join(root, filename) \
+                             for filename in filenames]
+                filepaths = filter(self.filepath_filter, filepaths)
+                if filepaths:
+                    yield filepaths

ext/testlib/log.py

@@ -0,0 +1,256 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+This module supplies the global `test_log` object which all testing
+results and messages are reported through.
+'''
+import wrappers
+
+
+class LogLevel():
+    Fatal = 0
+    Error = 1
+    Warn  = 2
+    Info  = 3
+    Debug = 4
+    Trace = 5
+
+
+class RecordTypeCounterMetaclass(type):
+    '''
+    Record type metaclass.
+
+    Adds a static integer value in addition to typeinfo so identifiers
+    are common across processes, networks and module reloads.
+    '''
+    counter = 0
+    def __init__(cls, name, bases, dct):
+        cls.type_id = RecordTypeCounterMetaclass.counter
+        RecordTypeCounterMetaclass.counter += 1
+
+
+class Record(object):
+    '''
+    A generic object that is passed to the :class:`Log` and its handlers.
+
+    ..note: Although not statically enforced, all items in the record should be
+        be pickleable. This enables logging accross multiple processes.
+    '''
+    __metaclass__ = RecordTypeCounterMetaclass
+
+    def __init__(self, **data):
+        self.data = data
+
+    def __getitem__(self, item):
+        if item not in self.data:
+            raise KeyError('%s not in record %s' %\
+                    (item, self.__class__.__name__))
+        return self.data[item]
+
+    def __str__(self):
+        return str(self.data)
+
+
+class StatusRecord(Record):
+    def __init__(self, obj, status):
+        Record.__init__(self, metadata=obj.metadata, status=status)
+class ResultRecord(Record):
+    def __init__(self, obj, result):
+        Record.__init__(self, metadata=obj.metadata, result=result)
+#TODO Refactor this shit... Not ideal. Should just specify attributes.
+class TestStatus(StatusRecord):
+    pass
+class SuiteStatus(StatusRecord):
+    pass
+class LibraryStatus(StatusRecord):
+    pass
+class TestResult(ResultRecord):
+    pass
+class SuiteResult(ResultRecord):
+    pass
+class LibraryResult(ResultRecord):
+    pass
+# Test Output Types
+class TestStderr(Record):
+    pass
+class TestStdout(Record):
+    pass
+# Message (Raw String) Types
+class TestMessage(Record):
+    pass
+class LibraryMessage(Record):
+    pass
+
+
+class Log(object):
+    def __init__(self):
+        self.handlers = []
+        self._opened = False # TODO Guards to methods
+        self._closed = False # TODO Guards to methods
+
+    def finish_init(self):
+        self._opened = True
+
+    def close(self):
+        self._closed = True
+        for handler in self.handlers:
+            handler.close()
+
+    def log(self, record):
+        if not self._opened:
+            self.finish_init()
+        if self._closed:
+            raise Exception('The log has been closed'
+                ' and is no longer available.')
+
+        map(lambda handler:handler.prehandle(), self.handlers)
+        for handler in self.handlers:
+            handler.handle(record)
+            handler.posthandle()
+
+    def add_handler(self, handler):
+        if self._opened:
+            raise Exception('Unable to add a handler once the log is open.')
+        self.handlers.append(handler)
+
+    def close_handler(self, handler):
+        handler.close()
+        self.handlers.remove(handler)
+
+
+class Handler(object):
+    '''
+    Empty implementation of the interface available to handlers which
+    is expected by the :class:`Log`.
+    '''
+    def __init__(self):
+        pass
+
+    def handle(self, record):
+        pass
+
+    def close(self):
+        pass
+
+    def prehandle(self):
+        pass
+
+    def posthandle(self):
+        pass
+
+
+class LogWrapper(object):
+    _result_typemap = {
+        wrappers.LoadedLibrary.__name__: LibraryResult,
+        wrappers.LoadedSuite.__name__: SuiteResult,
+        wrappers.LoadedTest.__name__: TestResult,
+    }
+    _status_typemap = {
+        wrappers.LoadedLibrary.__name__: LibraryStatus,
+        wrappers.LoadedSuite.__name__: SuiteStatus,
+        wrappers.LoadedTest.__name__: TestStatus,
+    }
+    def __init__(self, log):
+        self.log_obj = log
+
+    def log(self, *args, **kwargs):
+        self.log_obj.log(*args, **kwargs)
+
+    # Library Logging Methods
+    # TODO Replace these methods in a test/create a wrapper?
+    # That way they still can log like this it's just hidden that they
+    # capture the current test.
+    def message(self, message, level=LogLevel.Info, bold=False, **metadata):
+        self.log_obj.log(LibraryMessage(message=message, level=level,
+                bold=bold, **metadata))
+
+    def error(self, message):
+        self.message(message, LogLevel.Error)
+
+    def warn(self, message):
+        self.message(message, LogLevel.Warn)
+
+    def info(self, message):
+        self.message(message, LogLevel.Info)
+
+    def debug(self, message):
+        self.message(message, LogLevel.Debug)
+
+    def trace(self, message):
+        self.message(message, LogLevel.Trace)
+
+    # Ongoing Test Logging Methods
+    def status_update(self, obj, status):
+        self.log_obj.log(
+                self._status_typemap[obj.__class__.__name__](obj, status))
+
+    def result_update(self, obj, result):
+        self.log_obj.log(
+                self._result_typemap[obj.__class__.__name__](obj, result))
+
+    def test_message(self, test, message, level):
+        self.log_obj.log(TestMessage(message=message, level=level,
+                test_uid=test.uid, suite_uid=test.parent_suite.uid))
+
+    # NOTE If performance starts to drag on logging stdout/err
+    # replace metadata with just test and suite uid tags.
+    def test_stdout(self, test, suite, buf):
+        self.log_obj.log(TestStdout(buffer=buf, metadata=test.metadata))
+
+    def test_stderr(self, test, suite, buf):
+        self.log_obj.log(TestStderr(buffer=buf, metadata=test.metadata))
+
+    def close(self):
+        self.log_obj.close()
+
+class TestLogWrapper(object):
+    def __init__(self, log, test, suite):
+        self.log_obj = log
+        self.test = test
+
+    def test_message(self, message, level):
+        self.log_obj.test_message(test=self.test,
+                message=message, level=level)
+
+    def error(self, message):
+        self.test_message(message, LogLevel.Error)
+
+    def warn(self, message):
+        self.test_message(message, LogLevel.Warn)
+
+    def info(self, message):
+        self.test_message(message, LogLevel.Info)
+
+    def debug(self, message):
+        self.test_message(message, LogLevel.Debug)
+
+    def trace(self, message):
+        self.test_message(message, LogLevel.Trace)
+
+test_log = LogWrapper(Log())

ext/testlib/main.py

@@ -0,0 +1,328 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import itertools
+
+import config
+import fixture as fixture_mod
+import handlers
+import loader as loader_mod
+import log
+import query
+import result
+import runner
+import terminal
+import uid
+
+def entry_message():
+    log.test_log.message("Running the new gem5 testing script.")
+    log.test_log.message("For more information see TESTING.md.")
+    log.test_log.message("To see details as the testing scripts are"
+                         " running, use the option"
+                         " -v, -vv, or -vvv")
+
+class RunLogHandler():
+    def __init__(self):
+        term_handler = handlers.TerminalHandler(
+            verbosity=config.config.verbose+log.LogLevel.Info
+        )
+        summary_handler = handlers.SummaryHandler()
+        self.mp_handler = handlers.MultiprocessingHandlerWrapper(
+                summary_handler, term_handler)
+        self.mp_handler.async_process()
+        log.test_log.log_obj.add_handler(self.mp_handler)
+        entry_message()
+
+    def schedule_finalized(self, test_schedule):
+        # Create the result handler object.
+        self.result_handler = handlers.ResultHandler(
+                test_schedule, config.config.result_path)
+        self.mp_handler.add_handler(self.result_handler)
+
+    def finish_testing(self):
+        self.result_handler.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+        return False
+
+    def close(self):
+        self.mp_handler.close()
+
+def get_config_tags():
+    return getattr(config.config,
+            config.StorePositionalTagsAction.position_kword)
+
+def filter_with_config_tags(loaded_library):
+    tags = get_config_tags()
+    final_tags = []
+    regex_fmt = '^%s$'
+    cfg = config.config
+
+    def _append_inc_tag_filter(name):
+        if hasattr(cfg, name):
+            tag_opts = getattr(cfg, name)
+            for tag in tag_opts:
+                final_tags.append(config.TagRegex(True, regex_fmt % tag))
+
+    def _append_rem_tag_filter(name):
+        if hasattr(cfg, name):
+            tag_opts = getattr(cfg, name)
+            for tag in cfg.constants.supported_tags[name]:
+                if tag not in tag_opts:
+                    final_tags.append(config.TagRegex(False, regex_fmt % tag))
+
+    # Append additional tags for the isa, length, and variant options.
+    # They apply last (they take priority)
+    special_tags = (
+        cfg.constants.isa_tag_type,
+        cfg.constants.length_tag_type,
+        cfg.constants.variant_tag_type
+    )
+
+    for tagname in special_tags:
+        _append_inc_tag_filter(tagname)
+    for tagname in special_tags:
+        _append_rem_tag_filter(tagname)
+
+    if tags is None:
+        tags = tuple()
+
+    filters = list(itertools.chain(tags, final_tags))
+    string = 'Filtering suites with tags as follows:\n'
+    filter_string = '\t\n'.join((str(f) for f in filters))
+    log.test_log.trace(string + filter_string)
+
+    return filter_with_tags(loaded_library, filters)
+
+
+def filter_with_tags(loaded_library, filters):
+    '''
+    Filter logic supports two filter types:
+    --include-tags <regex>
+    --exclude-tags <regex>
+
+    The logic maintains a `set` of test suites.
+
+    If the regex provided with the `--include-tags` flag matches a tag of a
+    suite, that suite will added to the set.
+
+    If the regex provided with the `--exclude-tags` flag matches a tag of a
+    suite, that suite will removed to the set.
+
+    Suites can be added and removed multiple times.
+
+    First Flag Special Case Logic:
+    If include is the first flag, start with an empty set of suites.
+    If exclude is the first flag, start with the set of all collected suites.
+
+
+    Let's trace out the set as we go through the flags to clarify::
+
+        # Say our collection of suites looks like this: set(suite_ARM64,
+        # suite_X86, suite_Other).
+        #
+        # Additionally, we've passed the flags in the following order:
+        #  --include-tags "ARM64"  --exclude-tags ".*" --include-tags "X86"
+
+        # Process --include-tags "ARM64"
+        set(suite_ARM64)    # Suite begins empty, but adds the ARM64 suite
+        # Process --exclude-tags ".*"
+        set()               # Removed all suites which have tags
+        # Process --include-tags "X86"
+        set(suite_X86)
+    '''
+    if not filters:
+        return
+
+    query_runner = query.QueryRunner(loaded_library)
+    tags = query_runner.tags()
+
+    if not filters[0].include:
+        suites = set(query_runner.suites())
+    else:
+        suites = set()
+
+    def exclude(excludes):
+        return suites - excludes
+    def include(includes):
+        return suites | includes
+
+    for tag_regex in filters:
+        matched_tags = (tag for tag in tags if tag_regex.regex.search(tag))
+        for tag in matched_tags:
+            matched_suites = set(query_runner.suites_with_tag(tag))
+            suites = include(matched_suites) if tag_regex.include \
+                    else exclude(matched_suites)
+
+    # Set the library's suites to only those which where accepted by our filter
+    loaded_library.suites = [suite for suite in loaded_library.suites
+            if suite in suites]
+
+# TODO Add results command for listing previous results.
+
+def load_tests():
+    '''
+    Create a TestLoader and load tests for the directory given by the config.
+    '''
+    testloader = loader_mod.Loader()
+    log.test_log.message(terminal.separator())
+    log.test_log.message('Loading Tests', bold=True)
+    testloader.load_root(config.config.directory)
+    return testloader
+
+def do_list():
+    term_handler = handlers.TerminalHandler(
+        verbosity=config.config.verbose+log.LogLevel.Info,
+        machine_only=config.config.quiet
+    )
+    log.test_log.log_obj.add_handler(term_handler)
+
+    entry_message()
+
+    test_schedule = load_tests().schedule
+    filter_with_config_tags(test_schedule)
+
+    qrunner = query.QueryRunner(test_schedule)
+
+    if config.config.suites:
+        qrunner.list_suites()
+    elif config.config.tests:
+        qrunner.list_tests()
+    elif config.config.all_tags:
+        qrunner.list_tags()
+    else:
+        qrunner.list_suites()
+        qrunner.list_tests()
+        qrunner.list_tags()
+
+def run_schedule(test_schedule, log_handler):
+    '''
+    Test Phases
+    -----------
+    * Test Collection
+    * Fixture Parameterization
+    * Global Fixture Setup
+    * Iteratevely run suites:
+       * Suite Fixture Setup
+       * Iteratively run tests:
+          * Test Fixture Setup
+          * Run Test
+          * Test Fixture Teardown
+       * Suite Fixture Teardown
+    * Global Fixture Teardown
+    '''
+
+    log_handler.schedule_finalized(test_schedule)
+
+    # Iterate through all fixtures notifying them of the test schedule.
+    for suite in test_schedule:
+        copied_fixtures = []
+        for fixture in suite.fixtures:
+            copied_fixtures.append(fixture.schedule_finalized(test_schedule))
+        suite.fixtures = copied_fixtures
+
+        for test in suite:
+            copied_fixtures = []
+            for fixture in test.fixtures:
+                copied_fixtures.append(fixture.schedule_finalized(
+                        test_schedule))
+            test.fixtures = copied_fixtures
+
+    log.test_log.message(terminal.separator())
+    log.test_log.message('Running Tests from {} suites'
+            .format(len(test_schedule.suites)), bold=True)
+    log.test_log.message("Results will be stored in {}".format(
+                config.config.result_path))
+    log.test_log.message(terminal.separator())
+
+    # Build global fixtures and exectute scheduled test suites.
+    if config.config.test_threads > 1:
+        library_runner = runner.LibraryParallelRunner(test_schedule)
+        library_runner.set_threads(config.config.test_threads)
+    else:
+        library_runner = runner.LibraryRunner(test_schedule)
+    library_runner.run()
+
+    log_handler.finish_testing()
+
+def do_run():
+    # Initialize early parts of the log.
+    with RunLogHandler() as log_handler:
+        if config.config.uid:
+            uid_ = uid.UID.from_uid(config.config.uid)
+            if isinstance(uid_, uid.TestUID):
+                log.test_log.error('Unable to run a standalone test.\n'
+                        'Gem5 expects test suites to be the smallest unit '
+                        ' of test.\n\n'
+                        'Pass a SuiteUID instead.')
+                return
+            test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
+            if get_config_tags():
+                log.test_log.warn(
+                    "The '--uid' flag was supplied,"
+                    " '--include-tags' and '--exclude-tags' will be ignored."
+                )
+        else:
+            test_schedule = load_tests().schedule
+            # Filter tests based on tags
+            filter_with_config_tags(test_schedule)
+        # Execute the tests
+        run_schedule(test_schedule, log_handler)
+
+
+def do_rerun():
+    # Init early parts of log
+    with RunLogHandler() as log_handler:
+        # Load previous results
+        results = result.InternalSavedResults.load(
+                os.path.join(config.config.result_path,
+                config.constants.pickle_filename))
+
+        rerun_suites = (suite.uid for suite in results if suite.unsucessful)
+
+        # Use loader to load suites
+        loader = loader_mod.Loader()
+        test_schedule = loader.load_schedule_for_suites(*rerun_suites)
+
+        # Execute the tests
+        run_schedule(test_schedule, log_handler)
+
+def main():
+    '''
+    Main entrypoint for the testlib test library.
+    '''
+    config.initialize_config()
+
+    # 'do' the given command.
+    globals()['do_'+config.config.command]()
+    log.test_log.close()

ext/testlib/query.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import terminal
+import log
+
+# TODO Refactor print logic out of this so the objects
+# created are separate from print logic.
+class QueryRunner(object):
+    def __init__(self, test_schedule):
+        self.schedule = test_schedule
+
+    def tags(self):
+        tags = set()
+        for suite in self.schedule:
+            tags = tags | set(suite.tags)
+        return tags
+
+    def suites(self):
+        return [suite for suite in self.schedule]
+
+    def suites_with_tag(self, tag):
+        return filter(lambda suite: tag in suite.tags, self.suites())
+
+    def list_tests(self):
+        log.test_log.message(terminal.separator())
+        log.test_log.message('Listing all Test Cases.', bold=True)
+        log.test_log.message(terminal.separator())
+        for suite in self.schedule:
+            for test in suite:
+                log.test_log.message(test.uid, machine_readable=True)
+
+    def list_suites(self):
+        log.test_log.message(terminal.separator())
+        log.test_log.message('Listing all Test Suites.', bold=True)
+        log.test_log.message(terminal.separator())
+        for suite in self.suites():
+            log.test_log.message(suite.uid, machine_readable=True)
+
+    def list_tags(self):
+        log.test_log.message(terminal.separator())
+        log.test_log.message('Listing all Test Tags.', bold=True)
+        log.test_log.message(terminal.separator())
+
+        for tag in self.tags():
+            log.test_log.message(tag, machine_readable=True)

ext/testlib/result.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import pickle
+import xml.sax.saxutils
+
+from config import config
+import helper
+import state
+import log
+
+def _create_uid_index(iterable):
+    index = {}
+    for item in iterable:
+        assert item.uid not in index
+        index[item.uid] = item
+    return index
+
+
+class _CommonMetadataMixin:
+    @property
+    def name(self):
+        return self._metadata.name
+    @property
+    def uid(self):
+        return self._metadata.uid
+    @property
+    def result(self):
+        return self._metadata.result
+    @result.setter
+    def result(self, result):
+        self._metadata.result = result
+
+    @property
+    def unsucessful(self):
+        return self._metadata.result.value != state.Result.Passed
+
+
+class InternalTestResult(object, _CommonMetadataMixin):
+    def __init__(self, obj, suite, directory):
+        self._metadata = obj.metadata
+        self.suite = suite
+
+        self.stderr = os.path.join(
+            InternalSavedResults.output_path(self.uid, suite.uid),
+            'stderr'
+        )
+        self.stdout = os.path.join(
+            InternalSavedResults.output_path(self.uid, suite.uid),
+            'stdout'
+        )
+
+
+class InternalSuiteResult(object, _CommonMetadataMixin):
+    def __init__(self, obj, directory):
+        self._metadata = obj.metadata
+        self.directory = directory
+        self._wrap_tests(obj)
+
+    def _wrap_tests(self, obj):
+        self._tests = [InternalTestResult(test, self, self.directory)
+                       for test in obj]
+        self._tests_index = _create_uid_index(self._tests)
+
+    def get_test(self, uid):
+        return self._tests_index[uid]
+
+    def __iter__(self):
+        return iter(self._tests)
+
+    def get_test_result(self, uid):
+        return self.get_test(uid)
+
+    def aggregate_test_results(self):
+        results = {}
+        for test in self:
+            helper.append_dictlist(results, test.result.value, test)
+        return results
+
+
+class InternalLibraryResults(object, _CommonMetadataMixin):
+    def __init__(self, obj, directory):
+        self.directory = directory
+        self._metadata = obj.metadata
+        self._wrap_suites(obj)
+
+    def __iter__(self):
+        return iter(self._suites)
+
+    def _wrap_suites(self, obj):
+        self._suites = [InternalSuiteResult(suite, self.directory)
+                        for suite in obj]
+        self._suites_index = _create_uid_index(self._suites)
+
+    def add_suite(self, suite):
+        if suite.uid in self._suites:
+            raise ValueError('Cannot have duplicate suite UIDs.')
+        self._suites[suite.uid] = suite
+
+    def get_suite_result(self, suite_uid):
+        return self._suites_index[suite_uid]
+
+    def get_test_result(self, test_uid, suite_uid):
+        return self.get_suite_result(suite_uid).get_test_result(test_uid)
+
+    def aggregate_test_results(self):
+        results = {}
+        for suite in self._suites:
+            for test in suite:
+                helper.append_dictlist(results, test.result.value, test)
+        return results
+
+class InternalSavedResults:
+    @staticmethod
+    def output_path(test_uid, suite_uid, base=None):
+        '''
+        Return the path which results for a specific test case should be
+        stored.
+        '''
+        if base is None:
+            base = config.result_path
+        return os.path.join(
+                base,
+                str(suite_uid).replace(os.path.sep, '-'),
+                str(test_uid).replace(os.path.sep, '-'))
+
+    @staticmethod
+    def save(results, path, protocol=pickle.HIGHEST_PROTOCOL):
+        if not os.path.exists(os.path.dirname(path)):
+           try:
+               os.makedirs(os.path.dirname(path))
+           except OSError as exc: # Guard against race condition
+               if exc.errno != errno.EEXIST:
+                   raise
+
+        with open(path, 'w') as f:
+            pickle.dump(results, f, protocol)
+
+    @staticmethod
+    def load(path):
+        with open(path, 'r') as f:
+            return pickle.load(f)
+
+
+class XMLElement(object):
+    def write(self, file_):
+        self.begin(file_)
+        self.end(file_)
+
+    def begin(self, file_):
+        file_.write('<')
+        file_.write(self.name)
+        for attr in self.attributes:
+            file_.write(' ')
+            attr.write(file_)
+        file_.write('>')
+
+        self.body(file_)
+
+    def body(self, file_):
+        for elem in self.elements:
+            file_.write('\n')
+            elem.write(file_)
+        file_.write('\n')
+
+    def end(self, file_):
+        file_.write('</%s>' % self.name)
+
+class XMLAttribute(object):
+    def __init__(self, name, value):
+        self.name = name
+        self.value = value
+
+    def write(self, file_):
+        file_.write('%s=%s' % (self.name,
+                xml.sax.saxutils.quoteattr(self.value)))
+
+
+class JUnitTestSuites(XMLElement):
+    name = 'testsuites'
+    result_map = {
+        state.Result.Errored: 'errors',
+        state.Result.Failed: 'failures',
+        state.Result.Passed: 'tests'
+    }
+
+    def __init__(self, internal_results):
+        results = internal_results.aggregate_test_results()
+
+        self.attributes = []
+        for result, tests in results.items():
+            self.attributes.append(self.result_attribute(result,
+                    str(len(tests))))
+
+        self.elements = []
+        for suite in internal_results:
+            self.elements.append(JUnitTestSuite(suite))
+
+    def result_attribute(self, result, count):
+        return XMLAttribute(self.result_map[result], count)
+
+class JUnitTestSuite(JUnitTestSuites):
+    name = 'testsuite'
+    result_map = {
+        state.Result.Errored: 'errors',
+        state.Result.Failed: 'failures',
+        state.Result.Passed: 'tests',
+        state.Result.Skipped: 'skipped'
+    }
+
+    def __init__(self, suite_result):
+        results = suite_result.aggregate_test_results()
+
+        self.attributes = [
+            XMLAttribute('name', suite_result.name)
+        ]
+        for result, tests in results.items():
+            self.attributes.append(self.result_attribute(result,
+                    str(len(tests))))
+
+        self.elements = []
+        for test in suite_result:
+            self.elements.append(JUnitTestCase(test))
+
+    def result_attribute(self, result, count):
+        return XMLAttribute(self.result_map[result], count)
+
+class JUnitTestCase(XMLElement):
+    name = 'testcase'
+    def __init__(self, test_result):
+        self.attributes = [
+            XMLAttribute('name', test_result.name),
+             # TODO JUnit expects class of test.. add as test metadata.
+            XMLAttribute('classname', str(test_result.uid)),
+            XMLAttribute('status', str(test_result.result)),
+        ]
+
+        # TODO JUnit expects a message for the reason a test was
+        # skipped or errored, save this with the test metadata.
+        # http://llg.cubic.org/docs/junit/
+        self.elements = [
+            LargeFileElement('system-err', test_result.stderr),
+            LargeFileElement('system-out', test_result.stdout),
+        ]
+
+class LargeFileElement(XMLElement):
+    def __init__(self, name, filename):
+        self.name = name
+        self.filename = filename
+        self.attributes = []
+
+    def body(self, file_):
+        try:
+            with open(self.filename, 'r') as f:
+                for line in f:
+                    file_.write(xml.sax.saxutils.escape(line))
+        except IOError:
+            # TODO Better error logic, this is sometimes O.K.
+            # if there was no stdout/stderr captured for the test
+            #
+            # TODO If that was the case, the file should still be made and it
+            # should just be empty instead of not existing.
+            pass
+
+
+
+class JUnitSavedResults:
+    @staticmethod
+    def save(results, path):
+        '''
+        Compile the internal results into JUnit format writting it to the
+        given file.
+        '''
+        results = JUnitTestSuites(results)
+        with open(path, 'w') as f:
+            results.write(f)
+

ext/testlib/runner.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import multiprocessing.dummy
+import threading
+import traceback
+
+import helper
+import state
+import log
+import sandbox
+
+from state import Status, Result
+from fixture import SkipException
+
+def compute_aggregate_result(iterable):
+    '''
+    Status of the test suite by default is:
+    * Passed if all contained tests passed
+    * Errored if any contained tests errored
+    * Failed if no tests errored, but one or more failed.
+    * Skipped if all contained tests were skipped
+    '''
+    failed = []
+    skipped = []
+    for testitem in iterable:
+        result = testitem.result
+
+        if result.value == Result.Errored:
+            return Result(result.value, result.reason)
+        elif result.value == Result.Failed:
+            failed.append(result.reason)
+        elif result.value == result.Skipped:
+            skipped.append(result.reason)
+    if failed:
+        return Result(Result.Failed, failed)
+    elif skipped:
+        return Result(Result.Skipped, skipped)
+    else:
+        return Result(Result.Passed)
+
+class TestParameters(object):
+    def __init__(self, test, suite):
+        self.test = test
+        self.suite = suite
+        self.log = log.TestLogWrapper(log.test_log, test, suite)
+
+    @helper.cacheresult
+    def _fixtures(self):
+        fixtures = {fixture.name:fixture for fixture in self.suite.fixtures}
+        for fixture in self.test.fixtures:
+            fixtures[fixture.name] = fixture
+        return fixtures
+
+    @property
+    def fixtures(self):
+        return self._fixtures()
+
+
+class RunnerPattern:
+    def __init__(self, loaded_testable):
+        self.testable = loaded_testable
+        self.builder = FixtureBuilder(self.testable.fixtures)
+
+    def handle_error(self, trace):
+        self.testable.result = Result(Result.Errored, trace)
+        self.avoid_children(trace)
+
+    def handle_skip(self, trace):
+        self.testable.result = Result(Result.Skipped, trace)
+        self.avoid_children(trace)
+
+    def avoid_children(self, reason):
+        for testable in self.testable:
+            testable.result = Result(self.testable.result.value, reason)
+            testable.status = Status.Avoided
+
+    def test(self):
+        pass
+
+    def run(self):
+        avoided = False
+        try:
+            self.testable.status = Status.Building
+            self.builder.setup(self.testable)
+        except SkipException:
+            self.handle_skip(traceback.format_exc())
+            avoided = True
+        except BrokenFixtureException:
+            self.handle_error(traceback.format_exc())
+            avoided = True
+        else:
+            self.testable.status = Status.Running
+            self.test()
+        finally:
+            self.testable.status = Status.TearingDown
+            self.builder.teardown(self.testable)
+
+        if avoided:
+            self.testable.status = Status.Avoided
+        else:
+            self.testable.status = Status.Complete
+
+class TestRunner(RunnerPattern):
+    def test(self):
+        self.sandbox_test()
+
+    def sandbox_test(self):
+        try:
+            sandbox.Sandbox(TestParameters(
+                    self.testable,
+                    self.testable.parent_suite))
+        except sandbox.SubprocessException:
+            self.testable.result = Result(Result.Failed,
+                    traceback.format_exc())
+        else:
+            self.testable.result = Result(Result.Passed)
+
+
+class SuiteRunner(RunnerPattern):
+    def test(self):
+        for test in self.testable:
+            test.runner(test).run()
+        self.testable.result = compute_aggregate_result(
+                iter(self.testable))
+
+
+class LibraryRunner(SuiteRunner):
+    pass
+
+
+class LibraryParallelRunner(RunnerPattern):
+    def set_threads(self, threads):
+        self.threads = threads
+
+    def _entrypoint(self, suite):
+        suite.runner(suite).run()
+
+    def test(self):
+        pool = multiprocessing.dummy.Pool(self.threads)
+        pool.map(lambda suite : suite.runner(suite).run(), self.testable)
+        self.testable.result = compute_aggregate_result(
+                iter(self.testable))
+
+
+class BrokenFixtureException(Exception):
+    def __init__(self, fixture, testitem, trace):
+        self.fixture = fixture
+        self.testitem = testitem
+        self.trace = trace
+
+        self.msg = ('%s\n'
+                   'Exception raised building "%s" raised SkipException'
+                   ' for "%s".' %
+                   (trace, fixture.name, testitem.name)
+        )
+        super(BrokenFixtureException, self).__init__(self.msg)
+
+class FixtureBuilder(object):
+    def __init__(self, fixtures):
+        self.fixtures = fixtures
+        self.built_fixtures = []
+
+    def setup(self, testitem):
+        for fixture in self.fixtures:
+            # Mark as built before, so if the build fails
+            # we still try to tear it down.
+            self.built_fixtures.append(fixture)
+            try:
+                fixture.setup(testitem)
+            except SkipException:
+                raise
+            except Exception as e:
+                exc = traceback.format_exc()
+                msg = 'Exception raised while setting up fixture for %s' %\
+                        testitem.uid
+                log.test_log.warn('%s\n%s' % (exc, msg))
+
+                raise BrokenFixtureException(fixture, testitem,
+                        traceback.format_exc())
+
+    def teardown(self, testitem):
+        for fixture in self.built_fixtures:
+            try:
+                fixture.teardown(testitem)
+            except Exception:
+                # Log exception but keep cleaning up.
+                exc = traceback.format_exc()
+                msg = 'Exception raised while tearing down fixture for %s' %\
+                        testitem.uid
+                log.test_log.warn('%s\n%s' % (exc, msg))

ext/testlib/sandbox.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import multiprocessing
+import pdb
+import os
+import sys
+import threading
+import traceback
+
+import log
+
+pdb._Pdb = pdb.Pdb
+class ForkedPdb(pdb._Pdb):
+    '''
+    A Pdb subclass that may be used from a forked multiprocessing child
+    '''
+    io_manager = None
+    def interaction(self, *args, **kwargs):
+        _stdin = sys.stdin
+        self.io_manager.restore_pipes()
+        try:
+            sys.stdin = open('/dev/stdin')
+            pdb._Pdb.interaction(self, *args, **kwargs)
+        finally:
+            sys.stdin = _stdin
+            self.io_manager.replace_pipes()
+
+
+#TODO Refactor duplicate stdout, stderr logic
+class IoManager(object):
+    def __init__(self, test, suite):
+        self.test = test
+        self.suite = suite
+        self.log = log.test_log
+        self._init_pipes()
+
+    def _init_pipes(self):
+        self.stdout_rp, self.stdout_wp = os.pipe()
+        self.stderr_rp, self.stderr_wp = os.pipe()
+
+    def close_parent_pipes(self):
+        os.close(self.stdout_wp)
+        os.close(self.stderr_wp)
+
+    def setup(self):
+        self.replace_pipes()
+        self.fixup_pdb()
+
+    def fixup_pdb(self):
+        ForkedPdb.io_manager = self
+        pdb.Pdb = ForkedPdb
+
+    def replace_pipes(self):
+        self.old_stderr = os.dup(sys.stderr.fileno())
+        self.old_stdout = os.dup(sys.stdout.fileno())
+
+        os.dup2(self.stderr_wp, sys.stderr.fileno())
+        sys.stderr = os.fdopen(self.stderr_wp, 'w', 0)
+        os.dup2(self.stdout_wp, sys.stdout.fileno())
+        sys.stdout = os.fdopen(self.stdout_wp, 'w', 0)
+
+    def restore_pipes(self):
+        self.stderr_wp = os.dup(sys.stderr.fileno())
+        self.stdout_wp = os.dup(sys.stdout.fileno())
+
+        os.dup2(self.old_stderr, sys.stderr.fileno())
+        sys.stderr = os.fdopen(self.old_stderr, 'w', 0)
+        os.dup2(self.old_stdout, sys.stdout.fileno())
+        sys.stdout = os.fdopen(self.old_stdout, 'w', 0)
+
+    def start_loggers(self):
+        self.log_ouput()
+
+    def log_ouput(self):
+        def _log_output(pipe, log_callback):
+            with os.fdopen(pipe, 'r') as pipe:
+                # Read iteractively, don't allow input to fill the pipe.
+                for line in iter(pipe.readline, ''):
+                    log_callback(line)
+
+        # Don't keep a backpointer to self in the thread.
+        log = self.log
+        test = self.test
+        suite = self.suite
+
+        self.stdout_thread = threading.Thread(
+                target=_log_output,
+                args=(self.stdout_rp,
+                      lambda buf: log.test_stdout(test, suite, buf))
+        )
+        self.stderr_thread = threading.Thread(
+                target=_log_output,
+                args=(self.stderr_rp,
+                      lambda buf: log.test_stderr(test, suite, buf))
+        )
+
+        # Daemon + Join to not lock up main thread if something breaks
+        # but provide consistent execution if nothing goes wrong.
+        self.stdout_thread.daemon = True
+        self.stderr_thread.daemon = True
+        self.stdout_thread.start()
+        self.stderr_thread.start()
+
+    def join_loggers(self):
+        self.stdout_thread.join()
+        self.stderr_thread.join()
+
+
+class SubprocessException(Exception):
+    def __init__(self, exception, trace):
+        super(SubprocessException, self).__init__(trace)
+
+class ExceptionProcess(multiprocessing.Process):
+    class Status():
+        def __init__(self, exitcode, exception_tuple):
+            self.exitcode = exitcode
+            if exception_tuple is not None:
+                self.trace = exception_tuple[1]
+                self.exception = exception_tuple[0]
+            else:
+                self.exception = None
+                self.trace = None
+
+    def __init__(self, *args, **kwargs):
+        multiprocessing.Process.__init__(self, *args, **kwargs)
+        self._pconn, self._cconn = multiprocessing.Pipe()
+        self._exception = None
+
+    def run(self):
+        try:
+            super(ExceptionProcess, self).run()
+            self._cconn.send(None)
+        except Exception as e:
+            tb = traceback.format_exc()
+            self._cconn.send((e, tb))
+            raise
+
+    @property
+    def status(self):
+        if self._pconn.poll():
+            self._exception = self._pconn.recv()
+
+        return self.Status(self.exitcode, self._exception)
+
+
+class Sandbox(object):
+    def __init__(self, test_parameters):
+
+        self.params = test_parameters
+        self.io_manager = IoManager(self.params.test, self.params.suite)
+
+        self.p = ExceptionProcess(target=self.entrypoint)
+        # Daemon + Join to not lock up main thread if something breaks
+        self.p.daemon = True
+        self.io_manager.start_loggers()
+        self.p.start()
+        self.io_manager.close_parent_pipes()
+        self.p.join()
+        self.io_manager.join_loggers()
+
+        status = self.p.status
+        if status.exitcode:
+            raise SubprocessException(status.exception, status.trace)
+
+    def entrypoint(self):
+        self.io_manager.setup()
+        self.params.test.test(self.params)

ext/testlib/state.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+class Result:
+    enums = '''
+        NotRun
+        Skipped
+        Passed
+        Failed
+        Errored
+    '''.split()
+    for idx, enum in enumerate(enums):
+        locals()[enum] = idx
+
+    @classmethod
+    def name(cls, enum):
+        return cls.enums[enum]
+
+    def __init__(self, value, reason=None):
+        self.value = value
+        self.reason = reason
+
+    def __str__(self):
+        return self.name(self.value)
+
+class Status:
+    enums = '''
+        Unscheduled
+        Building
+        Running
+        TearingDown
+        Complete
+        Avoided
+    '''.split()
+    for idx, enum in enumerate(enums):
+        locals()[enum] = idx
+
+    @classmethod
+    def name(cls, enum):
+        return cls.enums[enum]

ext/testlib/suite.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+
+import helper
+import runner as runner_mod
+
+class TestSuite(object):
+    '''
+    An object grouping a collection of tests. It provides tags which enable
+    filtering during list and run selection. All tests held in the suite must
+    have a unique name.
+
+    ..note::
+        The :func:`__new__` method enables collection of test cases, it must
+        be called in order for test cases to be collected.
+
+    ..note::
+        To reduce test definition boilerplate, the :func:`init` method is
+        forwarded all `*args` and `**kwargs`. This means derived classes can
+        define init without boilerplate super().__init__(*args, **kwargs).
+    '''
+    runner = runner_mod.SuiteRunner
+    collector = helper.InstanceCollector()
+    fixtures = []
+    tests = []
+    tags = set()
+
+    def __new__(klass, *args, **kwargs):
+        obj = super(TestSuite, klass).__new__(klass, *args, **kwargs)
+        TestSuite.collector.collect(obj)
+        return obj
+
+    def __init__(self, name=None, fixtures=tuple(), tests=tuple(),
+                 tags=tuple(), **kwargs):
+        self.fixtures = self.fixtures + list(fixtures)
+        self.tags = self.tags | set(tags)
+        self.tests = self.tests + list(tests)
+        if name is None:
+            name = self.__class__.__name__
+        self.name = name
+
+    def __iter__(self):
+        return iter(self.tests)

ext/testlib/terminal.py

@@ -0,0 +1,165 @@
+# Copyright (c) 2011 Advanced Micro Devices, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Author: Steve Reinhardt
+
+import sys
+import fcntl
+import termios
+import struct
+
+# Intended usage example:
+#
+# if force_colors:
+#    from m5.util.terminal import termcap
+# elif no_colors:
+#    from m5.util.terminal import no_termcap as termcap
+# else:
+#    from m5.util.terminal import tty_termcap as termcap
+# print termcap.Blue + "This could be blue!" + termcap.Normal
+
+# ANSI color names in index order
+color_names = "Black Red Green Yellow Blue Magenta Cyan White".split()
+default_separator = '='
+
+# Character attribute capabilities.  Note that not all terminals
+# support all of these capabilities, or support them
+# differently/meaningfully.  For example:
+#
+# - In PuTTY (with the default settings), Dim has no effect, Standout
+#   is the same as Reverse, and Blink does not blink but switches to a
+#   gray background.
+#
+# Please feel free to add information about other terminals here.
+#
+capability_map = {
+         'Bold': 'bold',
+          'Dim': 'dim',
+        'Blink': 'blink',
+    'Underline': 'smul',
+      'Reverse': 'rev',
+     'Standout': 'smso',
+       'Normal': 'sgr0'
+}
+
+capability_names = capability_map.keys()
+
+def null_cap_string(s, *args):
+    return ''
+
+try:
+    import curses
+    curses.setupterm()
+    def cap_string(s, *args):
+        cap = curses.tigetstr(s)
+        if cap:
+            return curses.tparm(cap, *args)
+        else:
+            return ''
+except:
+    cap_string = null_cap_string
+
+class ColorStrings(object):
+    def __init__(self, cap_string):
+        for i, c in enumerate(color_names):
+            setattr(self, c, cap_string('setaf', i))
+        for name, cap in capability_map.iteritems():
+            setattr(self, name, cap_string(cap))
+
+termcap = ColorStrings(cap_string)
+no_termcap = ColorStrings(null_cap_string)
+
+if sys.stdout.isatty():
+    tty_termcap = termcap
+else:
+    tty_termcap = no_termcap
+
+def get_termcap(use_colors = None):
+    if use_colors:
+        return termcap
+    elif use_colors is None:
+        # option unspecified; default behavior is to use colors iff isatty
+        return tty_termcap
+    else:
+        return no_termcap
+
+def terminal_size():
+    '''Return the (width, heigth) of the terminal screen.'''
+    h, w, hp, wp = struct.unpack('HHHH',
+        fcntl.ioctl(0, termios.TIOCGWINSZ,
+        struct.pack('HHHH', 0, 0, 0, 0)))
+    return w, h
+
+def separator(char=default_separator, color=None):
+    '''
+    Return a separator of the given character that is the length of the full
+    width of the terminal screen.
+    '''
+    (w, h) = terminal_size()
+    if color:
+        return color + char*w + termcap.Normal
+    else:
+        return char*w
+
+def insert_separator(inside, char=default_separator,
+                     min_barrier=3, color=None):
+    '''
+    Place the given string inside of the separator. If it does not fit inside,
+    expand the separator to fit it with at least min_barrier.
+
+    .. seealso:: :func:`separator`
+    '''
+    # Use a bytearray so it's efficient to manipulate
+    string = bytearray(separator(char, color=color))
+
+    # Check if we can fit inside with at least min_barrier.
+    gap = (len(string) - len(inside)) - min_barrier * 2
+    if gap > 0:
+        # We'll need to expand the string to fit us.
+        string.extend([ char for _ in range(-gap)])
+    # Emplace inside
+    middle = ((len(string)-1)/2)
+    start_idx = middle - len(inside)/2
+    string[start_idx:len(inside)+start_idx] = inside
+    return str(string)
+
+
+if __name__ == '__main__':
+    def test_termcap(obj):
+        for c_name in color_names:
+            c_str = getattr(obj, c_name)
+            print c_str + c_name + obj.Normal
+            for attr_name in capability_names:
+                if attr_name == 'Normal':
+                    continue
+                attr_str = getattr(obj, attr_name)
+                print attr_str + c_str + attr_name + " " + c_name + obj.Normal
+            print obj.Bold + obj.Underline + \
+                  c_name + "Bold Underline " + c_str + obj.Normal
+
+    print "=== termcap enabled ==="
+    test_termcap(termcap)
+    print termcap.Normal
+    print "=== termcap disabled ==="
+    test_termcap(no_termcap)

ext/testlib/test.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import functools
+
+import helper
+import runner as runner_mod
+
+class TestCase(object):
+    '''
+    Base class for all tests.
+
+    ..note::
+        The :func:`__new__` method enables collection of test cases, it must
+        be called in order for test cases to be collected.
+    '''
+    fixtures = []
+
+    # TODO, remove explicit dependency. Use the loader to set the
+    # default runner
+    runner = runner_mod.TestRunner
+    collector = helper.InstanceCollector()
+
+    def __new__(cls, *args, **kwargs):
+        obj = super(TestCase, cls).__new__(cls, *args, **kwargs)
+        TestCase.collector.collect(obj)
+        return obj
+
+    def __init__(self, name=None, fixtures=tuple(), **kwargs):
+        self.fixtures = self.fixtures + list(fixtures)
+        if name is None:
+            name = self.__class__.__name__
+        self.name = name
+
+class TestFunction(TestCase):
+    '''
+    TestCase implementation which uses a callable object as a test.
+    '''
+    def __init__(self, function, name=None, **kwargs):
+        self.test_function = function
+        if name is None:
+            name = function.__name__
+        TestCase.__init__(self, name=name, **kwargs)
+
+    def test(self, *args, **kwargs):
+        self.test_function(*args, **kwargs)
+
+# TODO Change the decorator to make this easier to create copy tests.
+# Good way to do so might be return by reference.
+def testfunction(function=None, name=None, fixtures=tuple()):
+    '''
+    A decorator used to wrap a function as a TestFunction.
+    '''
+    def testfunctiondecorator(function):
+        '''Decorator used to mark a function as a test case.'''
+        kwargs = {}
+        if name is not None:
+            kwargs['name'] = name
+        if fixtures is not None:
+            kwargs['fixtures'] = fixtures
+        TestFunction(function, **kwargs)
+        return function
+    if function is not None:
+        return testfunctiondecorator(function)
+    else:
+        return testfunctiondecorator

ext/testlib/uid.py

@@ -0,0 +1,110 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import itertools
+
+import config
+
+class UID(object):
+    sep = ':'
+    type_idx, path_idx = range(2)
+
+    def __init__(self, path, *args):
+        self.path = self._shorten_path(path)
+        self.attributes = args
+
+    @staticmethod
+    def _shorten_path(path):
+        return os.path.relpath(path,
+                os.path.commonprefix((config.constants.testing_base,
+                                      path)))
+
+    @staticmethod
+    def _full_path(short_path):
+        return os.path.join(config.constants.testing_base, short_path)
+
+    @classmethod
+    def uid_to_path(cls, uid):
+        split_path = str(uid).split(cls.sep)[cls.path_idx]
+        return cls._full_path(split_path)
+
+    @classmethod
+    def uid_to_class(cls, uid):
+        return globals()[uid.split(cls.sep)[cls.type_idx]]
+
+    @classmethod
+    def from_suite(self, suite, filepath):
+        return SuiteUID(filepath, suite.name)
+
+    @classmethod
+    def from_test(self, test, filepath):
+        return TestUID(filepath, test.name, test.parent_suite.name)
+
+    @classmethod
+    def from_uid(cls, uid):
+        args = uid.split(cls.sep)
+        del args[cls.type_idx]
+        return cls.uid_to_class(uid)(*args)
+
+    def __str__(self):
+        common_opts = {
+            self.path_idx: self.path,
+            self.type_idx: self.__class__.__name__
+        }
+        return self.sep.join(itertools.chain(
+            [common_opts[0], common_opts[1]],
+            self.attributes))
+
+    def __hash__(self):
+        return hash(str(self))
+
+    def __eq__(self, other):
+        return type(self) == type(other) and str(self) == str(other)
+
+
+class TestUID(UID):
+    def __init__(self, filename, test_name, suite_name):
+        UID.__init__(self, filename, suite_name, test_name)
+
+    @property
+    def test(self):
+        return self.attributes[1]
+
+    @property
+    def suite(self):
+        return self.attributes[0]
+
+
+class SuiteUID(UID):
+    def __init__(self, filename, suite_name):
+        UID.__init__(self, filename, suite_name)
+
+    @property
+    def suite(self):
+        return self.attributes[0]

ext/testlib/wrappers.py

@@ -0,0 +1,236 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Module contains wrappers for test items that have been
+loaded by the testlib :class:`testlib.loader.Loader`.
+'''
+import itertools
+
+import log
+import uid
+from state import Status, Result
+
+class TestCaseMetadata():
+    def __init__(self, name, uid, path, result, status, suite_uid):
+        self.name = name
+        self.uid = uid
+        self.path = path
+        self.status = status
+        self.result = result
+        self.suite_uid = suite_uid
+
+
+class TestSuiteMetadata():
+    def __init__(self, name, uid, tags, path, status, result):
+        self.name = name
+        self.uid = uid
+        self.tags = tags
+        self.path = path
+        self.status = status
+        self.result = result
+
+
+class LibraryMetadata():
+    def __init__(self, name, result, status):
+        self.name = name
+        self.result = result
+        self.status = status
+
+
+class LoadedTestable(object):
+    '''
+    Base class for loaded test items.
+
+    :property:`result` and :property:`status` setters
+    notify testlib via the :func:`log_result` and :func:`log_status`
+    of the updated status.
+    '''
+    def __init__(self, obj):
+        self.obj = obj
+        self.metadata = self._generate_metadata()
+
+    @property
+    def status(self):
+        return self.metadata.status
+
+    @status.setter
+    def status(self, status):
+        self.log_status(status)
+        self.metadata.status = status
+
+    @property
+    def result(self):
+        return self.metadata.result
+
+    @result.setter
+    def result(self, result):
+        self.log_result(result)
+        self.metadata.result = result
+
+    @property
+    def uid(self):
+        return self.metadata.uid
+
+    @property
+    def name(self):
+        return self.metadata.name
+
+    @property
+    def fixtures(self):
+        return self.obj.fixtures
+
+    @fixtures.setter
+    def fixtures(self, fixtures):
+        self.obj.fixtures = fixtures
+
+    @property
+    def runner(self):
+        return self.obj.runner
+
+    # TODO Change log to provide status_update, result_update for all types.
+    def log_status(self, status):
+        log.test_log.status_update(self, status)
+
+    def log_result(self, result):
+        log.test_log.result_update(self, result)
+
+    def __iter__(self):
+        return iter(())
+
+
+class LoadedTest(LoadedTestable):
+    def __init__(self, test_obj, loaded_suite, path):
+        self.parent_suite = loaded_suite
+        self._path = path
+        LoadedTestable.__init__(self, test_obj)
+
+    def test(self, *args, **kwargs):
+        self.obj.test(*args, **kwargs)
+
+    def _generate_metadata(self):
+        return TestCaseMetadata( **{
+            'name':self.obj.name,
+            'path': self._path,
+            'uid': uid.TestUID(self._path,
+                               self.obj.name,
+                               self.parent_suite.name),
+            'status': Status.Unscheduled,
+            'result': Result(Result.NotRun),
+            'suite_uid': self.parent_suite.metadata.uid
+        })
+
+
+class LoadedSuite(LoadedTestable):
+    def __init__(self, suite_obj, path):
+        self._path = path
+        LoadedTestable.__init__(self, suite_obj)
+        self.tests = self._wrap_children(suite_obj)
+
+    def _wrap_children(self, suite_obj):
+        return [LoadedTest(test, self, self.metadata.path)
+                for test in suite_obj]
+
+    def _generate_metadata(self):
+        return TestSuiteMetadata( **{
+            'name': self.obj.name,
+            'tags':self.obj.tags,
+            'path': self._path,
+            'uid': uid.SuiteUID(self._path, self.obj.name),
+            'status': Status.Unscheduled,
+            'result': Result(Result.NotRun)
+        })
+
+    def __iter__(self):
+        return iter(self.tests)
+
+    @property
+    def tags(self):
+        return self.metadata.tags
+
+
+class LoadedLibrary(LoadedTestable):
+    '''
+    Wraps a collection of all loaded test suites and
+    provides utility functions for accessing fixtures.
+    '''
+    def __init__(self, suites, global_fixtures):
+        LoadedTestable.__init__(self, suites)
+        self.global_fixtures = global_fixtures
+
+    def _generate_metadata(self):
+        return LibraryMetadata( **{
+            'name': 'Test Library',
+            'status': Status.Unscheduled,
+            'result': Result(Result.NotRun)
+        })
+
+    def __iter__(self):
+        '''
+        :returns: an iterator over contained :class:`TestSuite` objects.
+        '''
+        return iter(self.obj)
+
+    def all_fixture_tuples(self):
+        return itertools.chain(
+                self.global_fixtures,
+                *(suite.fixtures for suite in self.obj))
+
+    def all_fixtures(self):
+        '''
+        :returns: an interator overall all global, suite,
+          and test fixtures
+        '''
+        return itertools.chain(itertools.chain(
+                self.global_fixtures,
+                *(suite.fixtures for suite in self.obj)),
+            *(self.test_fixtures(suite) for suite in self.obj)
+        )
+
+    def test_fixtures(self, suite):
+        '''
+        :returns: an interator over all fixtures of each
+          test contained in the given suite
+        '''
+        return itertools.chain(*(test.fixtures for test in suite))
+
+    @property
+    def fixtures(self):
+        return self.global_fixtures
+
+    @property
+    def uid(self):
+        return self.name
+
+    @property
+    def suites(self):
+        return self.obj
+
+    @suites.setter
+    def suites(self, suites):
+        self.obj = suites

tests/.gitignore

@@ -0,0 +1 @@
+.testing-results

tests/gem5/__init__.py

@@ -0,0 +1,33 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import suite
+import fixture
+
+from suite import *
+from fixture import *

tests/gem5/fixture.py

@@ -0,0 +1,258 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import tempfile
+import shutil
+
+from testlib.fixture import Fixture, globalfixture
+from testlib.config import config, constants
+from testlib.helper import log_call, cacheresult, joinpath, absdirpath
+import testlib.log as log
+
+
+class VariableFixture(Fixture):
+    def __init__(self, value=None, name=None):
+        super(VariableFixture, self).__init__(name=name)
+        self.value = value
+
+
+class TempdirFixture(Fixture):
+    def __init__(self):
+        self.path = None
+        super(TempdirFixture, self).__init__(
+                name=constants.tempdir_fixture_name)
+
+    def setup(self, testitem):
+        self.path = tempfile.mkdtemp(prefix='gem5out')
+
+    def teardown(self, testitem):
+        if self.path is not None:
+            shutil.rmtree(self.path)
+
+
+class SConsFixture(Fixture):
+    '''
+    Fixture will wait until all SCons targets are collected and tests are
+    about to be ran, then will invocate a single instance of SCons for all
+    targets.
+
+    :param directory: The directory which scons will -C (cd) into before
+        executing. If None is provided, will choose the config base_dir.
+    '''
+    def __init__(self, directory=None, target_class=None):
+        self.directory = directory if directory else config.base_dir
+        self.target_class = target_class if target_class else SConsTarget
+        self.threads = config.threads
+        self.targets = set()
+        super(SConsFixture, self).__init__()
+
+    def setup(self, testitem):
+        if config.skip_build:
+            return
+
+        command = [
+            'scons', '-C', self.directory,
+            '-j', str(self.threads),
+            '--ignore-style'
+        ]
+
+        if not self.targets:
+            log.test_log.warn(
+                'No SCons targets specified, this will'
+                ' build the default all target.\n'
+                'This is likely unintended, and you'
+                ' may wish to kill testlib and reconfigure.')
+        else:
+            log.test_log.message(
+                    'Building the following targets.'
+                    ' This may take a while.')
+            log.test_log.message('%s' % (', '.join(self.targets)))
+            log.test_log.message(
+                    "You may want to run with only a single ISA"
+                    "(--isa=), use --skip-build, or use 'rerun'.")
+
+
+
+        command.extend(self.targets)
+        log_call(log.test_log, command)
+
+
+class SConsTarget(Fixture):
+    # The singleton scons fixture we'll use for all targets.
+    default_scons_invocation = None
+
+    def __init__(self, target, build_dir=None, invocation=None):
+        '''
+        Represents a target to be built by an 'invocation' of scons.
+
+        :param target: The target known to scons.
+
+        :param build_dir: The 'build' directory path which will be prepended
+            to the target name.
+
+        :param invocation: Represents an invocation of scons which we will
+            automatically attach this target to. If None provided, uses the
+            main 'scons' invocation.
+        '''
+
+        if build_dir is None:
+            build_dir = config.build_dir
+        self.target = os.path.join(build_dir, target)
+        super(SConsTarget, self).__init__(name=target)
+
+        if invocation is None:
+            if self.default_scons_invocation is None:
+                SConsTarget.default_scons_invocation = SConsFixture()
+                globalfixture(SConsTarget.default_scons_invocation)
+
+            invocation = self.default_scons_invocation
+        self.invocation = invocation
+
+    def schedule_finalized(self, schedule):
+        self.invocation.targets.add(self.target)
+        return Fixture.schedule_finalized(self, schedule)
+
+class Gem5Fixture(SConsTarget):
+    def __init__(self, isa, variant):
+        target = joinpath(isa.upper(), 'gem5.%s' % variant)
+        super(Gem5Fixture, self).__init__(target)
+
+        self.name = constants.gem5_binary_fixture_name
+        self.path = self.target
+        self.isa = isa
+        self.variant = variant
+
+
+class MakeFixture(Fixture):
+    def __init__(self, directory, *args, **kwargs):
+        name = 'make -C %s' % directory
+        super(MakeFixture, self).__init__(build_once=True, lazy_init=False,
+                                          name=name,
+                                          *args, **kwargs)
+        self.targets = []
+        self.directory = directory
+
+    def setup(self):
+        super(MakeFixture, self).setup()
+        targets = set(self.required_by)
+        command = ['make', '-C', self.directory]
+        command.extend([target.target for target in targets])
+        log_call(command)
+
+
+class MakeTarget(Fixture):
+    def __init__(self, target, make_fixture=None, *args, **kwargs):
+        '''
+        :param make_fixture: The make invocation we will be attached to.
+        Since we don't have a single global instance of make in gem5 like we do
+        scons we need to know what invocation to attach to. If none given,
+        creates its own.
+        '''
+        super(MakeTarget, self).__init__(name=target, *args, **kwargs)
+        self.target = self.name
+
+        if make_fixture is None:
+            make_fixture = MakeFixture(
+                    absdirpath(target),
+                    lazy_init=True,
+                    build_once=False)
+
+        self.make_fixture = make_fixture
+
+        # Add our self to the required targets of the main MakeFixture
+        self.require(self.make_fixture)
+
+    def setup(self, testitem):
+        super(MakeTarget, self).setup()
+        self.make_fixture.setup()
+        return self
+
+class TestProgram(MakeTarget):
+    def __init__(self, program, isa, os, recompile=False):
+        make_dir = joinpath('test-progs', program)
+        make_fixture = MakeFixture(make_dir)
+        target = joinpath('bin', isa, os, program)
+        super(TestProgram, self).__init__(target, make_fixture)
+        self.path = joinpath(make_dir, target)
+        self.recompile = recompile
+
+    def setup(self, testitem):
+        # Check if the program exists if it does then only compile if
+        # recompile was given.
+        if self.recompile:
+            super(MakeTarget, self).setup()
+        elif not os.path.exists(self.path):
+            super(MakeTarget, self).setup()
+
+class DownloadedProgram(Fixture):
+    """ Like TestProgram, but checks the version in the gem5 binary repository
+        and downloads an updated version if it is needed.
+    """
+    urlbase = "http://gem5.org/dist/current/"
+
+    def __init__(self, path, program, **kwargs):
+        super(DownloadedProgram, self).__init__("download-" + program,
+                                                build_once=True, **kwargs)
+
+        self.program_dir = joinpath('test-progs', path)
+        self.path = joinpath(self.program_dir, program)
+
+        self.url = self.urlbase + self.path
+
+    def _download(self):
+        import urllib
+        log.test_log.debug("Downloading " + self.url + " to " + self.path)
+        if not os.path.exists(self.program_dir):
+            os.makedirs(self.program_dir)
+        urllib.urlretrieve(self.url, self.path)
+
+    def _getremotetime(self):
+        import  urllib2, datetime, time
+        import _strptime # Needed for python threading bug
+
+        u = urllib2.urlopen(self.url)
+        return time.mktime(datetime.datetime.strptime( \
+                    u.info().getheaders("Last-Modified")[0],
+                    "%a, %d %b %Y %X GMT").timetuple())
+
+    def setup(self, testitem):
+        import urllib2
+        # Check to see if there is a file downloaded
+        if not os.path.exists(self.path):
+            self._download()
+        else:
+            try:
+                t = self._getremotetime()
+            except urllib2.URLError:
+                # Problem checking the server, use the old files.
+                log.debug("Could not contact server. Binaries may be old.")
+                return
+            # If the server version is more recent, download it
+            if t > os.path.getmtime(self.path):
+                self._download()

tests/gem5/suite.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+import os
+import copy
+import subprocess
+
+from testlib.test import TestFunction
+from testlib.suite import TestSuite
+from testlib.helper import log_call
+from testlib.config import constants, config
+from fixture import TempdirFixture, Gem5Fixture, VariableFixture
+import verifier
+
+def gem5_verify_config(name,
+                       config,
+                       config_args,
+                       verifiers,
+                       gem5_args=tuple(),
+                       fixtures=[],
+                       valid_isas=constants.supported_isas,
+                       valid_variants=constants.supported_variants,
+                       length=constants.supported_lengths[0]):
+    '''
+    Helper class to generate common gem5 tests using verifiers.
+
+    The generated TestSuite will run gem5 with the provided config and
+    config_args. After that it will run any provided verifiers to verify
+    details about the gem5 run.
+
+    .. seealso::  For the verifiers see :mod:`testlib.gem5.verifier`
+
+    :param name: Name of the test.
+    :param config: The config to give gem5.
+    :param config_args: A list of arguments to pass to the given config.
+
+    :param verifiers: An iterable with Verifier instances which will be placed
+        into a suite that will be ran after a gem5 run.
+
+    :param gem5_args: An iterable with arguments to give to gem5. (Arguments
+        that would normally go before the config path.)
+
+    :param valid_isas: An iterable with the isas that this test can be ran
+        for. If None given, will run for all supported_isas.
+
+    :param valid_variants: An iterable with the variant levels that
+        this test can be ran for. (E.g. opt, debug)
+    '''
+    fixtures = list(fixtures)
+    testsuites = []
+    for opt in valid_variants:
+        for isa in valid_isas:
+
+            # Create a tempdir fixture to be shared throughout the test.
+            tempdir = TempdirFixture()
+            gem5_returncode = VariableFixture(
+                    name=constants.gem5_returncode_fixture_name)
+
+            # Common name of this generated testcase.
+            _name = '{given_name}-{isa}-{opt}'.format(
+                    given_name=name,
+                    isa=isa,
+                    opt=opt)
+
+            # Create the running of gem5 subtest.
+            # NOTE: We specifically create this test before our verifiers so
+            # this is listed first.
+            tests = []
+            gem5_execution = TestFunction(
+                    _create_test_run_gem5(config, config_args, gem5_args),
+                    name=_name)
+            tests.append(gem5_execution)
+
+            # Create copies of the verifier subtests for this isa and
+            # variant.
+            for verifier in verifiers:
+                tests.append(verifier.instantiate_test(_name))
+
+            # Add the isa and variant to tags list.
+            tags = [isa, opt, length]
+
+            # Create the gem5 target for the specific architecture and
+            # variant.
+            _fixtures = copy.copy(fixtures)
+            _fixtures.append(Gem5Fixture(isa, opt))
+            _fixtures.append(tempdir)
+            _fixtures.append(gem5_returncode)
+
+            # Finally construct the self contained TestSuite out of our
+            # tests.
+            testsuites.append(TestSuite(
+                name=_name,
+                fixtures=_fixtures,
+                tags=tags,
+                tests=tests))
+    return testsuites
+
+def _create_test_run_gem5(config, config_args, gem5_args):
+    def test_run_gem5(params):
+        '''
+        Simple \'test\' which runs gem5 and saves the result into a tempdir.
+
+        NOTE: Requires fixtures: tempdir, gem5
+        '''
+        fixtures = params.fixtures
+
+        if gem5_args is None:
+            _gem5_args = tuple()
+        elif isinstance(gem5_args, str):
+            # If just a single str, place it in an iterable
+            _gem5_args = (gem5_args,)
+        else:
+            _gem5_args = gem5_args
+
+        # FIXME/TODO: I don't like the idea of having to modify this test run
+        # or always collect results even if not using a verifier. There should
+        # be some configuration in here that only gathers certain results for
+        # certain verifiers.
+        #
+        # I.E. Only the returncode verifier will use the gem5_returncode
+        # fixture, but we always require it even if that verifier isn't being
+        # ran.
+        returncode = fixtures[constants.gem5_returncode_fixture_name]
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+        gem5 = fixtures[constants.gem5_binary_fixture_name].path
+        command = [
+            gem5,
+            '-d',  # Set redirect dir to tempdir.
+            tempdir,
+            '-re',# TODO: Change to const. Redirect stdout and stderr
+        ]
+        command.extend(_gem5_args)
+        command.append(config)
+        # Config_args should set up the program args.
+        command.extend(config_args)
+        returncode.value = log_call(params.log, command)
+
+    return test_run_gem5

tests/gem5/verifier.py

@@ -0,0 +1,205 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+Built in test cases that verify particular details about a gem5 run.
+'''
+import re
+
+from testlib import test
+from testlib.config import constants
+from testlib.helper import joinpath, diff_out_file
+
+class Verifier(object):
+    def __init__(self, fixtures=tuple()):
+        self.fixtures = fixtures
+
+    def _test(self, *args, **kwargs):
+        # Use a callback wrapper to make stack
+        # traces easier to understand.
+        self.test(*args, **kwargs)
+
+    def instantiate_test(self, name_pfx):
+        name = '-'.join([name_pfx, self.__class__.__name__])
+        return test.TestFunction(self._test,
+                name=name, fixtures=self.fixtures)
+
+    def failed(self, fixtures):
+        '''
+        Called if this verifier fails to cleanup (or not) as needed.
+        '''
+        try:
+            fixtures[constants.tempdir_fixture_name].skip_cleanup()
+        except KeyError:
+            pass # No need to do anything if the tempdir fixture doesn't exist
+
+
+class MatchGoldStandard(Verifier):
+    '''
+    Compares a standard output to the test output and passes if they match,
+    fails if they do not.
+    '''
+    def __init__(self, standard_filename, ignore_regex=None,
+                 test_filename='simout'):
+        '''
+        :param standard_filename: The path of the standard file to compare
+        output to.
+
+        :param ignore_regex: A string, compiled regex, or iterable containing
+        either which will be ignored in 'standard' and test output files when
+        diffing.
+        '''
+        super(MatchGoldStandard, self).__init__()
+        self.standard_filename = standard_filename
+        self.test_filename = test_filename
+
+        self.ignore_regex = _iterable_regex(ignore_regex)
+
+    def test(self, params):
+        # We need a tempdir fixture from our parent verifier suite.
+        fixtures = params.fixtures
+        # Get the file from the tempdir of the test.
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+        self.test_filename = joinpath(tempdir, self.test_filename)
+
+        diff = diff_out_file(self.standard_filename,
+                            self.test_filename,
+                            ignore_regexes=self.ignore_regex,
+                            logger=params.log)
+        if diff is not None:
+            self.failed(fixtures)
+            test.fail('Stdout did not match:\n%s\nSee %s for full results'
+                      % (diff, tempdir))
+
+    def _generic_instance_warning(self, kwargs):
+        '''
+        Method for helper classes to tell users to use this more generic class
+        if they are going to manually override the test_filename param.
+        '''
+        if 'test_filename' in kwargs:
+            raise ValueError('If you are setting test_filename use the more'
+                             ' generic %s'
+                             ' instead' % MatchGoldStandard.__name__)
+
+class DerivedGoldStandard(MatchGoldStandard):
+    __ignore_regex_sentinel = object()
+    _file = None
+    _default_ignore_regex = []
+
+    def __init__(self, standard_filename,
+                 ignore_regex=__ignore_regex_sentinel, **kwargs):
+
+        if ignore_regex == self.__ignore_regex_sentinel:
+            ignore_regex = self._default_ignore_regex
+
+        self._generic_instance_warning(kwargs)
+
+        super(DerivedGoldStandard, self).__init__(
+            standard_filename,
+            test_filename=self._file,
+            ignore_regex=ignore_regex,
+            **kwargs)
+
+class MatchStdout(DerivedGoldStandard):
+    _file = constants.gem5_simulation_stdout
+    _default_ignore_regex = [
+            re.compile('^Redirecting (stdout|stderr) to'),
+            re.compile('^gem5 compiled '),
+            re.compile('^gem5 started '),
+            re.compile('^gem5 executing on '),
+            re.compile('^command line:'),
+            re.compile("^Couldn't import dot_parser,"),
+            re.compile("^info: kernel located at:"),
+            re.compile("^Couldn't unlink "),
+            re.compile("^Using GPU kernel code file\(s\) "),
+        ]
+
+class MatchStdoutNoPerf(MatchStdout):
+    _file = constants.gem5_simulation_stdout
+    _default_ignore_regex = MatchStdout._default_ignore_regex + [
+            re.compile('^Exiting @ tick'),
+        ]
+
+class MatchStderr(DerivedGoldStandard):
+    _file = constants.gem5_simulation_stderr
+    _default_ignore_regex = []
+
+class MatchStats(DerivedGoldStandard):
+    # TODO: Likely will want to change this verifier since we have the weird
+    # perl script right now. A simple diff probably isn't going to work.
+    _file = constants.gem5_simulation_stats
+    _default_ignore_regex = []
+
+class MatchConfigINI(DerivedGoldStandard):
+    _file = constants.gem5_simulation_config_ini
+    _default_ignore_regex = (
+            re.compile("^(executable|readfile|kernel|image_file)="),
+            re.compile("^(cwd|input|codefile)="),
+            )
+
+class MatchConfigJSON(DerivedGoldStandard):
+    _file = constants.gem5_simulation_config_json
+    _default_ignore_regex = (
+            re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
+            re.compile(r'''^\s*"(cwd|input|codefile)":'''),
+            )
+
+class MatchRegex(Verifier):
+    def __init__(self, regex, match_stderr=True, match_stdout=True):
+        super(MatchRegex, self).__init__()
+        self.regex = _iterable_regex(regex)
+        self.match_stderr = match_stderr
+        self.match_stdout = match_stdout
+
+    def test(self, params):
+        fixtures = params.fixtures
+        # Get the file from the tempdir of the test.
+        tempdir = fixtures[constants.tempdir_fixture_name].path
+
+        def parse_file(fname):
+            with open(fname, 'r') as file_:
+                for line in file_:
+                    for regex in self.regex:
+                        if re.match(regex, line):
+                            return True
+        if self.match_stdout:
+            if parse_file(joinpath(tempdir,
+                                   constants.gem5_simulation_stdout)):
+                return # Success
+        if self.match_stderr:
+            if parse_file(joinpath(tempdir,
+                                   constants.gem5_simulation_stderr)):
+                return # Success
+        self.failed(fixtures)
+        test.fail('Could not match regex.')
+
+_re_type = type(re.compile(''))
+def _iterable_regex(regex):
+    if isinstance(regex, _re_type) or isinstance(regex, str):
+        regex = (regex,)
+    return regex

tests/legacy-configs/run.py

@@ -0,0 +1,121 @@
+# Copyright (c) 2017 Mark D. Hill and David A. Wood
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met: redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer;
+# redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution;
+# neither the name of the copyright holders nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Authors: Sean Wilson
+
+'''
+New version of the run.py script. For this, all dependencies should be
+handled outside of the script.
+
+.. warning:: This script is NOT the recommended way to handle configurations
+    for new tests. This exists for legacy support only. New Tests should
+    either use configs from the normal gem5 configs or create their own for
+    a test.
+'''
+import argparse
+import sys
+import os
+from os.path import abspath, join as joinpath, dirname
+
+import m5
+
+# Add the normal gem5 config path to system path.
+# This requirement should be removed if possible from all legacy scripts, but
+# I've left it here for now.
+sys.path.insert(0, abspath(joinpath(dirname(__file__), '../../configs')))
+
+# set default maxtick... script can override
+# -1 means run forever
+maxtick = m5.MaxTick
+
+def run_test(root):
+    """Default run_test implementations. Scripts can override it."""
+
+    # instantiate configuration
+    m5.instantiate()
+
+    # simulate until program terminates
+    exit_event = m5.simulate(maxtick)
+    print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()
+
+test_progs = os.environ.get('M5_TEST_PROGS', '/dist/m5/regression/test-progs')
+
+# Since we're in batch mode, dont allow tcp socket connections
+m5.disableAllListeners()
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--cmd',
+                    action='store',
+                    type=str,
+                    help='Command to pass to the test system')
+parser.add_argument('--executable',
+                    action='store',
+                    type=str,
+                    help='Executable to pass to the test system')
+parser.add_argument('--config',
+                    action='append',
+                    type=str,
+                    help='A config file to initialize the system with.'\
+                    + ' If more than one given, loads them in order given.')
+args = parser.parse_args()
+
+executable = args.executable
+
+for config in args.config:
+    execfile(config)
+
+# Initialize all CPUs in a system
+def initCPUs(sys):
+    def initCPU(cpu):
+        # We might actually have a MemTest object or something similar
+        # here that just pretends to be a CPU.
+        try:
+            cpu.createThreads()
+        except:
+            pass
+
+    # The CPU attribute doesn't exist in some cases, e.g. the Ruby testers.
+    if not hasattr(sys, "cpu"):
+        return
+
+    # The CPU can either be a list of CPUs or a single object.
+    if isinstance(sys.cpu, list):
+        [ initCPU(cpu) for cpu in sys.cpu ]
+    else:
+        initCPU(sys.cpu)
+
+# TODO: Might want to automatically place the cmd and executable on the
+# cpu[0].workload, although I think most legacy configs do this automatically
+# or somewhere in their `test.py` config.
+
+
+# We might be creating a single system or a dual system. Try
+# initializing the CPUs in all known system attributes.
+for sysattr in [ "system", "testsys", "drivesys" ]:
+    if hasattr(root, sysattr):
+        initCPUs(getattr(root, sysattr))
+
+run_test(root)

tests/main.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python2
+'''
+The main source for testlib. Ties together the default test runners and
+loaders.
+
+Discovers and runs all tests from a given root directory.
+'''
+from __future__ import print_function
+
+import sys
+import os
+
+base_dir = os.path.dirname(os.path.abspath(__name__))
+ext_path = os.path.join(base_dir, os.pardir, 'ext')
+
+sys.path.insert(0, base_dir)
+sys.path.insert(0, ext_path)
+
+import testlib.main as testlib
+import testlib.config as config
+import testlib.helper as helper
+
+config.basedir = helper.absdirpath(__file__)
+testlib()