tests: Get rid of the tests/testing python package.
This was used by the now deleted tests/tests.py script. Change-Id: I18481b02a78432b88e6cd9226a4c046bc6433743 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/32119 Tested-by: kokoro <noreply+kokoro@google.com> Reviewed-by: Jason Lowe-Power <power.jg@gmail.com> Maintainer: Bobby R. Bruce <bbruce@ucdavis.edu>
This commit is contained in:
Gabe Black
@@ -1,36 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2016 ARM Limited
|
||||
# All rights reserved
|
||||
#
|
||||
# The license below extends only to copyright in the software and shall
|
||||
# not be construed as granting a license to any other intellectual
|
||||
# property including but not limited to intellectual property relating
|
||||
# to a hardware implementation of the functionality of the software
|
||||
# licensed hereunder. You may use the software subject to the license
|
||||
# terms below provided that you ensure that this notice is replicated
|
||||
# unmodified and in its entirety in all distributions of the software,
|
||||
# modified or unmodified, in source code or in binary form.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2016 ARM Limited
|
||||
# All rights reserved
|
||||
#
|
||||
# The license below extends only to copyright in the software and shall
|
||||
# not be construed as granting a license to any other intellectual
|
||||
# property including but not limited to intellectual property relating
|
||||
# to a hardware implementation of the functionality of the software
|
||||
# licensed hereunder. You may use the software subject to the license
|
||||
# terms below provided that you ensure that this notice is replicated
|
||||
# unmodified and in its entirety in all distributions of the software,
|
||||
# modified or unmodified, in source code or in binary form.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import subprocess
|
||||
from threading import Timer
|
||||
import time
|
||||
import re
|
||||
|
||||
class CallTimeoutException(Exception):
|
||||
"""Exception that indicates that a process call timed out"""
|
||||
|
||||
def __init__(self, status, stdout, stderr):
|
||||
self.status = status
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
class ProcessHelper(subprocess.Popen):
|
||||
"""Helper class to run child processes.
|
||||
|
||||
This class wraps a subprocess.Popen class and adds support for
|
||||
using it in a with block. When the process goes out of scope, it's
|
||||
automatically terminated.
|
||||
|
||||
with ProcessHelper(["/bin/ls"], stdout=subprocess.PIPE) as p:
|
||||
return p.call()
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ProcessHelper, self).__init__(*args, **kwargs)
|
||||
|
||||
def _terminate_nicely(self, timeout=5):
|
||||
def on_timeout():
|
||||
self.kill()
|
||||
|
||||
if self.returncode is not None:
|
||||
return self.returncode
|
||||
|
||||
timer = Timer(timeout, on_timeout)
|
||||
self.terminate()
|
||||
status = self.wait()
|
||||
timer.cancel()
|
||||
|
||||
return status
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
if self.returncode is None:
|
||||
self._terminate_nicely()
|
||||
|
||||
def call(self, timeout=0):
|
||||
self._timeout = False
|
||||
def on_timeout():
|
||||
self._timeout = True
|
||||
self._terminate_nicely()
|
||||
|
||||
status, stdout, stderr = None, None, None
|
||||
timer = Timer(timeout, on_timeout)
|
||||
if timeout:
|
||||
timer.start()
|
||||
|
||||
stdout, stderr = self.communicate()
|
||||
status = self.wait()
|
||||
|
||||
timer.cancel()
|
||||
|
||||
if self._timeout:
|
||||
self._terminate_nicely()
|
||||
raise CallTimeoutException(self.returncode, stdout, stderr)
|
||||
else:
|
||||
return status, stdout, stderr
|
||||
|
||||
class FileIgnoreList(object):
|
||||
"""Helper class to implement file ignore lists.
|
||||
|
||||
This class implements ignore lists using plain string matching and
|
||||
regular expressions. In the simplest use case, rules are created
|
||||
statically upon initialization:
|
||||
|
||||
ignore_list = FileIgnoreList(name=("ignore_me.txt", ), rex=(r".*~", )
|
||||
|
||||
Ignores can be queried using in the same ways as normal Python
|
||||
containers:
|
||||
|
||||
if file_name in ignore_list:
|
||||
print "Ignoring %s" % file_name
|
||||
|
||||
|
||||
New rules can be added at runtime by extending the list in the
|
||||
rules attribute:
|
||||
|
||||
ignore_list.rules.append(FileIgnoreList.simple("bar.txt"))
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def simple(r):
|
||||
return lambda f: f == r
|
||||
|
||||
@staticmethod
|
||||
def rex(r):
|
||||
re_obj = r if hasattr(r, "search") else re.compile(r)
|
||||
return lambda name: re_obj.search(name)
|
||||
|
||||
def __init__(self, names=(), rex=()):
|
||||
self.rules = [ FileIgnoreList.simple(n) for n in names ] + \
|
||||
[ FileIgnoreList.rex(r) for r in rex ]
|
||||
|
||||
def __contains__(self, name):
|
||||
for rule in self.rules:
|
||||
if rule(name):
|
||||
return True
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run internal self tests to ensure that the helpers are working
|
||||
# properly. The expected output when running this script is
|
||||
# "SUCCESS!".
|
||||
|
||||
cmd_foo = [ "/bin/echo", "-n", "foo" ]
|
||||
cmd_sleep = [ "/bin/sleep", "10" ]
|
||||
|
||||
# Test that things don't break if the process hasn't been started
|
||||
with ProcessHelper(cmd_foo) as p:
|
||||
pass
|
||||
|
||||
with ProcessHelper(cmd_foo, stdout=subprocess.PIPE) as p:
|
||||
status, stdout, stderr = p.call()
|
||||
assert stdout == "foo"
|
||||
assert status == 0
|
||||
|
||||
try:
|
||||
with ProcessHelper(cmd_sleep) as p:
|
||||
status, stdout, stderr = p.call(timeout=1)
|
||||
assert False, "Timeout not triggered"
|
||||
except CallTimeoutException:
|
||||
pass
|
||||
|
||||
ignore_list = FileIgnoreList(
|
||||
names=("ignore.txt", "foo/test.txt"),
|
||||
rex=(r"~$", re.compile("^#")))
|
||||
|
||||
assert "ignore.txt" in ignore_list
|
||||
assert "bar.txt" not in ignore_list
|
||||
assert "foo/test.txt" in ignore_list
|
||||
assert "test.txt" not in ignore_list
|
||||
assert "file1.c~" in ignore_list
|
||||
assert "file1.c" not in ignore_list
|
||||
assert "#foo" in ignore_list
|
||||
assert "foo#" not in ignore_list
|
||||
|
||||
ignore_list.rules.append(FileIgnoreList.simple("bar.txt"))
|
||||
assert "bar.txt" in ignore_list
|
||||
|
||||
print("SUCCESS!")
|
||||
@@ -1,298 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2016 ARM Limited
|
||||
# All rights reserved
|
||||
#
|
||||
# The license below extends only to copyright in the software and shall
|
||||
# not be construed as granting a license to any other intellectual
|
||||
# property including but not limited to intellectual property relating
|
||||
# to a hardware implementation of the functionality of the software
|
||||
# licensed hereunder. You may use the software subject to the license
|
||||
# terms below provided that you ensure that this notice is replicated
|
||||
# unmodified and in its entirety in all distributions of the software,
|
||||
# modified or unmodified, in source code or in binary form.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
import inspect
|
||||
import pickle
|
||||
from six import add_metaclass
|
||||
import string
|
||||
import sys
|
||||
|
||||
import xml.etree.cElementTree as ET
|
||||
|
||||
class UnitResult(object):
|
||||
"""Results of a single test unit.
|
||||
|
||||
A test result can be one of:
|
||||
- STATE_OK: Test ran successfully.
|
||||
- STATE_SKIPPED: The test was skipped.
|
||||
- STATE_ERROR: The test failed to run.
|
||||
- STATE_FAILED: Test ran, but failed.
|
||||
|
||||
The difference between STATE_ERROR and STATE_FAILED is very
|
||||
subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
|
||||
to start or crashed, while STATE_FAILED would mean that a test
|
||||
failed (e.g., statistics mismatch).
|
||||
|
||||
"""
|
||||
|
||||
STATE_OK = 0
|
||||
STATE_SKIPPED = 1
|
||||
STATE_ERROR = 2
|
||||
STATE_FAILURE = 3
|
||||
|
||||
state_names = {
|
||||
STATE_OK : "OK",
|
||||
STATE_SKIPPED : "SKIPPED",
|
||||
STATE_ERROR : "ERROR",
|
||||
STATE_FAILURE : "FAILURE",
|
||||
}
|
||||
|
||||
def __init__(self, name, state, message="", stderr="", stdout="",
|
||||
runtime=0.0):
|
||||
self.name = name
|
||||
self.state = state
|
||||
self.message = message
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.runtime = runtime
|
||||
|
||||
def skipped(self):
|
||||
return self.state == UnitResult.STATE_SKIPPED
|
||||
|
||||
def success(self):
|
||||
return self.state == UnitResult.STATE_OK
|
||||
|
||||
def state_name(self):
|
||||
return UnitResult.state_names[self.state]
|
||||
|
||||
def __nonzero__(self):
|
||||
return self.success() or self.skipped()
|
||||
|
||||
def __str__(self):
|
||||
state_name = self.state_name()
|
||||
|
||||
status = "%s: %s" % (state_name, self.message) if self.message else \
|
||||
state_name
|
||||
|
||||
return "%s: %s" % (self.name, status)
|
||||
|
||||
class TestResult(object):
|
||||
"""Results for from a single test consisting of one or more units."""
|
||||
|
||||
def __init__(self, name, run_results=[], verify_results=[]):
|
||||
self.name = name
|
||||
self.results = run_results + verify_results
|
||||
self.run_results = run_results
|
||||
self.verify_results = verify_results
|
||||
|
||||
def success(self):
|
||||
return self.success_run() and self.success_verify()
|
||||
|
||||
def success_run(self):
|
||||
return all([ r.success() for r in self.run_results ])
|
||||
|
||||
def success_verify(self):
|
||||
return all([ r.success() for r in self.verify_results ])
|
||||
|
||||
def failed(self):
|
||||
return self.failed_run() or self.failed_verify()
|
||||
|
||||
def failed_run(self):
|
||||
return any([ not r for r in self.run_results ])
|
||||
|
||||
def failed_verify(self):
|
||||
return any([ not r for r in self.verify_results ])
|
||||
|
||||
def skipped(self):
|
||||
return all([ r.skipped() for r in self.run_results ])
|
||||
|
||||
def changed(self):
|
||||
return self.success_run() and self.failed_verify()
|
||||
|
||||
def runtime(self):
|
||||
return sum([ r.runtime for r in self.results ])
|
||||
|
||||
def __nonzero__(self):
|
||||
return all([ r for r in self.results ])
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class ResultFormatter(object):
|
||||
|
||||
def __init__(self, fout=sys.stdout, verbose=False):
|
||||
self.verbose = verbose
|
||||
self.fout = fout
|
||||
|
||||
@abstractmethod
|
||||
def dump_suites(self, suites):
|
||||
pass
|
||||
|
||||
class Pickle(ResultFormatter):
|
||||
"""Save test results as a binary using Python's pickle
|
||||
functionality.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(Pickle, self).__init__(**kwargs)
|
||||
|
||||
def dump_suites(self, suites):
|
||||
pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
class Text(ResultFormatter):
|
||||
"""Output test results as text."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(Text, self).__init__(**kwargs)
|
||||
|
||||
def dump_suites(self, suites):
|
||||
fout = self.fout
|
||||
for suite in suites:
|
||||
print("--- %s ---" % suite.name, file=fout)
|
||||
|
||||
for t in suite.results:
|
||||
print("*** %s" % t, file=fout)
|
||||
|
||||
if t and not self.verbose:
|
||||
continue
|
||||
|
||||
if t.message:
|
||||
print(t.message, file=fout)
|
||||
|
||||
if t.stderr:
|
||||
print(t.stderr, file=fout)
|
||||
if t.stdout:
|
||||
print(t.stdout, file=fout)
|
||||
|
||||
class TextSummary(ResultFormatter):
|
||||
"""Output test results as a text summary"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(TextSummary, self).__init__(**kwargs)
|
||||
|
||||
def test_status(self, suite):
|
||||
if suite.skipped():
|
||||
return "SKIPPED"
|
||||
elif suite.changed():
|
||||
return "CHANGED"
|
||||
elif suite:
|
||||
return "OK"
|
||||
else:
|
||||
return "FAILED"
|
||||
|
||||
def dump_suites(self, suites):
|
||||
fout = self.fout
|
||||
for suite in suites:
|
||||
status = self.test_status(suite)
|
||||
print("%s: %s" % (suite.name, status), file=fout)
|
||||
|
||||
class JUnit(ResultFormatter):
|
||||
"""Output test results as JUnit XML"""
|
||||
|
||||
def __init__(self, translate_names=True, **kwargs):
|
||||
super(JUnit, self).__init__(**kwargs)
|
||||
|
||||
if translate_names:
|
||||
self.name_table = string.maketrans(
|
||||
"/.",
|
||||
".-",
|
||||
)
|
||||
else:
|
||||
self.name_table = string.maketrans("", "")
|
||||
|
||||
def convert_unit(self, x_suite, test):
|
||||
x_test = ET.SubElement(x_suite, "testcase",
|
||||
name=test.name,
|
||||
time="%f" % test.runtime)
|
||||
|
||||
x_state = None
|
||||
if test.state == UnitResult.STATE_OK:
|
||||
pass
|
||||
elif test.state == UnitResult.STATE_SKIPPED:
|
||||
x_state = ET.SubElement(x_test, "skipped")
|
||||
elif test.state == UnitResult.STATE_FAILURE:
|
||||
x_state = ET.SubElement(x_test, "failure")
|
||||
elif test.state == UnitResult.STATE_ERROR:
|
||||
x_state = ET.SubElement(x_test, "error")
|
||||
else:
|
||||
assert False, "Unknown test state"
|
||||
|
||||
if x_state is not None:
|
||||
if test.message:
|
||||
x_state.set("message", test.message)
|
||||
|
||||
msg = []
|
||||
if test.stderr:
|
||||
msg.append("*** Standard Errror: ***")
|
||||
msg.append(test.stderr)
|
||||
if test.stdout:
|
||||
msg.append("*** Standard Out: ***")
|
||||
msg.append(test.stdout)
|
||||
|
||||
x_state.text = "\n".join(msg)
|
||||
|
||||
return x_test
|
||||
|
||||
def convert_suite(self, x_suites, suite):
|
||||
x_suite = ET.SubElement(x_suites, "testsuite",
|
||||
name=suite.name.translate(self.name_table),
|
||||
time="%f" % suite.runtime())
|
||||
errors = 0
|
||||
failures = 0
|
||||
skipped = 0
|
||||
|
||||
for test in suite.results:
|
||||
if test.state != UnitResult.STATE_OK:
|
||||
if test.state == UnitResult.STATE_SKIPPED:
|
||||
skipped += 1
|
||||
elif test.state == UnitResult.STATE_ERROR:
|
||||
errors += 1
|
||||
elif test.state == UnitResult.STATE_FAILURE:
|
||||
failures += 1
|
||||
|
||||
x_test = self.convert_unit(x_suite, test)
|
||||
|
||||
x_suite.set("errors", str(errors))
|
||||
x_suite.set("failures", str(failures))
|
||||
x_suite.set("skipped", str(skipped))
|
||||
x_suite.set("tests", str(len(suite.results)))
|
||||
|
||||
return x_suite
|
||||
|
||||
def convert_suites(self, suites):
|
||||
x_root = ET.Element("testsuites")
|
||||
|
||||
for suite in suites:
|
||||
self.convert_suite(x_root, suite)
|
||||
|
||||
return x_root
|
||||
|
||||
def dump_suites(self, suites):
|
||||
et = ET.ElementTree(self.convert_suites(suites))
|
||||
et.write(self.fout, encoding="UTF-8")
|
||||
@@ -1,374 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2016-2017 ARM Limited
|
||||
# All rights reserved
|
||||
#
|
||||
# The license below extends only to copyright in the software and shall
|
||||
# not be construed as granting a license to any other intellectual
|
||||
# property including but not limited to intellectual property relating
|
||||
# to a hardware implementation of the functionality of the software
|
||||
# licensed hereunder. You may use the software subject to the license
|
||||
# terms below provided that you ensure that this notice is replicated
|
||||
# unmodified and in its entirety in all distributions of the software,
|
||||
# modified or unmodified, in source code or in binary form.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
import os
|
||||
from collections import namedtuple
|
||||
|
||||
from six import add_metaclass
|
||||
|
||||
import sys
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
|
||||
from units import *
|
||||
from helpers import FileIgnoreList
|
||||
from results import TestResult
|
||||
import shutil
|
||||
|
||||
_test_base = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
||||
ClassicConfig = namedtuple("ClassicConfig", (
|
||||
"category",
|
||||
"mode",
|
||||
"workload",
|
||||
"isa",
|
||||
"os",
|
||||
"config",
|
||||
))
|
||||
|
||||
# There are currently two "classes" of test
|
||||
# configurations. Architecture-specific ones and generic ones
|
||||
# (typically SE mode tests). In both cases, the configuration name
|
||||
# matches a file in tests/configs/ that will be picked up by the test
|
||||
# runner (run.py).
|
||||
#
|
||||
# Architecture specific configurations are listed in the arch_configs
|
||||
# dictionary. This is indexed by a (cpu architecture, gpu
|
||||
# architecture) tuple. GPU architecture is optional and may be None.
|
||||
#
|
||||
# Generic configurations are listed in the generic_configs tuple.
|
||||
#
|
||||
# When discovering available test cases, this script look uses the
|
||||
# test list as a list of /candidate/ configurations. A configuration
|
||||
# is only used if a test has a reference output for that
|
||||
# configuration. In addition to the base configurations from
|
||||
# arch_configs and generic_configs, a Ruby configuration may be
|
||||
# appended to the base name (this is probed /in addition/ to the
|
||||
# original name. See get_tests() for details.
|
||||
#
|
||||
arch_configs = {
|
||||
("arm", None) : (
|
||||
'simple-atomic-dummychecker',
|
||||
'o3-timing-checker',
|
||||
'realview-simple-atomic',
|
||||
'realview-simple-atomic-dual',
|
||||
'realview-simple-atomic-checkpoint',
|
||||
'realview-simple-timing',
|
||||
'realview-simple-timing-dual',
|
||||
'realview-o3',
|
||||
'realview-o3-checker',
|
||||
'realview-o3-dual',
|
||||
'realview-minor',
|
||||
'realview-minor-dual',
|
||||
'realview-switcheroo-atomic',
|
||||
'realview-switcheroo-timing',
|
||||
'realview-switcheroo-noncaching-timing',
|
||||
'realview-switcheroo-o3',
|
||||
'realview-switcheroo-full',
|
||||
'realview64-simple-atomic',
|
||||
'realview64-simple-atomic-checkpoint',
|
||||
'realview64-simple-atomic-dual',
|
||||
'realview64-simple-timing',
|
||||
'realview64-simple-timing-dual',
|
||||
'realview64-o3',
|
||||
'realview64-o3-checker',
|
||||
'realview64-o3-dual',
|
||||
'realview64-minor',
|
||||
'realview64-minor-dual',
|
||||
'realview64-switcheroo-atomic',
|
||||
'realview64-switcheroo-timing',
|
||||
'realview64-switcheroo-o3',
|
||||
'realview64-switcheroo-full',
|
||||
),
|
||||
|
||||
("sparc", None) : (
|
||||
't1000-simple-atomic',
|
||||
't1000-simple-x86',
|
||||
),
|
||||
|
||||
("x86", None) : (
|
||||
'pc-simple-atomic',
|
||||
'pc-simple-timing',
|
||||
'pc-o3-timing',
|
||||
'pc-switcheroo-full',
|
||||
),
|
||||
|
||||
("x86", "hsail") : (
|
||||
'gpu',
|
||||
),
|
||||
}
|
||||
|
||||
generic_configs = (
|
||||
'simple-atomic',
|
||||
'simple-atomic-mp',
|
||||
'simple-timing',
|
||||
'simple-timing-mp',
|
||||
|
||||
'minor-timing',
|
||||
'minor-timing-mp',
|
||||
|
||||
'o3-timing',
|
||||
'o3-timing-mt',
|
||||
'o3-timing-mp',
|
||||
|
||||
'rubytest',
|
||||
'memcheck',
|
||||
'memtest',
|
||||
'memtest-filter',
|
||||
'tgen-simple-mem',
|
||||
'tgen-dram-ctrl',
|
||||
'dram-lowp',
|
||||
|
||||
'learning-gem5-p1-simple',
|
||||
'learning-gem5-p1-two-level',
|
||||
)
|
||||
|
||||
default_ruby_protocol = {
|
||||
"arm" : "MOESI_CMP_directory",
|
||||
}
|
||||
|
||||
def get_default_protocol(arch):
|
||||
return default_ruby_protocol.get(arch, 'MI_example')
|
||||
|
||||
all_categories = ("quick", "long")
|
||||
all_modes = ("fs", "se")
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class Test(object):
|
||||
"""Test case base class.
|
||||
|
||||
Test cases consists of one or more test units that are run in two
|
||||
phases. A run phase (units produced by run_units() and a verify
|
||||
phase (units from verify_units()). The verify phase is skipped if
|
||||
the run phase fails.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, name):
|
||||
self.test_name = name
|
||||
|
||||
@abstractmethod
|
||||
def ref_files(self):
|
||||
"""Get a list of reference files used by this test case"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def run_units(self):
|
||||
"""Units (typically RunGem5 instances) that describe the run phase of
|
||||
this test.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def verify_units(self):
|
||||
"""Verify the output from the run phase (see run_units())."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def update_ref(self):
|
||||
"""Update reference files with files from a test run"""
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
"""Run this test case and return a list of results"""
|
||||
|
||||
run_results = [ u.run() for u in self.run_units() ]
|
||||
run_ok = all([not r.skipped() and r for r in run_results ])
|
||||
|
||||
verify_results = [
|
||||
u.run() if run_ok else u.skip()
|
||||
for u in self.verify_units()
|
||||
]
|
||||
|
||||
return TestResult(self.test_name,
|
||||
run_results=run_results,
|
||||
verify_results=verify_results)
|
||||
|
||||
def __str__(self):
|
||||
return self.test_name
|
||||
|
||||
class ClassicTest(Test):
|
||||
# The diff ignore list contains all files that shouldn't be diffed
|
||||
# using DiffOutFile. These files typically use special-purpose
|
||||
# diff tools (e.g., DiffStatFile).
|
||||
diff_ignore_files = FileIgnoreList(
|
||||
names=(
|
||||
# Stat files use a special stat differ
|
||||
"stats.txt",
|
||||
), rex=(
|
||||
))
|
||||
|
||||
# These files should never be included in the list of
|
||||
# reference files. This list should include temporary files
|
||||
# and other files that we don't care about.
|
||||
ref_ignore_files = FileIgnoreList(
|
||||
names=(
|
||||
"EMPTY",
|
||||
), rex=(
|
||||
# Mercurial sometimes leaves backups when applying MQ patches
|
||||
r"\.orig$",
|
||||
r"\.rej$",
|
||||
))
|
||||
|
||||
def __init__(self, gem5, output_dir, config_tuple,
|
||||
timeout=None,
|
||||
skip=False, skip_diff_out=False, skip_diff_stat=False):
|
||||
|
||||
super(ClassicTest, self).__init__("/".join(config_tuple))
|
||||
|
||||
ct = config_tuple
|
||||
|
||||
self.gem5 = os.path.abspath(gem5)
|
||||
self.script = os.path.join(_test_base, "run.py")
|
||||
self.config_tuple = ct
|
||||
self.timeout = timeout
|
||||
|
||||
self.output_dir = output_dir
|
||||
self.ref_dir = os.path.join(_test_base,
|
||||
ct.category, ct.mode, ct.workload,
|
||||
"ref", ct.isa, ct.os, ct.config)
|
||||
self.skip_run = skip
|
||||
self.skip_diff_out = skip or skip_diff_out
|
||||
self.skip_diff_stat = skip or skip_diff_stat
|
||||
|
||||
def ref_files(self):
|
||||
ref_dir = os.path.abspath(self.ref_dir)
|
||||
for root, dirs, files in os.walk(ref_dir, topdown=False):
|
||||
for f in files:
|
||||
fpath = os.path.join(root[len(ref_dir) + 1:], f)
|
||||
if fpath not in ClassicTest.ref_ignore_files:
|
||||
yield fpath
|
||||
|
||||
def run_units(self):
|
||||
args = [
|
||||
self.script,
|
||||
"/".join(self.config_tuple),
|
||||
]
|
||||
|
||||
return [
|
||||
RunGem5(self.gem5, args,
|
||||
ref_dir=self.ref_dir, test_dir=self.output_dir,
|
||||
skip=self.skip_run),
|
||||
]
|
||||
|
||||
def verify_units(self):
|
||||
ref_files = set(self.ref_files())
|
||||
units = []
|
||||
if "stats.txt" in ref_files:
|
||||
units.append(
|
||||
DiffStatFile(ref_dir=self.ref_dir, test_dir=self.output_dir,
|
||||
skip=self.skip_diff_stat))
|
||||
units += [
|
||||
DiffOutFile(f,
|
||||
ref_dir=self.ref_dir, test_dir=self.output_dir,
|
||||
skip=self.skip_diff_out)
|
||||
for f in ref_files if f not in ClassicTest.diff_ignore_files
|
||||
]
|
||||
|
||||
return units
|
||||
|
||||
def update_ref(self):
|
||||
for fname in self.ref_files():
|
||||
shutil.copy(
|
||||
os.path.join(self.output_dir, fname),
|
||||
os.path.join(self.ref_dir, fname))
|
||||
|
||||
def parse_test_filter(test_filter):
|
||||
wildcards = ("", "*")
|
||||
|
||||
_filter = list(test_filter.split("/"))
|
||||
if len(_filter) > 3:
|
||||
raise RuntimeError("Illegal test filter string")
|
||||
_filter += [ "", ] * (3 - len(_filter))
|
||||
|
||||
isa, cat, mode = _filter
|
||||
|
||||
if isa in wildcards:
|
||||
raise RuntimeError("No ISA specified")
|
||||
|
||||
cat = all_categories if cat in wildcards else (cat, )
|
||||
mode = all_modes if mode in wildcards else (mode, )
|
||||
|
||||
return isa, cat, mode
|
||||
|
||||
def get_tests(isa,
|
||||
categories=all_categories, modes=all_modes,
|
||||
ruby_protocol=None, gpu_isa=None):
|
||||
|
||||
# Generate a list of candidate configs
|
||||
configs = list(arch_configs.get((isa, gpu_isa), []))
|
||||
|
||||
if (isa, gpu_isa) == ("x86", "hsail"):
|
||||
if ruby_protocol == "GPU_RfO":
|
||||
configs += ['gpu-randomtest']
|
||||
else:
|
||||
configs += generic_configs
|
||||
|
||||
if ruby_protocol == get_default_protocol(isa):
|
||||
if ruby_protocol == 'MI_example':
|
||||
configs += [ "%s-ruby" % (c, ) for c in configs ]
|
||||
else:
|
||||
configs += [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
|
||||
elif ruby_protocol is not None:
|
||||
# Override generic ISA configs when using Ruby (excluding
|
||||
# MI_example which is included in all ISAs by default). This
|
||||
# reduces the number of generic tests we re-run for when
|
||||
# compiling Ruby targets.
|
||||
configs = [ "%s-ruby-%s" % (c, ruby_protocol) for c in configs ]
|
||||
|
||||
# /(quick|long)/(fs|se)/workload/ref/arch/guest/config/
|
||||
for conf_script in configs:
|
||||
for cat in categories:
|
||||
for mode in modes:
|
||||
mode_dir = os.path.join(_test_base, cat, mode)
|
||||
if not os.path.exists(mode_dir):
|
||||
continue
|
||||
|
||||
for workload in os.listdir(mode_dir):
|
||||
isa_dir = os.path.join(mode_dir, workload, "ref", isa)
|
||||
if not os.path.isdir(isa_dir):
|
||||
continue
|
||||
|
||||
for _os in os.listdir(isa_dir):
|
||||
test_dir = os.path.join(isa_dir, _os, conf_script)
|
||||
if not os.path.exists(test_dir) or \
|
||||
os.path.exists(os.path.join(test_dir, "skip")):
|
||||
continue
|
||||
|
||||
yield ClassicConfig(cat, mode, workload, isa, _os,
|
||||
conf_script)
|
||||
@@ -1,294 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) 2016 ARM Limited
|
||||
# All rights reserved
|
||||
#
|
||||
# The license below extends only to copyright in the software and shall
|
||||
# not be construed as granting a license to any other intellectual
|
||||
# property including but not limited to intellectual property relating
|
||||
# to a hardware implementation of the functionality of the software
|
||||
# licensed hereunder. You may use the software subject to the license
|
||||
# terms below provided that you ensure that this notice is replicated
|
||||
# unmodified and in its entirety in all distributions of the software,
|
||||
# modified or unmodified, in source code or in binary form.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met: redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer;
|
||||
# redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution;
|
||||
# neither the name of the copyright holders nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from datetime import datetime
|
||||
import difflib
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
from six import add_metaclass
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
from results import UnitResult
|
||||
from helpers import *
|
||||
|
||||
_test_base = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class TestUnit(object):
|
||||
"""Base class for all test units.
|
||||
|
||||
A test unit is a part of a larger test case. Test cases usually
|
||||
contain two types of units, run units (run gem5) and verify units
|
||||
(diff output files). All unit implementations inherit from this
|
||||
class.
|
||||
|
||||
A unit implementation overrides the _run() method. The test runner
|
||||
calls the run() method, which wraps _run() to protect against
|
||||
exceptions.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, name, ref_dir, test_dir, skip=False):
|
||||
self.name = name
|
||||
self.ref_dir = ref_dir
|
||||
self.test_dir = test_dir
|
||||
self.force_skip = skip
|
||||
self.start_time = None
|
||||
self.stop_time = None
|
||||
|
||||
def result(self, state, **kwargs):
|
||||
if self.start_time is not None and "runtime" not in kwargs:
|
||||
self.stop_time = datetime.utcnow()
|
||||
delta = self.stop_time - self.start_time
|
||||
kwargs["runtime"] = delta.total_seconds()
|
||||
|
||||
return UnitResult(self.name, state, **kwargs)
|
||||
|
||||
def ok(self, **kwargs):
|
||||
return self.result(UnitResult.STATE_OK, **kwargs)
|
||||
|
||||
def skip(self, **kwargs):
|
||||
return self.result(UnitResult.STATE_SKIPPED, **kwargs)
|
||||
|
||||
def error(self, message, **kwargs):
|
||||
return self.result(UnitResult.STATE_ERROR, message=message, **kwargs)
|
||||
|
||||
def failure(self, message, **kwargs):
|
||||
return self.result(UnitResult.STATE_FAILURE, message=message, **kwargs)
|
||||
|
||||
def ref_file(self, fname):
|
||||
return os.path.join(self.ref_dir, fname)
|
||||
|
||||
def out_file(self, fname):
|
||||
return os.path.join(self.test_dir, fname)
|
||||
|
||||
def _read_output(self, fname, default=""):
|
||||
try:
|
||||
with open(self.out_file(fname), "r") as f:
|
||||
return f.read()
|
||||
except IOError:
|
||||
return default
|
||||
|
||||
def run(self):
|
||||
self.start_time = datetime.utcnow()
|
||||
try:
|
||||
if self.force_skip:
|
||||
return self.skip()
|
||||
else:
|
||||
return self._run()
|
||||
except:
|
||||
return self.error("Python exception:\n%s" % traceback.format_exc())
|
||||
|
||||
@abstractmethod
|
||||
def _run(self):
|
||||
pass
|
||||
|
||||
class RunGem5(TestUnit):
|
||||
"""Test unit representing a gem5 run.
|
||||
|
||||
Possible failure modes:
|
||||
- gem5 failed to run -> STATE_ERROR
|
||||
- timeout -> STATE_ERROR
|
||||
- non-zero exit code -> STATE_ERROR
|
||||
|
||||
Possible non-failure results:
|
||||
- exit code == 0 -> STATE_OK
|
||||
- exit code == 2 -> STATE_SKIPPED
|
||||
"""
|
||||
|
||||
def __init__(self, gem5, gem5_args, timeout=0, **kwargs):
|
||||
super(RunGem5, self).__init__("gem5", **kwargs)
|
||||
self.gem5 = gem5
|
||||
self.args = gem5_args
|
||||
self.timeout = timeout
|
||||
|
||||
def _run(self):
|
||||
gem5_cmd = [
|
||||
self.gem5,
|
||||
"-d", self.test_dir,
|
||||
"--stats-file", "text://stats.txt?desc=False",
|
||||
"-re",
|
||||
] + self.args
|
||||
|
||||
try:
|
||||
with ProcessHelper(gem5_cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE) as p:
|
||||
status, gem5_stdout, gem5_stderr = p.call(timeout=self.timeout)
|
||||
except CallTimeoutException as te:
|
||||
return self.error("Timeout", stdout=te.stdout, stderr=te.stderr)
|
||||
except OSError as ose:
|
||||
return self.error("Failed to launch gem5: %s" % ose)
|
||||
|
||||
stderr = "\n".join([
|
||||
"*** gem5 stderr ***",
|
||||
gem5_stderr,
|
||||
"",
|
||||
"*** m5out/simerr ***",
|
||||
self._read_output("simerr"),
|
||||
])
|
||||
|
||||
stdout = "\n".join([
|
||||
"*** gem5 stdout ***",
|
||||
gem5_stdout,
|
||||
"",
|
||||
"*** m5out/simout ***",
|
||||
self._read_output("simout"),
|
||||
])
|
||||
|
||||
# Signal
|
||||
if status < 0:
|
||||
return self.error("gem5 terminated by signal %i" % (-status, ),
|
||||
stdout=stdout, stderr=stderr)
|
||||
elif status == 2:
|
||||
return self.skip(stdout=stdout, stderr=stderr)
|
||||
elif status > 0:
|
||||
return self.error("gem5 exited with non-zero status: %i" % status,
|
||||
stdout=stdout, stderr=stderr)
|
||||
else:
|
||||
return self.ok(stdout=stdout, stderr=stderr)
|
||||
|
||||
class DiffOutFile(TestUnit):
|
||||
"""Test unit comparing and output file and a reference file."""
|
||||
|
||||
# regular expressions of lines to ignore when diffing outputs
|
||||
diff_ignore_regexes = {
|
||||
"simout" : [
|
||||
re.compile('^Redirecting (stdout|stderr) to'),
|
||||
re.compile('^gem5 compiled '),
|
||||
re.compile('^gem5 started '),
|
||||
re.compile('^gem5 executing on '),
|
||||
re.compile('^command line:'),
|
||||
re.compile("^Couldn't import dot_parser,"),
|
||||
re.compile("^info: kernel located at:"),
|
||||
re.compile("^Couldn't unlink "),
|
||||
re.compile("^Using GPU kernel code file\(s\) "),
|
||||
],
|
||||
"simerr" : [
|
||||
#re.compile('^Simulation complete at'),
|
||||
],
|
||||
"config.ini" : [
|
||||
re.compile("^(executable|readfile|kernel|image_file)="),
|
||||
re.compile("^(cwd|input|codefile)="),
|
||||
],
|
||||
"config.json" : [
|
||||
re.compile(r'''^\s*"(executable|readfile|kernel|image_file)":'''),
|
||||
re.compile(r'''^\s*"(cwd|input|codefile)":'''),
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self, fname, **kwargs):
|
||||
super(DiffOutFile, self).__init__("diff[%s]" % fname,
|
||||
**kwargs)
|
||||
|
||||
self.fname = fname
|
||||
self.line_filters = DiffOutFile.diff_ignore_regexes.get(fname, tuple())
|
||||
|
||||
def _filter_file(self, fname):
|
||||
def match_line(l):
|
||||
for r in self.line_filters:
|
||||
if r.match(l):
|
||||
return True
|
||||
return False
|
||||
|
||||
with open(fname, "r") as f:
|
||||
for l in f:
|
||||
if not match_line(l):
|
||||
yield l
|
||||
|
||||
|
||||
def _run(self):
|
||||
fname = self.fname
|
||||
ref = self.ref_file(fname)
|
||||
out = self.out_file(fname)
|
||||
|
||||
if not os.path.exists(ref):
|
||||
return self.error("%s doesn't exist in reference directory" \
|
||||
% fname)
|
||||
|
||||
if not os.path.exists(out):
|
||||
return self.error("%s doesn't exist in output directory" % fname)
|
||||
|
||||
diff = difflib.unified_diff(
|
||||
tuple(self._filter_file(ref)),
|
||||
tuple(self._filter_file(out)),
|
||||
fromfile="ref/%s" % fname, tofile="out/%s" % fname)
|
||||
|
||||
diff = list(diff)
|
||||
if diff:
|
||||
return self.error("ref/%s and out/%s differ" % (fname, fname),
|
||||
stderr="".join(diff))
|
||||
else:
|
||||
return self.ok(stdout="-- ref/%s and out/%s are identical --" \
|
||||
% (fname, fname))
|
||||
|
||||
class DiffStatFile(TestUnit):
|
||||
"""Test unit comparing two gem5 stat files."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super(DiffStatFile, self).__init__("stat_diff", **kwargs)
|
||||
|
||||
self.stat_diff = os.path.join(_test_base, "diff-out")
|
||||
|
||||
def _run(self):
|
||||
STATUS_OK = 0
|
||||
STATUS_NEW_STATS = 1
|
||||
STATUS_FAILED = 2
|
||||
|
||||
stats = "stats.txt"
|
||||
|
||||
cmd = [
|
||||
self.stat_diff,
|
||||
self.ref_file(stats), self.out_file(stats),
|
||||
]
|
||||
with ProcessHelper(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE) as p:
|
||||
status, stdout, stderr = p.call()
|
||||
|
||||
if status in (STATUS_OK, STATUS_NEW_STATS):
|
||||
return self.ok(stdout=stdout, stderr=stderr)
|
||||
elif status == STATUS_FAILED:
|
||||
return self.failure("Statistics mismatch",
|
||||
stdout=stdout, stderr=stderr)
|
||||
else:
|
||||
return self.error("diff-out returned an error: %i" % status,
|
||||
stdout=stdout, stderr=stderr)
|
||||
Reference in New Issue
Block a user