d1e2f18230
"failure" is a child of the testcase node: https://llg.cubic.org/docs/junit/ It allows xml parsers to understand which testcase failed the run. Otherwise CI frameworks like jenkins wouldn't be able to classify every single testcase. Prior to this patch testlib was using testsuites.failures and testsuite.failures only. These are simply reporting the number of failures. Change-Id: I0d498eca029c3232f2a588b153b6b6829b789394 Signed-off-by: Giacomo Travaglini <giacomo.travaglini@arm.com> Reviewed-by: Nikos Nikoleris <nikos.nikoleris@arm.com> Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/25083 Tested-by: kokoro <noreply+kokoro@google.com> Reviewed-by: Ciro Santilli <ciro.santilli@arm.com>
318 lines
9.7 KiB
Python
318 lines
9.7 KiB
Python
# Copyright (c) 2017 Mark D. Hill and David A. Wood
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met: redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer;
|
|
# redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution;
|
|
# neither the name of the copyright holders nor the names of its
|
|
# contributors may be used to endorse or promote products derived from
|
|
# this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
# Authors: Sean Wilson
|
|
|
|
import os
|
|
import pickle
|
|
import xml.sax.saxutils
|
|
|
|
from config import config
|
|
import helper
|
|
import state
|
|
import log
|
|
|
|
def _create_uid_index(iterable):
|
|
index = {}
|
|
for item in iterable:
|
|
assert item.uid not in index
|
|
index[item.uid] = item
|
|
return index
|
|
|
|
|
|
class _CommonMetadataMixin:
|
|
@property
|
|
def name(self):
|
|
return self._metadata.name
|
|
@property
|
|
def uid(self):
|
|
return self._metadata.uid
|
|
@property
|
|
def result(self):
|
|
return self._metadata.result
|
|
@result.setter
|
|
def result(self, result):
|
|
self._metadata.result = result
|
|
|
|
@property
|
|
def unsuccessful(self):
|
|
return self._metadata.result.value != state.Result.Passed
|
|
|
|
|
|
class InternalTestResult(object, _CommonMetadataMixin):
|
|
def __init__(self, obj, suite, directory):
|
|
self._metadata = obj.metadata
|
|
self.suite = suite
|
|
|
|
self.stderr = os.path.join(
|
|
InternalSavedResults.output_path(self.uid, suite.uid),
|
|
'stderr'
|
|
)
|
|
self.stdout = os.path.join(
|
|
InternalSavedResults.output_path(self.uid, suite.uid),
|
|
'stdout'
|
|
)
|
|
|
|
|
|
class InternalSuiteResult(object, _CommonMetadataMixin):
|
|
def __init__(self, obj, directory):
|
|
self._metadata = obj.metadata
|
|
self.directory = directory
|
|
self._wrap_tests(obj)
|
|
|
|
def _wrap_tests(self, obj):
|
|
self._tests = [InternalTestResult(test, self, self.directory)
|
|
for test in obj]
|
|
self._tests_index = _create_uid_index(self._tests)
|
|
|
|
def get_test(self, uid):
|
|
return self._tests_index[uid]
|
|
|
|
def __iter__(self):
|
|
return iter(self._tests)
|
|
|
|
def get_test_result(self, uid):
|
|
return self.get_test(uid)
|
|
|
|
def aggregate_test_results(self):
|
|
results = {}
|
|
for test in self:
|
|
helper.append_dictlist(results, test.result.value, test)
|
|
return results
|
|
|
|
|
|
class InternalLibraryResults(object, _CommonMetadataMixin):
|
|
def __init__(self, obj, directory):
|
|
self.directory = directory
|
|
self._metadata = obj.metadata
|
|
self._wrap_suites(obj)
|
|
|
|
def __iter__(self):
|
|
return iter(self._suites)
|
|
|
|
def _wrap_suites(self, obj):
|
|
self._suites = [InternalSuiteResult(suite, self.directory)
|
|
for suite in obj]
|
|
self._suites_index = _create_uid_index(self._suites)
|
|
|
|
def add_suite(self, suite):
|
|
if suite.uid in self._suites:
|
|
raise ValueError('Cannot have duplicate suite UIDs.')
|
|
self._suites[suite.uid] = suite
|
|
|
|
def get_suite_result(self, suite_uid):
|
|
return self._suites_index[suite_uid]
|
|
|
|
def get_test_result(self, test_uid, suite_uid):
|
|
return self.get_suite_result(suite_uid).get_test_result(test_uid)
|
|
|
|
def aggregate_test_results(self):
|
|
results = {}
|
|
for suite in self._suites:
|
|
for test in suite:
|
|
helper.append_dictlist(results, test.result.value, test)
|
|
return results
|
|
|
|
class InternalSavedResults:
|
|
@staticmethod
|
|
def output_path(test_uid, suite_uid, base=None):
|
|
'''
|
|
Return the path which results for a specific test case should be
|
|
stored.
|
|
'''
|
|
if base is None:
|
|
base = config.result_path
|
|
return os.path.join(
|
|
base,
|
|
str(suite_uid).replace(os.path.sep, '-'),
|
|
str(test_uid).replace(os.path.sep, '-'))
|
|
|
|
@staticmethod
|
|
def save(results, path, protocol=pickle.HIGHEST_PROTOCOL):
|
|
if not os.path.exists(os.path.dirname(path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(path))
|
|
except OSError as exc: # Guard against race condition
|
|
if exc.errno != errno.EEXIST:
|
|
raise
|
|
|
|
with open(path, 'w') as f:
|
|
pickle.dump(results, f, protocol)
|
|
|
|
@staticmethod
|
|
def load(path):
|
|
with open(path, 'r') as f:
|
|
return pickle.load(f)
|
|
|
|
|
|
class XMLElement(object):
|
|
def write(self, file_):
|
|
self.begin(file_)
|
|
self.end(file_)
|
|
|
|
def begin(self, file_):
|
|
file_.write('<')
|
|
file_.write(self.name)
|
|
for attr in self.attributes:
|
|
file_.write(' ')
|
|
attr.write(file_)
|
|
file_.write('>')
|
|
|
|
self.body(file_)
|
|
|
|
def body(self, file_):
|
|
for elem in self.elements:
|
|
file_.write('\n')
|
|
elem.write(file_)
|
|
file_.write('\n')
|
|
|
|
def end(self, file_):
|
|
file_.write('</%s>' % self.name)
|
|
|
|
class XMLAttribute(object):
|
|
def __init__(self, name, value):
|
|
self.name = name
|
|
self.value = value
|
|
|
|
def write(self, file_):
|
|
file_.write('%s=%s' % (self.name,
|
|
xml.sax.saxutils.quoteattr(self.value)))
|
|
|
|
|
|
class JUnitTestSuites(XMLElement):
|
|
name = 'testsuites'
|
|
result_map = {
|
|
state.Result.Errored: 'errors',
|
|
state.Result.Failed: 'failures',
|
|
state.Result.Passed: 'tests'
|
|
}
|
|
|
|
def __init__(self, internal_results):
|
|
results = internal_results.aggregate_test_results()
|
|
|
|
self.attributes = []
|
|
for result, tests in results.items():
|
|
self.attributes.append(self.result_attribute(result,
|
|
str(len(tests))))
|
|
|
|
self.elements = []
|
|
for suite in internal_results:
|
|
self.elements.append(JUnitTestSuite(suite))
|
|
|
|
def result_attribute(self, result, count):
|
|
return XMLAttribute(self.result_map[result], count)
|
|
|
|
class JUnitTestSuite(JUnitTestSuites):
|
|
name = 'testsuite'
|
|
result_map = {
|
|
state.Result.Errored: 'errors',
|
|
state.Result.Failed: 'failures',
|
|
state.Result.Passed: 'tests',
|
|
state.Result.Skipped: 'skipped'
|
|
}
|
|
|
|
def __init__(self, suite_result):
|
|
results = suite_result.aggregate_test_results()
|
|
|
|
self.attributes = [
|
|
XMLAttribute('name', suite_result.name)
|
|
]
|
|
for result, tests in results.items():
|
|
self.attributes.append(self.result_attribute(result,
|
|
str(len(tests))))
|
|
|
|
self.elements = []
|
|
for test in suite_result:
|
|
self.elements.append(JUnitTestCase(test))
|
|
|
|
def result_attribute(self, result, count):
|
|
return XMLAttribute(self.result_map[result], count)
|
|
|
|
class JUnitTestCase(XMLElement):
|
|
name = 'testcase'
|
|
def __init__(self, test_result):
|
|
self.attributes = [
|
|
XMLAttribute('name', test_result.name),
|
|
# TODO JUnit expects class of test.. add as test metadata.
|
|
XMLAttribute('classname', str(test_result.uid)),
|
|
XMLAttribute('status', str(test_result.result)),
|
|
]
|
|
|
|
# TODO JUnit expects a message for the reason a test was
|
|
# skipped or errored, save this with the test metadata.
|
|
# http://llg.cubic.org/docs/junit/
|
|
self.elements = [
|
|
LargeFileElement('system-err', test_result.stderr),
|
|
LargeFileElement('system-out', test_result.stdout),
|
|
]
|
|
|
|
if str(test_result.result) == 'Failed':
|
|
self.elements.append(JUnitFailure('Test failed', 'ERROR'))
|
|
|
|
|
|
class JUnitFailure(XMLElement):
|
|
name = 'failure'
|
|
def __init__(self, message, fail_type):
|
|
self.attributes = [
|
|
XMLAttribute('message', message),
|
|
XMLAttribute('type', fail_type),
|
|
]
|
|
self.elements = []
|
|
|
|
|
|
class LargeFileElement(XMLElement):
|
|
def __init__(self, name, filename):
|
|
self.name = name
|
|
self.filename = filename
|
|
self.attributes = []
|
|
|
|
def body(self, file_):
|
|
try:
|
|
with open(self.filename, 'r') as f:
|
|
for line in f:
|
|
file_.write(xml.sax.saxutils.escape(line))
|
|
except IOError:
|
|
# TODO Better error logic, this is sometimes O.K.
|
|
# if there was no stdout/stderr captured for the test
|
|
#
|
|
# TODO If that was the case, the file should still be made and it
|
|
# should just be empty instead of not existing.
|
|
pass
|
|
|
|
|
|
|
|
class JUnitSavedResults:
|
|
@staticmethod
|
|
def save(results, path):
|
|
'''
|
|
Compile the internal results into JUnit format writting it to the
|
|
given file.
|
|
'''
|
|
results = JUnitTestSuites(results)
|
|
with open(path, 'w') as f:
|
|
results.write(f)
|
|
|