ext,tests: Make return code based on test results

This patch also fixes a spelling mistake.

Change-Id: I8635216e512c10913a9cda54541d7e31e0d22a40
Signed-off-by: Jason Lowe-Power <jason@lowepower.com>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/17450
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>

ext/testlib/handlers.py

@@ -166,6 +166,18 @@ class ResultHandler(log.Handler):
         self._closed = True
         self._save()
 
+    def unsuccessful(self):
+        '''
+        Performs an or reduce on all of the results.
+        Returns true if at least one test is unsuccessful, false when all tests
+        pass
+        '''
+        for suite_result in self.internal_results:
+            if suite_result.unsuccessful:
+                return True
+        # If all are successful, then this wasn't "unsuccessful"
+        return False
+
 
 #TODO Change from a handler to an internal post processor so it can be used
 # to reprint results

ext/testlib/main.py

@@ -78,6 +78,14 @@ class RunLogHandler():
     def close(self):
         self.mp_handler.close()
 
+    def unsuccessful(self):
+        '''
+        Performs an or reduce on all of the results.
+        Returns true if at least one test is unsuccessful, false when all tests
+        pass
+        '''
+        return self.result_handler.unsuccessful()
+
 def get_config_tags():
     return getattr(config.config,
             config.StorePositionalTagsAction.position_kword)
@@ -225,6 +233,8 @@ def do_list():
         qrunner.list_tests()
         qrunner.list_tags()
 
+    return 0
+
 def run_schedule(test_schedule, log_handler):
     '''
     Test Phases
@@ -273,8 +283,12 @@ def run_schedule(test_schedule, log_handler):
         library_runner = runner.LibraryRunner(test_schedule)
     library_runner.run()
 
+    failed = log_handler.unsuccessful()
+
     log_handler.finish_testing()
 
+    return 1 if failed else 0
+
 def do_run():
     # Initialize early parts of the log.
     with RunLogHandler() as log_handler:
@@ -297,8 +311,7 @@ def do_run():
             # Filter tests based on tags
             filter_with_config_tags(test_schedule)
         # Execute the tests
-        run_schedule(test_schedule, log_handler)
-
+        return run_schedule(test_schedule, log_handler)
 
 def do_rerun():
     # Init early parts of log
@@ -308,21 +321,25 @@ def do_rerun():
                 os.path.join(config.config.result_path,
                 config.constants.pickle_filename))
 
-        rerun_suites = (suite.uid for suite in results if suite.unsucessful)
+        rerun_suites = (suite.uid for suite in results if suite.unsuccessful)
 
         # Use loader to load suites
         loader = loader_mod.Loader()
         test_schedule = loader.load_schedule_for_suites(*rerun_suites)
 
         # Execute the tests
-        run_schedule(test_schedule, log_handler)
+        return run_schedule(test_schedule, log_handler)
 
 def main():
     '''
     Main entrypoint for the testlib test library.
+    Returns 0 on success and 1 otherwise so it can be used as a return code
+    for scripts.
     '''
     config.initialize_config()
 
     # 'do' the given command.
-    globals()['do_'+config.config.command]()
+    result = globals()['do_'+config.config.command]()
     log.test_log.close()
+
+    return result

ext/testlib/result.py

@@ -58,7 +58,7 @@ class _CommonMetadataMixin:
         self._metadata.result = result
 
     @property
-    def unsucessful(self):
+    def unsuccessful(self):
         return self._metadata.result.value != state.Result.Passed
 
 

tests/main.py

@@ -21,4 +21,4 @@ import testlib.config as config
 import testlib.helper as helper
 
 config.basedir = helper.absdirpath(__file__)
-testlib()
+sys.exit(testlib())