misc,ext,tests: Automatically split CI TestLib tests across GitHub Action jobs (#263)

This PR utilizes GitHub Action's matrix's to automatically distribute
the CI testlib gem5 build and test jobs across available GitHub Action
Runners.

The CI tests (the `quick` testlib tests, i.e. those run with `./main.py
run`) are distributed across the runners on a per directory basis ---
all directories under "tests/gem5" are run as their own jobs.

The necessary gem5 builds for each workflow are now automatically
inferred via the introduction of `./main.py list`'s `--build-targets`
flag which returns the gem5 build target for a given test or collection
of tests. E.g., `./main.py list --build-targets` will return the build
targets for all the `quick` testlib tests and `./main.py list
--build-target --uid=<id>` will return the build targets the test suite
`<id>` requires.

Moving from monolithic jobs to fine-grained ones will make the locaiton
of test failures more obvious. Each job has it's own artifact containing
"test/testing-results" for the tests run in that job. In addition,
maintenance of these files should become less burdensome due to less
hardcoding.

.github/workflows/ci-tests.yaml

@@ -46,27 +46,6 @@ jobs:
             "curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request."
           exit 1
 
-  build-gem5:
-    runs-on: [self-hosted, linux, x64, build]
-    if: github.event.pull_request.draft == false
-    container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
-    needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes
-    outputs:
-      artifactname: ${{ steps.name.outputs.test }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: name
-        run: echo "test=$(date +"%Y-%m-%d_%H.%M.%S")-artifact" >> $GITHUB_OUTPUT
-
-      - name: Build gem5
-        run: |
-          scons build/ALL/gem5.opt -j $(nproc)
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ steps.name.outputs.test }}
-          path: build/ALL/gem5.opt
-      - run: echo "This job's status is ${{ job.status }}."
-
   unittests-all-opt:
     runs-on: [self-hosted, linux, x64, run]
     if: github.event.pull_request.draft == false
@@ -80,39 +59,122 @@ jobs:
         run: scons build/ALL/unittests.opt -j $(nproc)
       - run: echo "This job's status is ${{ job.status }}."
 
-  testlib-quick:
+  testlib-quick-matrix:
+    runs-on: [self-hosted, linux, x64, run]
+    if: github.event.pull_request.draft == false
+    # In order to make sure the environment is exactly the same, we run in
+    # the same container we use to build gem5 and run the testlib tests. This
+    container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
+    needs: [pre-commit, check-for-change-id]
+    steps:
+      - uses: actions/checkout@v3
+
+      # Unfortunately the 'ubunutu-latest' image doesn't have jq installed.
+      # We therefore need to install it as a step here.
+      - name: Install jq
+        run: apt install -y jq
+
+      - name: Get directories for testlib-quick
+        working-directory: "${{ github.workspace }}/tests"
+        id: dir-matrix
+        run: echo "test-dirs-matrix=$(find gem5/* -type d -maxdepth 0 | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT
+
+      - name: Get the build targets for testlib-quick-gem5-builds
+        working-directory: "${{ github.workspace }}/tests"
+        id: build-matrix
+        run: echo "build-matrix=$(./main.py list --build-targets -q | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT
+
+    outputs:
+        build-matrix: ${{ steps.build-matrix.outputs.build-matrix }}
+        test-dirs-matrix: ${{ steps.dir-matrix.outputs.test-dirs-matrix }}
+
+  testlib-quick-gem5-builds:
+    runs-on: [self-hosted, linux, x64, build]
+    if: github.event.pull_request.draft == false
+    container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
+    needs: [pre-commit, check-for-change-id, testlib-quick-matrix]
+    strategy:
+      matrix:
+        build-target: ${{ fromJson(needs.testlib-quick-matrix.outputs.build-matrix) }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build gem5
+        run: scons ${{ matrix.build-target }} -j $(nproc)
+
+        # Upload the gem5 binary as an artifact.
+        # Note: the "achor.txt" file is a hack to make sure the paths are
+        # preserverd in the artifact. The upload-artifact action finds the
+        # closest common directory and uploads everything relative to that.
+        # E.g., if we upload "build/ARM/gem5.opt" and "build/RISCV/gem5.opt"
+        # Then upload-artifact will upload "ARM/gem5.opt" and "RISCV/gem5.opt",
+        # stripping the "build" directory. By adding the "anchor.txt" file, we
+        # ensure the "build" directory is preserved.
+      - run: echo "anchor" > anchor.txt
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds
+          path: |
+            build/*/gem5.*
+            anchor.txt
+          retention-days: 7
+
+  testlib-quick-execution:
     runs-on: [self-hosted, linux, x64, run]
     if: github.event.pull_request.draft == false
     container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
-    needs: [pre-commit, build-gem5, check-for-change-id]
+    needs: [pre-commit, check-for-change-id, testlib-quick-matrix, testlib-quick-gem5-builds]
     timeout-minutes: 360     # 6 hours
+    strategy:
+      fail-fast: false
+      matrix:
+        test-dir: ${{ fromJson(needs.testlib-quick-matrix.outputs.test-dirs-matrix) }}
     steps:
       - name: Clean runner
         run:
           rm -rf ./* || true
           rm -rf ./.??* || true
           rm -rf ~/.cache || true
+
+        # Checkout the repository then download the gem5.opt artifact.
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
-          name: ${{needs.build-gem5.outputs.artifactname}}
-          path: build/ALL
-      - run: chmod u+x build/ALL/gem5.opt
-      - name: The TestLib CI Tests
-        working-directory: ${{ github.workspace }}/tests
-        run: ./main.py run --skip-build -vv
-      - name: create zip of results
-        if: success() || failure()
+          name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds
+
+        # Check that the gem5.opt artifact exists and is executable.
+      - name: Chmod gem5.{opt,debug,fast} to be executable
         run: |
-          apt-get -y install zip
-          zip -r output.zip tests/testing-results
-      - name: upload zip
+          find . -name "gem5.opt" -exec chmod u+x {} \;
+          find . -name "gem5.debug" -exec chmod u+x {} \;
+          find . -name "gem5.fast" -exec chmod u+x {} \;
+
+        # Run the testlib quick tests in the given directory.
+      - name: Run "tests/${{ matrix.test-dir }}" TestLib quick tests
+        id: run-tests
+        working-directory: ${{ github.workspace }}/tests
+        run: ./main.py run --skip-build -vv ${{ matrix.test-dir }}
+
+        # Get the basename of the matrix.test-dir path (to name the artifact).
+      - name: Sanatize test-dir for artifact name
+        id: sanitize-test-dir
+        if: success() || failure()
+        run: echo "sanatized-test-dir=$(echo '${{ matrix.test-dir }}' | sed 's/\//-/g')" >> $GITHUB_OUTPUT
+
+        # Upload the tests/testing-results directory as an artifact.
+      - name: Upload test results
         if: success() || failure()
         uses: actions/upload-artifact@v3
-        env:
-          MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}}
         with:
-          name: ${{ env.MY_STEP_VAR }}
-          path: output.zip
-          retention-days: 7
+          name: ci-tests-run-${{ github.run_number }}-attempt-${{ github.run_attempt }}-testlib-quick-${{ steps.sanitize-test-dir.outputs.sanatized-test-dir }}-status-${{ steps.run-tests.outcome }}-output
+          path: tests/testing-results
+          retention-days: 30
+
+  testlib-quick:
+    # It is 'testlib-quick' which needs to pass for the pull request to be
+    # merged. The 'testlib-quick-execution' is a matrix job which runs all the
+    # the testlib quick tests. This job is therefore a stub which will pass if
+    # all the testlib-quick-execution jobs pass.
+    runs-on: [self-hosted, linux, x64, run]
+    needs: testlib-quick-execution
+    steps:
       - run: echo "This job's status is ${{ job.status }}."

ext/testlib/configuration.py

@@ -744,6 +744,12 @@ class ListParser(ArgParser):
             default=False,
             help="List all tags.",
         ).add_to(parser)
+        Argument(
+            "--build-targets",
+            action="store_true",
+            default=False,
+            help="List all the gem5 build targets.",
+        ).add_to(parser)
         Argument(
             "-q",
             dest="quiet",
@@ -751,6 +757,12 @@ class ListParser(ArgParser):
             default=False,
             help="Quiet output (machine readable).",
         ).add_to(parser)
+        Argument(
+            "--uid",
+            action="store",
+            default=None,
+            help="UID of a specific test item to list.",
+        ).add_to(parser)
 
         common_args.directories.add_to(parser)
         common_args.bin_path.add_to(parser)

ext/testlib/fixture.py

@@ -27,6 +27,9 @@
 # Authors: Sean Wilson
 
 import testlib.helper as helper
+from testlib.configuration import constants
+
+from typing import Optional
 
 
 class SkipException(Exception):
@@ -79,6 +82,21 @@ class Fixture(object):
     def teardown(self, testitem):
         pass
 
+    def get_get_build_info(self) -> Optional[dict]:
+        # If this is a gem5 build it will return the target gem5 build path
+        # and any additional build information. E.g.:
+        #
+        # /path/to/gem5/build/NULL/gem5.opt--default=NULL PROTOCOL=MI_example
+        #
+        # In this example this may be passed to scons to build gem5 in
+        # accordance to the test's build requirements.
+        #
+        # If this fixtures is not a build of gem5, None is returned.
+        return None
+
+    def __str__(self):
+        return f"{self.name} fixture"
+
     def set_global(self):
         self._is_global = True
 

ext/testlib/main.py

@@ -242,7 +242,27 @@ def do_list():
 
     entry_message()
 
-    test_schedule = load_tests().schedule
+    if configuration.config.uid:
+        uid_ = uid.UID.from_uid(configuration.config.uid)
+        if isinstance(uid_, uid.TestUID):
+            log.test_log.error(
+                "Unable to list a standalone test.\n"
+                "Gem5 expects test suites to be the smallest unit "
+                " of test.\n\n"
+                "Pass a SuiteUID instead."
+            )
+            return
+        test_schedule = loader_mod.Loader().load_schedule_for_suites(uid_)
+        if get_config_tags():
+            log.test_log.warn(
+                "The '--uid' flag was supplied,"
+                " '--include-tags' and '--exclude-tags' will be ignored."
+            )
+    else:
+        test_schedule = load_tests().schedule
+        # Filter tests based on tags
+        filter_with_config_tags(test_schedule)
+
     filter_with_config_tags(test_schedule)
 
     qrunner = query.QueryRunner(test_schedule)
@@ -253,10 +273,15 @@ def do_list():
         qrunner.list_tests()
     elif configuration.config.all_tags:
         qrunner.list_tags()
+    elif configuration.config.fixtures:
+        qrunner.list_fixtures()
+    elif configuration.config.build_targets:
+        qrunner.list_build_targets()
     else:
         qrunner.list_suites()
         qrunner.list_tests()
         qrunner.list_tags()
+        qrunner.list_build_targets()
 
     return 0
 

ext/testlib/query.py

@@ -55,6 +55,25 @@ class QueryRunner(object):
             for test in suite:
                 log.test_log.message(test.uid, machine_readable=True)
 
+    def list_fixtures(self):
+        log.test_log.message(terminal.separator())
+        log.test_log.message("Listing all Test Fixtures.", bold=True)
+        log.test_log.message(terminal.separator())
+        for fixture in self.schedule.all_fixtures():
+            log.test_log.message(fixture, machine_readable=True)
+
+    def list_build_targets(self):
+        log.test_log.message(terminal.separator())
+        log.test_log.message("Listing all gem5 Build Targets.", bold=True)
+        log.test_log.message(terminal.separator())
+        builds = []
+        for fixture in self.schedule.all_fixtures():
+            build = fixture.get_get_build_info()
+            if build and build not in builds:
+                builds.append(build)
+        for build in builds:
+            log.test_log.message(build, machine_readable=True)
+
     def list_suites(self):
         log.test_log.message(terminal.separator())
         log.test_log.message("Listing all Test Suites.", bold=True)

src/python/m5/simulate.py

@@ -318,6 +318,10 @@ def checkpoint(dir):
 
     drain()
     memWriteback(root)
+
+    # Recursively create the checkpoint directory if it does not exist.
+    os.makedirs(dir, exist_ok=True)
+
     print("Writing checkpoint")
     _m5.core.serializeAll(dir)
 

tests/gem5/fixture.py

@@ -53,6 +53,8 @@ from testlib.helper import log_call, cacheresult, joinpath, absdirpath
 import testlib.log as log
 from testlib.state import Result
 
+from typing import Optional, List
+
 
 class VariableFixture(Fixture):
     def __init__(self, value=None, name=None):
@@ -74,7 +76,7 @@ class TempdirFixture(Fixture):
         suiteUID = testitem.metadata.uid.suite
         testUID = testitem.metadata.name
         testing_result_folder = os.path.join(
-            config.result_path, "SuiteUID:" + suiteUID, "TestUID:" + testUID
+            config.result_path, "SuiteUID-" + suiteUID, "TestUID-" + testUID
         )
 
         # Copy the output files of the run from /tmp to testing-results
@@ -147,6 +149,10 @@ class SConsFixture(UniqueFixture):
         obj = super(SConsFixture, cls).__new__(cls, target)
         return obj
 
+    def _setup(self, testitem):
+        if config.skip_build:
+            return
+
     def _setup(self, testitem):
         if config.skip_build:
             return
@@ -204,6 +210,12 @@ class Gem5Fixture(SConsFixture):
             self.options = ["--default=" + isa.upper(), "PROTOCOL=" + protocol]
         self.set_global()
 
+    def get_get_build_info(self) -> Optional[str]:
+        build_target = self.target
+        if self.options:
+            build_target += " ".join(self.options)
+        return build_target
+
 
 class MakeFixture(Fixture):
     def __init__(self, directory, *args, **kwargs):

tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py

@@ -29,6 +29,7 @@ This runs simple tests to ensure the examples in `configs/example/gem5_library`
 still function. They simply check the simulation completed.
 """
 from testlib import *
+from testlib.log import *
 import re
 import os
 
@@ -171,10 +172,11 @@ gem5_verify_config(
     length=constants.long_tag,
 )
 
-print(
-    "WARNING: PARSEC tests are disabled. This is due to our GitHub "
+log.test_log.message(
+    "PARSEC tests are disabled. This is due to our GitHub "
     "Actions self-hosted runners only having 60GB of disk space. The "
-    "PARSEC Disk image is too big to use."
+    "PARSEC Disk image is too big to use.",
+    level=LogLevel.Warn,
 )
 # 'False' is used to disable the tests.
 if False:  # os.access("/dev/kvm", mode=os.R_OK | os.W_OK):