misc,ext,tests: Automatically split CI TestLib tests across GitHub Action jobs (#263)
This PR utilizes GitHub Action's matrix's to automatically distribute the CI testlib gem5 build and test jobs across available GitHub Action Runners. The CI tests (the `quick` testlib tests, i.e. those run with `./main.py run`) are distributed across the runners on a per directory basis --- all directories under "tests/gem5" are run as their own jobs. The necessary gem5 builds for each workflow are now automatically inferred via the introduction of `./main.py list`'s `--build-targets` flag which returns the gem5 build target for a given test or collection of tests. E.g., `./main.py list --build-targets` will return the build targets for all the `quick` testlib tests and `./main.py list --build-target --uid=<id>` will return the build targets the test suite `<id>` requires. Moving from monolithic jobs to fine-grained ones will make the locaiton of test failures more obvious. Each job has it's own artifact containing "test/testing-results" for the tests run in that job. In addition, maintenance of these files should become less burdensome due to less hardcoding.
This commit is contained in:
Bobby R. Bruce
140
.github/workflows/ci-tests.yaml
vendored
140
.github/workflows/ci-tests.yaml
vendored
@@ -46,27 +46,6 @@ jobs:
|
||||
"curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request."
|
||||
exit 1
|
||||
|
||||
build-gem5:
|
||||
runs-on: [self-hosted, linux, x64, build]
|
||||
if: github.event.pull_request.draft == false
|
||||
container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
|
||||
needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes
|
||||
outputs:
|
||||
artifactname: ${{ steps.name.outputs.test }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- id: name
|
||||
run: echo "test=$(date +"%Y-%m-%d_%H.%M.%S")-artifact" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build gem5
|
||||
run: |
|
||||
scons build/ALL/gem5.opt -j $(nproc)
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ${{ steps.name.outputs.test }}
|
||||
path: build/ALL/gem5.opt
|
||||
- run: echo "This job's status is ${{ job.status }}."
|
||||
|
||||
unittests-all-opt:
|
||||
runs-on: [self-hosted, linux, x64, run]
|
||||
if: github.event.pull_request.draft == false
|
||||
@@ -80,39 +59,122 @@ jobs:
|
||||
run: scons build/ALL/unittests.opt -j $(nproc)
|
||||
- run: echo "This job's status is ${{ job.status }}."
|
||||
|
||||
testlib-quick:
|
||||
testlib-quick-matrix:
|
||||
runs-on: [self-hosted, linux, x64, run]
|
||||
if: github.event.pull_request.draft == false
|
||||
# In order to make sure the environment is exactly the same, we run in
|
||||
# the same container we use to build gem5 and run the testlib tests. This
|
||||
container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
|
||||
needs: [pre-commit, check-for-change-id]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
# Unfortunately the 'ubunutu-latest' image doesn't have jq installed.
|
||||
# We therefore need to install it as a step here.
|
||||
- name: Install jq
|
||||
run: apt install -y jq
|
||||
|
||||
- name: Get directories for testlib-quick
|
||||
working-directory: "${{ github.workspace }}/tests"
|
||||
id: dir-matrix
|
||||
run: echo "test-dirs-matrix=$(find gem5/* -type d -maxdepth 0 | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT
|
||||
|
||||
- name: Get the build targets for testlib-quick-gem5-builds
|
||||
working-directory: "${{ github.workspace }}/tests"
|
||||
id: build-matrix
|
||||
run: echo "build-matrix=$(./main.py list --build-targets -q | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT
|
||||
|
||||
outputs:
|
||||
build-matrix: ${{ steps.build-matrix.outputs.build-matrix }}
|
||||
test-dirs-matrix: ${{ steps.dir-matrix.outputs.test-dirs-matrix }}
|
||||
|
||||
testlib-quick-gem5-builds:
|
||||
runs-on: [self-hosted, linux, x64, build]
|
||||
if: github.event.pull_request.draft == false
|
||||
container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
|
||||
needs: [pre-commit, check-for-change-id, testlib-quick-matrix]
|
||||
strategy:
|
||||
matrix:
|
||||
build-target: ${{ fromJson(needs.testlib-quick-matrix.outputs.build-matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Build gem5
|
||||
run: scons ${{ matrix.build-target }} -j $(nproc)
|
||||
|
||||
# Upload the gem5 binary as an artifact.
|
||||
# Note: the "achor.txt" file is a hack to make sure the paths are
|
||||
# preserverd in the artifact. The upload-artifact action finds the
|
||||
# closest common directory and uploads everything relative to that.
|
||||
# E.g., if we upload "build/ARM/gem5.opt" and "build/RISCV/gem5.opt"
|
||||
# Then upload-artifact will upload "ARM/gem5.opt" and "RISCV/gem5.opt",
|
||||
# stripping the "build" directory. By adding the "anchor.txt" file, we
|
||||
# ensure the "build" directory is preserved.
|
||||
- run: echo "anchor" > anchor.txt
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds
|
||||
path: |
|
||||
build/*/gem5.*
|
||||
anchor.txt
|
||||
retention-days: 7
|
||||
|
||||
testlib-quick-execution:
|
||||
runs-on: [self-hosted, linux, x64, run]
|
||||
if: github.event.pull_request.draft == false
|
||||
container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest
|
||||
needs: [pre-commit, build-gem5, check-for-change-id]
|
||||
needs: [pre-commit, check-for-change-id, testlib-quick-matrix, testlib-quick-gem5-builds]
|
||||
timeout-minutes: 360 # 6 hours
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-dir: ${{ fromJson(needs.testlib-quick-matrix.outputs.test-dirs-matrix) }}
|
||||
steps:
|
||||
- name: Clean runner
|
||||
run:
|
||||
rm -rf ./* || true
|
||||
rm -rf ./.??* || true
|
||||
rm -rf ~/.cache || true
|
||||
|
||||
# Checkout the repository then download the gem5.opt artifact.
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: ${{needs.build-gem5.outputs.artifactname}}
|
||||
path: build/ALL
|
||||
- run: chmod u+x build/ALL/gem5.opt
|
||||
- name: The TestLib CI Tests
|
||||
working-directory: ${{ github.workspace }}/tests
|
||||
run: ./main.py run --skip-build -vv
|
||||
- name: create zip of results
|
||||
if: success() || failure()
|
||||
name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds
|
||||
|
||||
# Check that the gem5.opt artifact exists and is executable.
|
||||
- name: Chmod gem5.{opt,debug,fast} to be executable
|
||||
run: |
|
||||
apt-get -y install zip
|
||||
zip -r output.zip tests/testing-results
|
||||
- name: upload zip
|
||||
find . -name "gem5.opt" -exec chmod u+x {} \;
|
||||
find . -name "gem5.debug" -exec chmod u+x {} \;
|
||||
find . -name "gem5.fast" -exec chmod u+x {} \;
|
||||
|
||||
# Run the testlib quick tests in the given directory.
|
||||
- name: Run "tests/${{ matrix.test-dir }}" TestLib quick tests
|
||||
id: run-tests
|
||||
working-directory: ${{ github.workspace }}/tests
|
||||
run: ./main.py run --skip-build -vv ${{ matrix.test-dir }}
|
||||
|
||||
# Get the basename of the matrix.test-dir path (to name the artifact).
|
||||
- name: Sanatize test-dir for artifact name
|
||||
id: sanitize-test-dir
|
||||
if: success() || failure()
|
||||
run: echo "sanatized-test-dir=$(echo '${{ matrix.test-dir }}' | sed 's/\//-/g')" >> $GITHUB_OUTPUT
|
||||
|
||||
# Upload the tests/testing-results directory as an artifact.
|
||||
- name: Upload test results
|
||||
if: success() || failure()
|
||||
uses: actions/upload-artifact@v3
|
||||
env:
|
||||
MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}}
|
||||
with:
|
||||
name: ${{ env.MY_STEP_VAR }}
|
||||
path: output.zip
|
||||
retention-days: 7
|
||||
name: ci-tests-run-${{ github.run_number }}-attempt-${{ github.run_attempt }}-testlib-quick-${{ steps.sanitize-test-dir.outputs.sanatized-test-dir }}-status-${{ steps.run-tests.outcome }}-output
|
||||
path: tests/testing-results
|
||||
retention-days: 30
|
||||
|
||||
testlib-quick:
|
||||
# It is 'testlib-quick' which needs to pass for the pull request to be
|
||||
# merged. The 'testlib-quick-execution' is a matrix job which runs all the
|
||||
# the testlib quick tests. This job is therefore a stub which will pass if
|
||||
# all the testlib-quick-execution jobs pass.
|
||||
runs-on: [self-hosted, linux, x64, run]
|
||||
needs: testlib-quick-execution
|
||||
steps:
|
||||
- run: echo "This job's status is ${{ job.status }}."
|
||||
|
||||
Reference in New Issue
Block a user