From 561f3bd75b7bb940515c0e7acd8d46f4d2f0a4d7 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Sun, 3 Sep 2023 23:33:59 -0700 Subject: [PATCH] misc,tests: Split testlib CI Tests to one dir-per-job This splits the CI Tests to one job per sub-directory in "tests/gem5" via a matrix. Advantages: * We can utilize more runners to run the quick tests. This should mean tests run quicker. * This approach does not require editing of the workflow as more tests are added or taken away. * There is now an output artifact for each directory in "tests/gem5" instead of one for the entriety of every quick test in "tests". In addition: * The artifact retention for the test outputs has been increased to 30 days. * The output test artifacts have been renamed to be more descriptive of the job, run, attempt, directory run, and the status. * The 'tar' step has been removed. GitHub's 'action/artifact' can handle directories. Change-Id: I5b3132b424e3769d81d9cd75db2a8c59dbe4a7e5 --- .github/workflows/ci-tests.yaml | 140 +++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 656f41f4bc..5a92422e79 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -46,27 +46,6 @@ jobs: "curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request." exit 1 - build-gem5: - runs-on: [self-hosted, linux, x64, build] - if: github.event.pull_request.draft == false - container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes - outputs: - artifactname: ${{ steps.name.outputs.test }} - steps: - - uses: actions/checkout@v3 - - id: name - run: echo "test=$(date +"%Y-%m-%d_%H.%M.%S")-artifact" >> $GITHUB_OUTPUT - - - name: Build gem5 - run: | - scons build/ALL/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.name.outputs.test }} - path: build/ALL/gem5.opt - - run: echo "This job's status is ${{ job.status }}." - unittests-all-opt: runs-on: [self-hosted, linux, x64, run] if: github.event.pull_request.draft == false @@ -80,39 +59,122 @@ jobs: run: scons build/ALL/unittests.opt -j $(nproc) - run: echo "This job's status is ${{ job.status }}." - testlib-quick: + testlib-quick-matrix: + runs-on: [self-hosted, linux, x64, run] + if: github.event.pull_request.draft == false + # In order to make sure the environment is exactly the same, we run in + # the same container we use to build gem5 and run the testlib tests. This + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id] + steps: + - uses: actions/checkout@v3 + + # Unfortunately the 'ubunutu-latest' image doesn't have jq installed. + # We therefore need to install it as a step here. + - name: Install jq + run: apt install -y jq + + - name: Get directories for testlib-quick + working-directory: "${{ github.workspace }}/tests" + id: dir-matrix + run: echo "test-dirs-matrix=$(find gem5/* -type d -maxdepth 0 | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT + + - name: Get the build targets for testlib-quick-gem5-builds + working-directory: "${{ github.workspace }}/tests" + id: build-matrix + run: echo "build-matrix=$(./main.py list --build-targets -q | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT + + outputs: + build-matrix: ${{ steps.build-matrix.outputs.build-matrix }} + test-dirs-matrix: ${{ steps.dir-matrix.outputs.test-dirs-matrix }} + + testlib-quick-gem5-builds: + runs-on: [self-hosted, linux, x64, build] + if: github.event.pull_request.draft == false + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id, testlib-quick-matrix] + strategy: + matrix: + build-target: ${{ fromJson(needs.testlib-quick-matrix.outputs.build-matrix) }} + steps: + - uses: actions/checkout@v3 + - name: Build gem5 + run: scons ${{ matrix.build-target }} -j $(nproc) + + # Upload the gem5 binary as an artifact. + # Note: the "achor.txt" file is a hack to make sure the paths are + # preserverd in the artifact. The upload-artifact action finds the + # closest common directory and uploads everything relative to that. + # E.g., if we upload "build/ARM/gem5.opt" and "build/RISCV/gem5.opt" + # Then upload-artifact will upload "ARM/gem5.opt" and "RISCV/gem5.opt", + # stripping the "build" directory. By adding the "anchor.txt" file, we + # ensure the "build" directory is preserved. + - run: echo "anchor" > anchor.txt + - uses: actions/upload-artifact@v3 + with: + name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds + path: | + build/*/gem5.* + anchor.txt + retention-days: 7 + + testlib-quick-execution: runs-on: [self-hosted, linux, x64, run] if: github.event.pull_request.draft == false container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, build-gem5, check-for-change-id] + needs: [pre-commit, check-for-change-id, testlib-quick-matrix, testlib-quick-gem5-builds] timeout-minutes: 360 # 6 hours + strategy: + fail-fast: false + matrix: + test-dir: ${{ fromJson(needs.testlib-quick-matrix.outputs.test-dirs-matrix) }} steps: - name: Clean runner run: rm -rf ./* || true rm -rf ./.??* || true rm -rf ~/.cache || true + + # Checkout the repository then download the gem5.opt artifact. - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.artifactname}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: The TestLib CI Tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run --skip-build -vv - - name: create zip of results - if: success() || failure() + name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds + + # Check that the gem5.opt artifact exists and is executable. + - name: Chmod gem5.{opt,debug,fast} to be executable run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip + find . -name "gem5.opt" -exec chmod u+x {} \; + find . -name "gem5.debug" -exec chmod u+x {} \; + find . -name "gem5.fast" -exec chmod u+x {} \; + + # Run the testlib quick tests in the given directory. + - name: Run "tests/${{ matrix.test-dir }}" TestLib quick tests + id: run-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run --skip-build -vv ${{ matrix.test-dir }} + + # Get the basename of the matrix.test-dir path (to name the artifact). + - name: Sanatize test-dir for artifact name + id: sanitize-test-dir + if: success() || failure() + run: echo "sanatized-test-dir=$(echo '${{ matrix.test-dir }}' | sed 's/\//-/g')" >> $GITHUB_OUTPUT + + # Upload the tests/testing-results directory as an artifact. + - name: Upload test results if: success() || failure() uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 + name: ci-tests-run-${{ github.run_number }}-attempt-${{ github.run_attempt }}-testlib-quick-${{ steps.sanitize-test-dir.outputs.sanatized-test-dir }}-status-${{ steps.run-tests.outcome }}-output + path: tests/testing-results + retention-days: 30 + + testlib-quick: + # It is 'testlib-quick' which needs to pass for the pull request to be + # merged. The 'testlib-quick-execution' is a matrix job which runs all the + # the testlib quick tests. This job is therefore a stub which will pass if + # all the testlib-quick-execution jobs pass. + runs-on: [self-hosted, linux, x64, run] + needs: testlib-quick-execution + steps: - run: echo "This job's status is ${{ job.status }}."