From a74695f5bcc3dd6c424e4d708732cc686a98794d Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 30 May 2023 10:43:52 -0700 Subject: [PATCH 01/35] stdlib: Fix incorrect path and checks for DRAMsim3 There are three bugs fixed in this patch: 1. The `dram_3_dir` was missing the "dramsim3" directory. 2. Missing `not` when checking if configs is a directory. 3. Missing `not` when checking if input file is a file. Change-Id: I185f4832c1c2f1ecc4e138c148ad7969ef9b6fd4 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71058 Maintainer: Jason Lowe-Power Reviewed-by: Jason Lowe-Power Tested-by: kokoro --- src/python/gem5/components/memory/dramsim_3.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/gem5/components/memory/dramsim_3.py b/src/python/gem5/components/memory/dramsim_3.py index e5c1877fb5..f154ba354f 100644 --- a/src/python/gem5/components/memory/dramsim_3.py +++ b/src/python/gem5/components/memory/dramsim_3.py @@ -32,6 +32,7 @@ def config_ds3(mem_type: str, num_chnls: int) -> Tuple[str, str]: os.pardir, os.pardir, "ext", + "dramsim3", "DRAMsim3", ) @@ -47,11 +48,11 @@ def config_ds3(mem_type: str, num_chnls: int) -> Tuple[str, str]: "Please navigate to `ext` and run:\n" "git clone git@github.com:umd-memsys/DRAMsim3.git" ) - elif os.path.isdir(dramsim_3_mem_configs): + elif not os.path.isdir(dramsim_3_mem_configs): raise Exception( "The `ext/DRAMsim3/configs` directory cannot be found." ) - elif os.path.isfile(input_file): + elif not os.path.isfile(input_file): raise Exception( "The configuration file '" + input_file + "' cannot be found." ) From cf2d5b68a91aafb46efd7e59b789b35681dd6ff3 Mon Sep 17 00:00:00 2001 From: Matthew Poremba Date: Tue, 30 May 2023 13:57:09 -0500 Subject: [PATCH 02/35] gpu-compute: Gfx version check for FS and SE mode There is no GPU device in SE mode to get version from and no GPU driver in FS mode to get version from, so a conditional needs to be added depending on the mode to get the gfx version. Change-Id: I33fdafb60d351ebc5148e2248244537fb5bebd31 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71078 Tested-by: kokoro Maintainer: Matt Sinclair Reviewed-by: Matt Sinclair (cherry picked from commit ebd5b3e4ae49b24dec4efc299d0b0198be9f4df3) Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71158 Reviewed-by: Matthew Poremba Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- src/gpu-compute/gpu_command_processor.cc | 4 +++- src/gpu-compute/gpu_compute_driver.hh | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/gpu-compute/gpu_command_processor.cc b/src/gpu-compute/gpu_command_processor.cc index 9755180204..8f748bdc31 100644 --- a/src/gpu-compute/gpu_command_processor.cc +++ b/src/gpu-compute/gpu_command_processor.cc @@ -227,9 +227,11 @@ GPUCommandProcessor::submitDispatchPkt(void *raw_pkt, uint32_t queue_id, DPRINTF(GPUKernelInfo, "Kernel name: %s\n", kernel_name.c_str()); + GfxVersion gfxVersion = FullSystem ? gpuDevice->getGfxVersion() + : driver()->getGfxVersion(); HSAQueueEntry *task = new HSAQueueEntry(kernel_name, queue_id, dynamic_task_id, raw_pkt, &akc, host_pkt_addr, machine_code_addr, - gpuDevice->getGfxVersion()); + gfxVersion); DPRINTF(GPUCommandProc, "Task ID: %i Got AQL: wg size (%dx%dx%d), " "grid size (%dx%dx%d) kernarg addr: %#x, completion " diff --git a/src/gpu-compute/gpu_compute_driver.hh b/src/gpu-compute/gpu_compute_driver.hh index def40f4557..9a3c6479c3 100644 --- a/src/gpu-compute/gpu_compute_driver.hh +++ b/src/gpu-compute/gpu_compute_driver.hh @@ -142,6 +142,8 @@ class GPUComputeDriver final : public EmulatedDriver }; typedef class EventTableEntry ETEntry; + GfxVersion getGfxVersion() const { return gfxVersion; } + private: /** * GPU that is controlled by this driver. From ad0a2d1beaa043c03c0e43406078b3a09a3861ac Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Mon, 5 Jun 2023 16:40:16 -0700 Subject: [PATCH 03/35] misc: Adding GitHub workflow files This changes adds all of the workflows that will be run as we move to using GitHub. It includes our compiler tests, our nightly runs, and the continuous integration tests to be run on every pull request. Change-Id: I54c79aca7c0603337bd9fca443a0e2ffef603b0e Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71298 Reviewed-by: Bobby Bruce Tested-by: kokoro Maintainer: Bobby Bruce --- .github/workflows/ci-tests.yaml | 71 +++ .github/workflows/compiler-tests.yaml | 48 ++ .github/workflows/daily-tests.yaml | 886 ++++++++++++++++++++++++++ 3 files changed, 1005 insertions(+) create mode 100644 .github/workflows/ci-tests.yaml create mode 100644 .github/workflows/compiler-tests.yaml create mode 100644 .github/workflows/daily-tests.yaml diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml new file mode 100644 index 0000000000..309fefce2a --- /dev/null +++ b/.github/workflows/ci-tests.yaml @@ -0,0 +1,71 @@ +# This workflow runs after a pull-request has been approved by a reviewer. + +name: CI Tests +run-name: ${{ github.actor }} is running the pull-request continuous integration tests + +on: + pull_request: + types: [opened, reopened, edited] + +jobs: + build-gem5: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + artifactname: ${{ steps.name.outputs.test }} + steps: + - uses: actions/checkout@v3 + - id: name + run: echo "test=$(date +"%Y-%m-%d_%H.%M.%S")-artifact" >> $GITHUB_OUTPUT + + - name: Build gem5 + run: | + scons build/ALL/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.name.outputs.test }} + path: build/ALL/gem5.opt + - run: echo "This job's status is ${{ job.status }}." + + unittests-all-opt: + runs-on: [self-hosted, linux, x64, run] + container: + image: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - name: CI Unittests + working-directory: ${{ github.workspace }} + run: scons build/ALL/unittests.opt -j $(nproc) + - run: echo "This job's status is ${{ job.status }}." + + testlib-quick: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 360 # 6 hours + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.artifactname}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: The TestLib CI Tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run --skip-build -vv + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml new file mode 100644 index 0000000000..300f9eba36 --- /dev/null +++ b/.github/workflows/compiler-tests.yaml @@ -0,0 +1,48 @@ +# This workflow runs all of the compiler tests + +name: Running main.py +run-name: ${{ github.actor }} is running compiler tests + + +on: + # Runs every day from 7AM UTC + schedule: + - cron: '0 7 * * *' + +jobs: + # replication of compiler-tests.sh + all-compilers: + strategy: + matrix: + image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, gcc-version-7, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, clang-version-6.0, ubuntu-18.04_all-dependencies, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] + opts: [.opt, .fast] + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/${{ matrix.image }}:latest + timeout-minutes: 2880 # 48 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} + run: scons build/ALL/gem5${{ matrix.opts }} + + # Tests the two latest gcc and clang supported compilers against all gem5 compilations. + latest-compilers-all-gem5-builds: + strategy: + matrix: + gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base] + image: [gcc-version-12, clang-version-14] + opts: [.opt, .fast] + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/${{ matrix.image }}:latest + timeout-minutes: 2880 # 48 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} + run: scons build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml new file mode 100644 index 0000000000..91bb7d6f8a --- /dev/null +++ b/.github/workflows/daily-tests.yaml @@ -0,0 +1,886 @@ +# This workflow runs all of the long tests within main.py, extra tests in nightly.sh, and unittests + +name: Running main.py +run-name: ${{ github.actor }} is running the daily tests + + +on: + # Runs every day from 7AM UTC + schedule: + - cron: '0 7 * * *' + +jobs: +# building all necessary versions of gem5 + build-gem5: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-ALL_CHI: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_CHI" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL_CHI/gem5.opt --default=ALL PROTOCOL=CHI -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL_CHI/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-ARM: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ARM" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ARM/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ARM/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-ALL_MSI: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_MSI" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL_MSI/gem5.opt --default=ALL PROTOCOL=MSI -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL_MSI/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-ALL_MESI_Two_Level: + runs-on: [self-hosted, linux, x64, build,] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_MESI_Two_Level" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL_MESI_Two_Level/gem5.opt --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL_MESI_Two_Level/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-NULL: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-NULL" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/NULL/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/NULL/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-NULL_MI_example: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-NULL_MI_example" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/NULL_MI_example/gem5.opt --default=NULL PROTOCOL=MI_example -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/NULL_MI_example/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-RISCV: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-RISCV" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/RISCV/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/RISCV/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + build-gem5-VEGA_X86: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-VEGA_X86" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/VEGA_X86/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/VEGA_X86/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + + # This runs the unit tests for the build/ALL/unittests.debug build. + unittests-all-debug: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: ALL/unittests.debug UnitTests + run: | + ls + scons build/ALL/unittests.debug -j $(nproc) + + # This runs the unit tests for the build/ALL/unittests.fast build. + unittests-all-fast: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: ALL/unittests.fast UnitTests + run: | + ls + scons build/ALL/unittests.fast -j $(nproc) + +# start running all of the long tests + testlib-long-arm-boot-tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5, build-gem5-ALL_CHI] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ALL_CHI.outputs.build-name}} + path: build/ALL_CHI + - run: chmod u+x build/ALL_CHI/gem5.opt + - name: long arm-boot-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/arm-boot-tests --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-fs: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5, build-gem5-ARM] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ARM.outputs.build-name}} + path: build/ARM + - run: chmod u+x build/ARM/gem5.opt + - name: long fs + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/fs --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-gem5_library_example_tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5, build-gem5-ALL_MESI_Two_Level] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ALL_MESI_Two_Level.outputs.build-name}} + path: build/ALL_MESI_Two_Level + - run: chmod u+x build/ALL_MESI_Two_Level/gem5.opt + - name: long gem5_library_example_tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/gem5_library_example_tests --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-gpu: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5, build-gem5-VEGA_X86] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt + - name: long gpu + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/gpu --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-insttest_se: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: long insttest_se + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/insttest_se --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + +# kvm tests don't work on github actions + # testlib-long-kvm-fork-tests: + # runs-on: [self-hosted, linux, x64, run] + # container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + # needs: build-gem5 + # timeout-minutes: 1440 # 24 hours + # steps: + # - uses: actions/checkout@v3 + # with: + # # Scheduled workflows run on the default branch by default. We + # # therefore need to explicitly checkout the develop branch. + # ref: develop + # - uses: actions/download-artifact@v3 + # with: + # name: ${{ env.artifact-name }} + # path: build/ALL/gem5.opt + # - run: chmod u+x build/ALL/gem5.opt + # - name: long kvm-fork-tests + # working-directory: ${{ github.workspace }}/tests + # run: ./main.py run gem5/kvm-fork-tests --length=long --skip-build -vv -t $(nproc) + # - uses: actions/upload-artifact@v3 + # env: + # MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + # with: + # name: ${{ env.MY_STEP_VAR }} + # path: tests/testing-results + # retention-days: 7 + # - run: echo "This job's status is ${{ job.status }}." + + # testlib-long-kvm-switch-tests: + # runs-on: [self-hosted, linux, x64, run] + # container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + # needs: build-gem5 + # timeout-minutes: 1440 # 24 hours + # steps: + # - uses: actions/checkout@v3 + # with: + # # Scheduled workflows run on the default branch by default. We + # # therefore need to explicitly checkout the develop branch. + # ref: develop + # - uses: actions/download-artifact@v3 + # with: + # name: ${{ env.artifact-name }} + # path: build/ALL/gem5.opt + # - run: chmod u+x build/ALL/gem5.opt + # - name: long kvm-switch-tests + # working-directory: ${{ github.workspace }}/tests + # run: ./main.py run gem5/kvm-switch-tests --length=long --skip-build -vv -t $(nproc) + # - uses: actions/upload-artifact@v3 + # env: + # MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + # with: + # name: ${{ env.MY_STEP_VAR }} + # path: tests/testing-results + # retention-days: 7 + # - run: echo "This job's status is ${{ job.status }}." + + testlib-long-learning_gem5: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5-ALL_MSI + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ALL_MSI.outputs.build-name}} + path: build/ALL_MSI + - run: chmod u+x build/ALL_MSI/gem5.opt + - name: long learning_gem5 + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/learning_gem5 --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-m5_threads: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: long m5_threads + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/m5threads_test_atomic --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-memory: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5-NULL + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-NULL.outputs.build-name}} + path: build/NULL + - run: chmod u+x build/NULL/gem5.opt + - name: long memory + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/memory --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-multi_isa: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5-ARM, build-gem5-VEGA_X86, build-gem5-RISCV] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ARM.outputs.build-name}} + path: build/ARM + - run: chmod u+x build/ARM/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-RISCV.outputs.build-name}} + path: build/RISCV + - run: chmod u+x build/RISCV/gem5.opt + - name: long multi_isa + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/multi_isa --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-replacement-policies: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5-NULL_MI_example + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-NULL_MI_example.outputs.build-name}} + path: build/NULL_MI_example + - run: chmod u+x build/NULL_MI_example/gem5.opt + - name: long replacement-policies + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/replacement-policies --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-riscv-boot-tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: long riscv-boot-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/riscv-boot-tests --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-stdlib: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5-ARM, build-gem5-VEGA_X86, build-gem5-RISCV] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-ARM.outputs.build-name}} + path: build/ARM + - run: chmod u+x build/ARM/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5-RISCV.outputs.build-name}} + path: build/RISCV + - run: chmod u+x build/RISCV/gem5.opt + - name: long stdlib + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/stdlib --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-long-x86-boot-tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: long x86-boot-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/x86-boot-tests --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + # This runs the SST-gem5 integration compilation and tests it with + # ext/sst/sst/example.py. + sst-test: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/sst-env:latest + timeout-minutes: 180 + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Build RISCV/libgem5_opt.so with SST + run: scons build/RISCV/libgem5_opt.so --without-tcmalloc --duplicate-sources --ignore-style -j $(nproc) + - name: Compile ext/sst + working-directory: ${{ github.workspace }}/ext/sst + run: make -j $(nproc) + - name: Run SST test + working-directory: ${{ github.workspace }}/ext/sst + run: sst --add-lib-path=./ sst/example.py + + # This runs the gem5 within SystemC ingration and runs a simple hello-world + # simulation with it. + systemc-test: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/systemc-env:latest + timeout-minutes: 180 + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Build ARM/gem5.opt + uses: docker://gcr.io/gem5-test/ubuntu-22.04_min-dependencies:latest + # This is a bit of a hack so we can run a different docker container. + with: + args: scons --ignore-style --duplicate-sources -j4 build/ARM/gem5.opt + - name: Build ARM/libgem5_opt.so + uses: docker://gcr.io/gem5-test/ubuntu-22.04_min-dependencies:latest + # This is a bit of a hack so we can run a different docker container. + with: + args: scons --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j4 --duplicate-sources build/ARM/libgem5_opt.so + - name: Compile gem5 withing SystemC + working-directory: ${{ github.workspace }}/util/systemc/gem5_within_systemc + run: make + - name: Run gem5 within SystemC test + run: ./build/ARM/gem5.opt configs/deprecated/example/se.py -c tests/test-progs/hello/bin/arm/linux/hello + - name: bla + run: LD_LIBRARY_PATH=build/ARM/:/opt/systemc/lib-linux64/ ./gem5.opt.sc m5out/config.ini + + # Runs the gem5 Nighyly GPU tests. + gpu-tests: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/gcn-gpu:latest + timeout-minutes: 720 # 12 hours + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Compile build/GCN3_X86/gem5.opt + run: scons build/GCN3_X86/gem5.opt -j $(nproc) + - name: Get Square test-prog from gem5-resources + uses: wei/wget@v1 + with: + args: -q http://dist.gem5.org/dist/develop/test-progs/square/square # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time: https://github.com/coder/sshcode/issues/102 + - name: Run Square test with GCN3_X8/gem5.opt (SE mode) + run: | + mkdir -p tests/testing-results + ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c square + - name: Get allSyncPrims-1kernel from gem5-resources + uses: wei/wget@v1 + with: + args: -q http://dist.gem5.org/dist/develop/test-progs/heterosync/gcn3/allSyncPrims-1kernel # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time + - name: Run allSyncPrims-1kernel sleepMutex test with GCN3_X8/gem5.opt (SE mode) + run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="sleepMutex 10 16 4" + - name: Run allSysncPrims-1kernel lfTreeBarrUsing test with GCN3_X8/gem5.opt (SE mode) + run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="lfTreeBarrUsing 10 16 4" From 70ec55ce2a4ee221fbb15bc2fafd38cee6ff8c65 Mon Sep 17 00:00:00 2001 From: Kunal Pai <62979320+kunpai@users.noreply.github.com> Date: Sun, 21 May 2023 23:35:40 -0700 Subject: [PATCH 04/35] stdlib, tests, configs: Introduce gem5 Vision to resources This patch makes changes to the stdlib based on the gem5 Vision project. Firstly, a MongoDB database is supported. A JSON database's support is continued. The JSON can either be a local path or a raw GitHub link. The data for these databases is stored in src/python under "gem5-config.json". This will be used by default. However, the configuration can be overridden: - by providing a path using the GEM5_CONFIG env variable. - by placing a gem5-config.json file in the current working directory. An AbstractClient is an abstract class that implements searching and sorting relevant to the databases. Clients is an optional list that can be passed while defining any Resource class and obtain_resource. These databases can be defined in the config JSON. Resources now have versions. This allows for a single version, e.g., 'x86-ubuntu-boot', to have multiple versions. As such, the key of a resource is its ID and Version (e.g., 'x86-ubuntu-boot/v2.1.0'). Different versions of a resource might be compatible with different versions of gem5. By default, it picks the latest version compatible with the gem5 Version of the user. A gem5 resource schema now has additional fields. These are: - source_url: Stores URL of GitHub Source of the resource. - license: License information of the resource. - tags: Words to identify a resource better, like hello for hello-world - example_usage: How to use the resource in a simulation. - gem5_versions: List of gem5 versions that resource is compatible with. - resource_version: The version of the resource itself. - size: The download size of the resource, if it exists. - code_examples: List of objects. These objects contain the path to where a resource is used in gem5 example config scripts, and if the resource itself is used in tests or not. - category: Category of the resource, as defined by classes in src/python/gem5/resources/resource.py. Some fields have been renamed: - "name" is changed to "id" - "documentation" is changed to "description" Besides these, the schema also supports resource specialization. It adds fields relevant to a specific resource as specified in src/python/gem5/resources/resource.py These changes have been made to better present information on the new gem5 Resources website. But, they do not affect the way resources are used by a gem5 user. This patch is also backwards compatible. Existing code doesn't break with this new infrastructure. Also, refs in the tests have been changed to match this new schema. Tests have been changed to work with the two clients. Change-Id: Ia9bf47f7900763827fd5e873bcd663cc3ecdba40 Co-authored-by: Kunal Pai Co-authored-by: Parth Shah Co-authored-by: Harshil Patel Co-authored-by: aarsli Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71278 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- src/python/SConscript | 12 + src/python/gem5/resources/client.py | 84 +++ .../gem5/resources/client_api/__init__.py | 0 .../resources/client_api/abstract_client.py | 71 +++ .../gem5/resources/client_api/atlasclient.py | 91 ++++ .../resources/client_api/client_wrapper.py | 228 ++++++++ .../gem5/resources/client_api/jsonclient.py | 70 +++ src/python/gem5/resources/downloader.py | 23 +- src/python/gem5/resources/resource.py | 155 ++++-- src/python/gem5/resources/workload.py | 27 +- src/python/gem5/utils/simpoint.py | 12 +- src/python/gem5_default_config.py | 39 ++ tests/gem5/verifier.py | 1 + .../resources/pyunit_client_wrapper_checks.py | 511 ++++++++++++++++++ .../resources/pyunit_downloader_checks.py | 155 ++++-- .../pyunit_obtain_resources_check.py | 192 +++++++ .../pyunit_resource_download_checks.py | 72 +++ .../pyunit_resource_specialization.py | 68 ++- .../resources/pyunit_workload_checks.py | 42 +- .../stdlib/resources/refs/mongo_dup_mock.json | 30 + .../stdlib/resources/refs/mongo_mock.json | 56 ++ .../resources/refs/obtain-resource.json | 59 ++ .../refs/resource-specialization.json | 311 ++++++----- .../stdlib/resources/refs/resources.json | 329 +++++++++++ .../refs/workload-checks-custom-workload.json | 33 +- .../resources/refs/workload-checks.json | 84 +-- 26 files changed, 2413 insertions(+), 342 deletions(-) create mode 100644 src/python/gem5/resources/client.py create mode 100644 src/python/gem5/resources/client_api/__init__.py create mode 100644 src/python/gem5/resources/client_api/abstract_client.py create mode 100644 src/python/gem5/resources/client_api/atlasclient.py create mode 100644 src/python/gem5/resources/client_api/client_wrapper.py create mode 100644 src/python/gem5/resources/client_api/jsonclient.py create mode 100644 src/python/gem5_default_config.py create mode 100644 tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py create mode 100644 tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py create mode 100644 tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py create mode 100644 tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json create mode 100644 tests/pyunit/stdlib/resources/refs/mongo_mock.json create mode 100644 tests/pyunit/stdlib/resources/refs/obtain-resource.json create mode 100644 tests/pyunit/stdlib/resources/refs/resources.json diff --git a/src/python/SConscript b/src/python/SConscript index e261bfa7f2..f98b5700c7 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -264,12 +264,24 @@ PySource('gem5.prebuilt.riscvmatched', PySource('gem5.prebuilt.riscvmatched', 'gem5/prebuilt/riscvmatched/riscvmatched_core.py') PySource('gem5.resources', 'gem5/resources/__init__.py') +PySource('gem5.resources', 'gem5/resources/client.py') PySource('gem5.resources', 'gem5/resources/downloader.py') PySource('gem5.resources', 'gem5/resources/md5_utils.py') PySource('gem5.resources', 'gem5/resources/resource.py') PySource('gem5.resources', 'gem5/resources/workload.py') PySource('gem5.resources', 'gem5/resources/looppoint.py') PySource('gem5.resources', 'gem5/resources/elfie.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/__init__.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/jsonclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/atlasclient.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/client_wrapper.py') +PySource('gem5.resources.client_api', + 'gem5/resources/client_api/abstract_client.py') +PySource('gem5', 'gem5_default_config.py') PySource('gem5.utils', 'gem5/utils/__init__.py') PySource('gem5.utils', 'gem5/utils/filelock.py') PySource('gem5.utils', 'gem5/utils/override.py') diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py new file mode 100644 index 0000000000..bd473eb038 --- /dev/null +++ b/src/python/gem5/resources/client.py @@ -0,0 +1,84 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +import os +from typing import Optional, Dict, List +from .client_api.client_wrapper import ClientWrapper +from gem5.gem5_default_config import config +from m5.util import inform + + +def getFileContent(file_path: Path) -> Dict: + """ + Get the content of the file at the given path + :param file_path: The path of the file + :return: The content of the file + """ + if file_path.exists(): + with open(file_path, "r") as file: + return json.load(file) + else: + raise Exception(f"File not found at {file_path}") + + +clientwrapper = None + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, +) -> Dict: + """ + Get the resource json object from the clients wrapper + :param resource_id: The resource id + :param resource_version: The resource version + :param clients: The list of clients to query + """ + global clientwrapper + if clientwrapper is None: + # First check if the config file path is provided in the environment variable + if "GEM5_CONFIG" in os.environ: + config_file_path = Path(os.environ["GEM5_CONFIG"]) + gem5_config = getFileContent(config_file_path) + inform("Using config file specified by $GEM5_CONFIG") + inform(f"Using config file at {os.environ['GEM5_CONFIG']}") + # If not, check if the config file is present in the current directory + elif (Path().cwd().resolve() / "gem5-config.json").exists(): + config_file_path = Path().resolve() / "gem5-config.json" + gem5_config = getFileContent(config_file_path) + inform(f"Using config file at {config_file_path}") + # If not, use the default config in the build directory + else: + gem5_config = config + inform("Using default config") + clientwrapper = ClientWrapper(gem5_config) + + return clientwrapper.get_resource_json_obj_from_client( + resource_id, resource_version, clients + ) diff --git a/src/python/gem5/resources/client_api/__init__.py b/src/python/gem5/resources/client_api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py new file mode 100644 index 0000000000..74a513fc56 --- /dev/null +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from abc import ABC, abstractmethod +from typing import Any, Dict, List +import urllib.parse + + +class AbstractClient(ABC): + def verify_status_code(self, status_code: int) -> None: + """ + Verifies that the status code is 200. + :param status_code: The status code to verify. + """ + if status_code == 200: + return + if status_code == 429: + raise Exception("Panic: Too many requests") + if status_code == 401: + raise Exception("Panic: Unauthorized") + if status_code == 404: + raise Exception("Panic: Not found") + if status_code == 400: + raise Exception("Panic: Bad request") + if status_code == 500: + raise Exception("Panic: Internal server error") + + raise Exception(f"Panic: Unknown status code {status_code}") + + def _url_validator(self, url: str) -> bool: + """ + Validates the provided URL. + :param url: The URL to be validated. + :return: True if the URL is valid, False otherwise. + """ + try: + result = urllib.parse.urlparse(url) + return all([result.scheme, result.netloc, result.path]) + except: + return False + + @abstractmethod + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + raise NotImplementedError diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py new file mode 100644 index 0000000000..4a6e5cf691 --- /dev/null +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -0,0 +1,91 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from urllib import request, parse +from urllib.error import HTTPError, URLError +from typing import Optional, Dict, Union, Type, Tuple, List, Any +import json +from .abstract_client import AbstractClient + + +class AtlasClient(AbstractClient): + def __init__(self, config: Dict[str, str]): + """ + Initializes a connection to a MongoDB Atlas database. + :param uri: The URI for connecting to the MongoDB server. + :param db: The name of the database to connect to. + :param collection: The name of the collection within the database. + """ + self.apiKey = config["apiKey"] + self.url = config["url"] + self.collection = config["collection"] + self.database = config["database"] + self.dataSource = config["dataSource"] + self.authUrl = config["authUrl"] + + def get_token(self): + data = {"key": self.apiKey} + data = json.dumps(data).encode("utf-8") + + req = request.Request( + self.authUrl, + data=data, + headers={"Content-Type": "application/json"}, + ) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + token = result["access_token"] + return token + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + url = f"{self.url}/action/find" + data = { + "dataSource": self.dataSource, + "collection": self.collection, + "database": self.database, + "filter": {"id": resource_id}, + } + data = json.dumps(data).encode("utf-8") + + headers = { + "Authorization": f"Bearer {self.get_token()}", + "Content-Type": "application/json", + } + + req = request.Request(url, data=data, headers=headers) + try: + response = request.urlopen(req) + except HTTPError as e: + self.verify_status_code(e.status) + return None + result = json.loads(response.read().decode("utf-8")) + resources = result["documents"] + + return resources diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py new file mode 100644 index 0000000000..74ee831c1f --- /dev/null +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -0,0 +1,228 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from .jsonclient import JSONClient +from .atlasclient import AtlasClient +from _m5 import core +from typing import Optional, Dict, List +from distutils.version import StrictVersion +import itertools +from m5.util import warn + + +class ClientWrapper: + def __init__(self, config): + self.clients = self.create_clients(config) + + def create_clients( + self, + config: Dict, + ) -> Dict: + """ + This function creates respective client object for each source in the + config file according to the type of source. + Params: config: config file containing the source information + Returns: clients: dictionary of clients for each source + """ + clients = {} + for client in config["sources"]: + client_source = config["sources"][client] + try: + if client_source["isMongo"]: + clients[client] = AtlasClient(client_source) + else: + clients[client] = JSONClient(client_source["url"]) + except Exception as e: + warn(f"Error creating client {client}: {str(e)}") + return clients + + def get_all_resources_by_id( + self, + resource_id: str, + clients: Optional[List[str]] = None, + ) -> List[Dict]: + """ + This function returns all the resources with the given id from all the + sources. + :param resource_id: The id of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: A list of resources as Python dictionaries. + """ + resources = [] + if not clients: + clients = list(self.clients.keys()) + for client in clients: + if client not in self.clients: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + self.clients[client].get_resources_by_id(resource_id) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + # check if no 2 resources have the same id and version + for res1, res2 in itertools.combinations(resources, 2): + if res1["resource_version"] == res2["resource_version"]: + raise Exception( + f"Resource {resource_id} has multiple resources with " + f"the same version: {res1['resource_version']}" + ) + return resources + + def get_resource_json_obj_from_client( + self, + resource_id: str, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, + ) -> Dict: + """ + This function returns the resource object from the client with the + given id and version. + :param resource_id: The id of the resource to search for. + :param resource_version: The version of the resource to search for. + :param clients: A list of clients to search through. If None, all + clients are searched. + :return: The resource object as a Python dictionary if found. + If not found, exception is thrown. + """ + # getting all the resources with the given id from the dictionary + resources = self.get_all_resources_by_id(resource_id, clients) + # if no resource with the given id is found, return None + if len(resources) == 0: + raise Exception(f"Resource with ID '{resource_id}' not found.") + + resource_to_return = None + + if resource_version: + resource_to_return = self._search_version_in_resources( + resources, resource_id, resource_version + ) + + else: + compatible_resources = ( + self._get_resources_compatible_with_gem5_version(resources) + ) + if len(compatible_resources) == 0: + resource_to_return = self._sort_resources(resources)[0] + else: + resource_to_return = self._sort_resources( + compatible_resources + )[0] + + self._check_resource_version_compatibility(resource_to_return) + + return resource_to_return + + def _search_version_in_resources( + self, resources: List, resource_id: str, resource_version: str + ) -> Dict: + """ + Searches for the resource with the given version. If the resource is + not found, an exception is thrown. + :param resources: A list of resources to search through. + :param resource_version: The version of the resource to search for. + :return: The resource object as a Python dictionary if found. + If not found, None is returned. + """ + return_resource = next( + iter( + [ + resource + for resource in resources + if resource["resource_version"] == resource_version + ] + ), + None, + ) + if not return_resource: + raise Exception( + f"Resource {resource_id} with version '{resource_version}'" + " not found.\nResource versions can be found at: " + "https://resources.gem5.org/" + f"resources/{resource_id}/versions" + ) + return return_resource + + def _get_resources_compatible_with_gem5_version( + self, resources: List, gem5_version: str = core.gem5Version + ) -> List: + """ + Returns a list of compatible resources with the current gem5 version. + :param resources: A list of resources to filter. + :return: A list of compatible resources as Python dictionaries. + If no compatible resources are found, the original list of resources + is returned. + """ + compatible_resources = [ + resource + for resource in resources + if gem5_version in resource["gem5_versions"] + ] + return compatible_resources + + def _sort_resources(self, resources: List) -> List: + """ + Sorts the resources by ID. + If the IDs are the same, the resources are sorted by version. + :param resources: A list of resources to sort. + :return: A list of sorted resources. + """ + return sorted( + resources, + key=lambda resource: ( + resource["id"].lower(), + StrictVersion(resource["resource_version"]), + ), + reverse=True, + ) + + def _check_resource_version_compatibility( + self, resource: dict, gem5_version: Optional[str] = core.gem5Version + ) -> bool: + """ + Checks if the resource is compatible with the gem5 version. + Prints a warning if the resource is not compatible. + :param resource: The resource to check. + :optional param gem5_version: The gem5 version to check + compatibility with. + :return: True if the resource is compatible, False otherwise. + """ + if not resource: + return False + if gem5_version not in resource["gem5_versions"]: + warn( + f"Resource {resource['id']} with version " + f"{resource['resource_version']} is not known to be compatible" + f" with gem5 version {gem5_version}. " + "This may cause problems with your simulation. " + "This resource's compatibility " + "with different gem5 versions can be found here: " + "https://resources.gem5.org" + f"/resources/{resource['id']}/versions" + ) + return False + return True diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py new file mode 100644 index 0000000000..225126e2a8 --- /dev/null +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +from pathlib import Path +from urllib import request +from typing import Optional, Dict, Union, Type, Tuple, List, Any +from .abstract_client import AbstractClient +from urllib.error import URLError +from m5.util import warn + + +class JSONClient(AbstractClient): + def __init__(self, path: str): + """ + Initializes a JSON client. + :param path: The path to the Resource, either URL or local. + """ + self.path = path + self.resources = [] + + if Path(self.path).is_file(): + self.resources = json.load(open(self.path)) + elif not self._url_validator(self.path): + raise Exception( + f"Resources location '{self.path}' is not a valid path or URL." + ) + else: + req = request.Request(self.path) + try: + response = request.urlopen(req) + except URLError as e: + raise Exception( + f"Unable to open Resources location '{self.path}': {e}" + ) + self.resources = json.loads(response.read().decode("utf-8")) + + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. + :return: A list of all the Resources with the given ID. + """ + return [ + resource + for resource in self.resources + if resource["id"] == resource_id + ] diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 16b0147b74..0781d9b15a 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -41,6 +41,7 @@ from tempfile import gettempdir from urllib.error import HTTPError from typing import List, Dict, Set, Optional +from .client import get_resource_json_obj from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -398,6 +399,8 @@ def get_resource( unzip: bool = True, untar: bool = True, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -419,6 +422,13 @@ def get_resource( will delete this local resource and re-download it if this parameter is True. True by default. + :param resource_version: The version of the resource to be obtained. If + None, the latest version of the resource compatible with the working + directory's gem5 version will be obtained. None by default. + + :param clients: A list of clients to use when obtaining the resource. If + None, all clients will be used. None by default. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -430,11 +440,13 @@ def get_resource( # minutes.Most resources should be downloaded and decompressed in this # timeframe, even on the most constrained of systems. with FileLock(f"{to_path}.lock", timeout=900): - - resource_json = get_resources_json_obj(resource_name) + resource_json = get_resource_json_obj( + resource_name, + resource_version=resource_version, + clients=clients, + ) if os.path.exists(to_path): - if os.path.isfile(to_path): md5 = md5_file(Path(to_path)) else: @@ -495,9 +507,8 @@ def get_resource( ) ) - # Get the URL. The URL may contain '{url_base}' which needs replaced - # with the correct value. - url = resource_json["url"].format(url_base=_get_url_base()) + # Get the URL. + url = resource_json["url"] _download(url=url, download_to=download_dest) print(f"Finished downloading resource '{resource_name}'.") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9cba9cf88f..22adf15670 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -29,13 +29,15 @@ import os from pathlib import Path from m5.util import warn, fatal -from .downloader import get_resource, get_resources_json_obj +from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str from typing import Optional, Dict, Union, Type, Tuple, List +from .client import get_resource_json_obj + """ Resources are items needed to run a simulation, such as a disk image, kernel, or binary. The gem5 project provides pre-built resources, with sources, at @@ -67,18 +69,20 @@ class AbstractResource: def __init__( self, + resource_version: Optional[str] = None, local_path: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, ): """ :param local_path: The path on the host system where this resource is located. - :param documentation: Documentation describing this resource. Not a + :param description: Description describing this resource. Not a required parameter. By default is None. :param source: The source (as in "source code") for this resource. This string should navigate users to where the source for this resource may be found. Not a required parameter. By default is None. + :param resource_version: Version of the resource itself. """ if local_path and not os.path.exists(local_path): @@ -88,16 +92,21 @@ class AbstractResource: ) self._local_path = local_path - self._documentation = documentation + self._description = description self._source = source + self._version = resource_version + + def get_resource_version(self) -> str: + """Returns the version of the resource.""" + return self._version def get_local_path(self) -> Optional[str]: """Returns the local path of the resource.""" return self._local_path - def get_documentation(self) -> Optional[str]: - """Returns documentation associated with this resource.""" - return self._documentation + def get_description(self) -> Optional[str]: + """Returns description associated with this resource.""" + return self._description def get_source(self) -> Optional[str]: """Returns information as to where the source for this resource may be @@ -112,7 +121,8 @@ class FileResource(AbstractResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -123,8 +133,9 @@ class FileResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -134,11 +145,11 @@ class DirectoryResource(AbstractResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): - if not os.path.isdir(local_path): raise Exception( f"DirectoryResource path specified, {local_path}, is not a " @@ -147,8 +158,9 @@ class DirectoryResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -158,15 +170,17 @@ class DiskImageResource(FileResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, root_partition: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._root_partition = root_partition @@ -181,15 +195,17 @@ class BinaryResource(FileResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._architecture = None @@ -210,16 +226,18 @@ class BootloaderResource(BinaryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, architecture=architecture, source=source, + resource_version=resource_version, ) @@ -229,14 +247,16 @@ class GitResource(DirectoryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -246,16 +266,18 @@ class KernelResource(BinaryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, architecture: Optional[Union[ISA, str]] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, architecture=architecture, + resource_version=resource_version, ) @@ -270,14 +292,16 @@ class CheckpointResource(DirectoryResource): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) @@ -290,12 +314,13 @@ class SimpointResource(AbstractResource): def __init__( self, + resource_version: Optional[str] = None, simpoint_interval: int = None, simpoint_list: List[int] = None, weight_list: List[float] = None, warmup_interval: int = 0, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, **kwargs, @@ -314,8 +339,9 @@ class SimpointResource(AbstractResource): super().__init__( local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) self._weight_list = weight_list @@ -402,15 +428,17 @@ class LooppointCsvResource(FileResource, LooppointCsvLoader): def __init__( self, local_path: str, - documentation: Optional[str] = None, + resource_version: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointCsvLoader.__init__(self, pinpoints_file=Path(local_path)) @@ -419,16 +447,18 @@ class LooppointJsonResource(FileResource, LooppointJsonLoader): def __init__( self, local_path: str, + resource_version: Optional[str] = None, region_id: Optional[Union[str, int]] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): FileResource.__init__( self, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) LooppointJsonLoader.__init__( self, looppoint_file=local_path, region_id=region_id @@ -446,8 +476,9 @@ class SimpointDirectoryResource(SimpointResource): weight_file: str, simpoint_interval: int, warmup_interval: int, + resource_version: Optional[str] = None, workload_name: Optional[str] = None, - documentation: Optional[str] = None, + description: Optional[str] = None, source: Optional[str] = None, **kwargs, ): @@ -478,8 +509,9 @@ class SimpointDirectoryResource(SimpointResource): warmup_interval=warmup_interval, workload_name=workload_name, local_path=local_path, - documentation=documentation, + description=description, source=source, + resource_version=resource_version, ) def get_simpoint_file(self) -> Path: @@ -522,9 +554,11 @@ class SimpointDirectoryResource(SimpointResource): def obtain_resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return @@ -544,10 +578,16 @@ def obtain_resource( have the correct md5 value, the resoruce will be deleted and re-downloaded if this value is True. Otherwise an exception will be thrown. True by default. + :param resource_version: Version of the resource itself. + Not a required parameter. None by default. + :param clients: A list of clients to search for the resource. If this + parameter is not set, it will default search all clients. """ - # Obtain the JSON resource entry for this resource - resource_json = get_resources_json_obj(resource_name) + # Obtain the resource object entry for this resource + resource_json = get_resource_json_obj( + resource_id, resource_version=resource_version, clients=clients + ) to_path = None # If the "url" field is specified, the resoruce must be downloaded. @@ -580,38 +620,37 @@ def obtain_resource( os.makedirs(resource_directory, exist_ok=True) # This is the path to which the resource is to be stored. - to_path = os.path.join(resource_directory, resource_name) + to_path = os.path.join(resource_directory, resource_id) # Download the resource if it does not already exist. get_resource( - resource_name=resource_name, - to_path=os.path.join(resource_directory, resource_name), + resource_name=resource_id, + to_path=os.path.join(resource_directory, resource_id), download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. - resources_type = resource_json["type"] + resources_category = resource_json["category"] - if resources_type == "resource": + if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their # respective specializations ASAP and this case removed. - if ( - "additional_metadata" in resource_json - and "root_partition" in resource_json["additional_metadata"] - ): + if "root_partition" in resource_json: # In this case we should return a DiskImageResource. - root_partition = resource_json["additional_metadata"][ - "root_partition" - ] + root_partition = resource_json["root_partition"] return DiskImageResource( - local_path=to_path, root_partition=root_partition + local_path=to_path, + root_partition=root_partition, + **resource_json, ) return CustomResource(local_path=to_path) - assert resources_type in _get_resource_json_type_map - resource_class = _get_resource_json_type_map[resources_type] + assert resources_category in _get_resource_json_type_map + resource_class = _get_resource_json_type_map[resources_category] # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the @@ -694,6 +733,7 @@ class CustomDiskImageResource(DiskImageResource): def __init__( self, local_path: str, + resource_version: Optional[str] = None, root_partition: Optional[str] = None, metadata: Dict = {}, ): @@ -702,6 +742,7 @@ class CustomDiskImageResource(DiskImageResource): :param root_partition: The root disk partition to use. :param metadata: Metadata for the resource. **Warning:** As of " "v22.1.1, this parameter is not used. + :param resource_version: Version of the resource itself. """ warn( "The `CustomDiskImageResource` class is deprecated. Please use " @@ -713,13 +754,19 @@ class CustomDiskImageResource(DiskImageResource): "`CustomDiskImageResource` constructor. This parameter is not " "used." ) - super().__init__(local_path=local_path, root_partition=root_partition) + super().__init__( + local_path=local_path, + root_partition=root_partition, + resource_version=resource_version, + ) def Resource( - resource_name: str, + resource_id: str, resource_directory: Optional[str] = None, download_md5_mismatch: bool = True, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, ) -> AbstractResource: """ This function was created to maintain backwards compability for v21.1.0 @@ -737,9 +784,11 @@ def Resource( ) return obtain_resource( - resource_name=resource_name, + resource_id=resource_id, resource_directory=resource_directory, download_md5_mismatch=download_md5_mismatch, + resource_version=resource_version, + clients=clients, ) diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index e0a19d0792..148ab3f35a 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -24,10 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .downloader import get_workload_json_obj from .resource import obtain_resource +from .client import get_resource_json_obj -from typing import Dict, Any, Optional +from typing import Dict, Any, List, Optional class AbstractWorkload: @@ -155,7 +155,11 @@ class Workload(AbstractWorkload): """ def __init__( - self, workload_name: str, resource_directory: Optional[str] = None + self, + workload_name: str, + resource_directory: Optional[str] = None, + resource_version: Optional[str] = None, + clients: Optional[List] = None, ) -> None: """ This constructor will load the workload details from the workload with @@ -167,13 +171,13 @@ class Workload(AbstractWorkload): ```json { - "type" : "workload", - "name" : "x86-ubuntu-18.04-echo-hello", - "documentation" : "Description of workload here", + "category" : "workload", + "id" : "x86-ubuntu-18.04-echo-hello", + "description" : "Description of workload here", "function" : "set_kernel_disk_workload", "resources" : { "kernel" : "x86-linux-kernel-5.4.49", - "disk_image" : "x86-ubuntu-18.04-img" + "disk-image" : "x86-ubuntu-18.04-img" }, "additional_params" : { "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" @@ -187,7 +191,7 @@ class Workload(AbstractWorkload): ```python board.set_kernel_disk_workload( kernel = Resource("x86-linux-kernel-5.4.49"), - disk_image = Resource("x86-ubuntu-18.04-img"), + disk-image = Resource("x86-ubuntu-18.04-img"), readfile_contents = "m5_exit; echo 'hello'; m5_exit", ) ``` @@ -198,7 +202,12 @@ class Workload(AbstractWorkload): any resources should be download and accessed from. If None, a default location will be used. None by default. """ - workload_json = get_workload_json_obj(workload_name=workload_name) + + workload_json = get_resource_json_obj( + workload_name, + resource_version=resource_version, + clients=clients, + ) func = workload_json["function"] assert isinstance(func, str) diff --git a/src/python/gem5/utils/simpoint.py b/src/python/gem5/utils/simpoint.py index eab92e2291..0d1af4b1cf 100644 --- a/src/python/gem5/utils/simpoint.py +++ b/src/python/gem5/utils/simpoint.py @@ -83,15 +83,11 @@ class SimPoint: simpoint_file_path = simpoint_directory.get_simpoint_file() weight_file_path = simpoint_resource.get_weight_file() - simpoint_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("simpoint_interval") + simpoint_interval = simpoint_resource.get_metadata().get( + "simpoint_interval" ) - warmup_interval = ( - simpoint_resource.get_metadata() - .get("additional_metadata") - .get("warmup_interval") + warmup_interval = simpoint_resource.get_metadata().get( + "warmup_interval" ) self._simpoint_interval = simpoint_interval diff --git a/src/python/gem5_default_config.py b/src/python/gem5_default_config.py new file mode 100644 index 0000000000..980c883109 --- /dev/null +++ b/src/python/gem5_default_config.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +config = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + } +} diff --git a/tests/gem5/verifier.py b/tests/gem5/verifier.py index 93d47c8d10..c725fc68b9 100644 --- a/tests/gem5/verifier.py +++ b/tests/gem5/verifier.py @@ -172,6 +172,7 @@ class MatchStdout(DerivedGoldStandard): re.compile("^Using GPU kernel code file\(s\) "), re.compile("^.* not found locally\. Downloading"), re.compile("^Finished downloading"), + re.compile("^info: Using default config"), ] diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py new file mode 100644 index 0000000000..96aadf6879 --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -0,0 +1,511 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from gem5.isas import ISA +from gem5.resources.client import get_resource_json_obj +import gem5.resources.client +from gem5.resources.client_api.client_wrapper import ClientWrapper +from typing import Dict +from unittest.mock import patch +from unittest import mock +import json +from urllib.error import HTTPError +import io +import contextlib +from pathlib import Path + +mock_json_path = Path(__file__).parent / "refs/resources.json" +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + +mock_config_mongo = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, +} + +mock_config_combined = { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + }, + "baba": { + "url": mock_json_path, + "isMongo": False, + }, + }, +} + +mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_mock.json", "r") as f: + mock_json = json.load(f) + +duplicate_mock_json = {} + +with open(Path(__file__).parent / "refs/mongo_dup_mock.json", "r") as f: + duplicate_mock_json = json.load(f) + + +def mocked_requests_post(*args): + # mokcing urllib.request.urlopen + class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status = status_code + + def read(self): + return json.dumps(self.json_data).encode("utf-8") + + data = json.loads(args[0].data) + if "/api-key/login" in args[0].full_url: + return MockResponse({"access_token": "test-token"}, 200) + if "/endpoint/data/v1/action/find" in args[0].full_url: + if data: + if data["filter"]["id"] == "x86-ubuntu-18.04-img": + return MockResponse( + { + "documents": mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-duplicate": + return MockResponse( + { + "documents": duplicate_mock_json, + }, + 200, + ) + if data["filter"]["id"] == "test-too-many": + error_file = io.BytesIO() + error_file.status = 429 + raise HTTPError( + args[0].full_url, 429, "Too Many Requests", {}, error_file + ) + return MockResponse( + { + "documents": [], + }, + 200, + ) + error_file = io.BytesIO() + error_file.status = 404 + raise HTTPError(args[0].full_url, 404, "Not Found", {}, error_file) + + +class ClientWrapperTestSuite(unittest.TestCase): + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj(self): + # Test that the resource object is correctly returned + resource = "this-is-a-test-resource" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual( + resource["description"], "This is a test resource but double newer" + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_invalid_client(self): + # Test that an exception is raised when an invalid client is passed + resource_id = "test-id" + client = "invalid" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=[client]) + self.assertTrue( + f"Client: {client} does not exist" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version(self): + # Test that the resource object is correctly returned + resource_id = "this-is-a-test-resource" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, resource_version=resource_version + ) + self.assertEqual(resource["id"], "this-is-a-test-resource") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "binary") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_1(self, mock_get): + resource = "x86-ubuntu-18.04-img" + resource = get_resource_json_obj(resource) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.1.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual( + resource["description"], + "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + ) + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_mongodb(self, mock_get): + # Test that the resource object is correctly returned + resource_id = "x86-ubuntu-18.04-img" + resource_version = "1.0.0" + resource = get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource["resource_version"], "1.0.0") + self.assertEqual(resource["category"], "disk-image") + self.assertEqual(resource["description"], "This is a test resource") + self.assertEqual( + resource["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): + resource_id = "invalid-id" + with self.assertRaises(Exception) as context: + get_resource_json_obj(resource_id, clients=["gem5-resources"]) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_mongo), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_with_version_invalid_mongodb( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + clients=["gem5-resources"], + ) + self.assertTrue( + f"Resource x86-ubuntu-18.04-img with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/x86-ubuntu-18.04-img/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), + ) + def test_get_resource_json_obj_with_version_invalid_json(self): + resource_id = "this-is-a-test-resource" + resource_version = "2.5.0" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + resource_version=resource_version, + ) + self.assertTrue( + f"Resource this-is-a-test-resource with version '2.5.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/this-is-a-test-resource/versions" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_combine(self, mock_get): + resource_id_mongo = "x86-ubuntu-18.04-img" + resource_version_mongo = "1.0.0" + resource_id_json = "this-is-a-test-resource" + resource_version_json = "1.0.0" + resource_mongo = get_resource_json_obj( + resource_id_mongo, + resource_version=resource_version_mongo, + clients=["gem5-resources"], + ) + resource_json = get_resource_json_obj( + resource_id_json, + resource_version=resource_version_json, + clients=["baba"], + ) + self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_mongo["resource_version"], "1.0.0") + self.assertEqual(resource_mongo["category"], "disk-image") + self.assertEqual( + resource_mongo["description"], "This is a test resource" + ) + self.assertEqual( + resource_mongo["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + ) + self.assertEqual(resource_mongo["architecture"], "X86") + + self.assertEqual(resource_json["id"], "this-is-a-test-resource") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "binary") + self.assertEqual( + resource_json["description"], "This is a test resource" + ) + self.assertEqual( + resource_json["source_url"], + "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + ) + self.assertEqual(resource_json["architecture"], "X86") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_obj_multi_database_second_only(self, mock_get): + resource_id = "simpoint-resource" + resource = get_resource_json_obj( + resource_id, + ) + self.assertEqual(resource["id"], resource_id) + self.assertEqual(resource["resource_version"], "0.2.0") + self.assertEqual(resource["category"], "file") + self.assertEqual( + resource["description"], + ( + "Simpoints for running the 'x86-print-this' resource with" + ' the parameters `"print this" 15000`. This is encapsulated' + " in the 'x86-print-this-15000-with-simpoints' workload." + ), + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_json_same_resource_different_versions( + self, mock_get + ): + resource_id = "x86-ubuntu-18.04-img" + resource_json = get_resource_json_obj( + resource_id, + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "2.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + resource_json = get_resource_json_obj( + resource_id, + resource_version="1.0.0", + ) + + self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") + self.assertEqual(resource_json["resource_version"], "1.0.0") + self.assertEqual(resource_json["category"], "disk-image") + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_combined), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_get_resource_same_resource_same_version(self, mock_get): + resource_id = "test-duplicate" + with self.assertRaises(Exception) as context: + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + f"Resource {resource_id} has multiple resources with" + f" the same version: 0.2.0" in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/logi", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_auth_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-resource" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Not found" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-resource' not found." + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper( + { + "sources": { + "gem5-resources": { + "dataSource": "gem5-vision", + "database": "gem5-vision", + "collection": "versions_test", + "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", + "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", + "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", + "isMongo": True, + } + }, + } + ), + ) + @patch("urllib.request.urlopen", side_effect=mocked_requests_post) + def test_invalid_url(self, mock_get): + resource_id = "test-too-many" + f = io.StringIO() + with self.assertRaises(Exception) as context: + with contextlib.redirect_stderr(f): + get_resource_json_obj( + resource_id, + ) + self.assertTrue( + "Error getting resources from client gem5-resources:" + " Panic: Too many requests" in str(f.getvalue()) + ) + self.assertTrue( + "Resource with ID 'test-too-many' not found." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py index 08736bbc32..19169e480e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,7 @@ import unittest import tempfile import os from typing import Dict +import json from gem5.resources.downloader import ( _get_resources_json_at_path, @@ -42,48 +43,102 @@ class ResourceDownloaderTestSuite(unittest.TestCase): @classmethod def setUpClass(cls) -> str: """ - This creates a simple resource.json temp file for testing purposes. + This creates a simple resources collection for testing """ - - file_contents = ( - "{" - + f'"version" : "{_resources_json_version_required()}",' - + """ - "url_base" : "http://dist.gem5.org/dist/v21-2", - "previous-versions" : {}, - "resources": [ - { - "type": "resource", - "name" : "riscv-disk-img", - "documentation" : "A simple RISCV disk image based on busybox.", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "d6126db9f6bed7774518ae25aa35f153", - "url": "{url_base}/images/riscv/busybox/riscv-disk.img.gz", - "source" : "src/riscv-fs", - "additional_metadata" : { - "root_partition": null - } - }, - { - "type": "resource", - "name" : "riscv-lupio-busybox-img", - "documentation" : "A RISCV disk image, based on busybox, to ...", - "architecture": "RISCV", - "is_zipped" : true, - "md5sum" : "e5bee8a31f45f4803f87c0d781553ccc", - "url": "{url_base}/images/riscv/busybox/riscv-lupio-busybox.img", - "source" : "src/lupv", - "additional_metadata" : { - "root_partition": "1" - } - } - ] -} - """ - ) + file_contents = [ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest", "testing", "riscv", "testing"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": ["asmtest"], + "is_zipped": False, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": ["23.1"], + "example_usage": 'get_resource(resource_name="rv64mi-p-sbreak")', + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": False, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": True, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": ["23.0"], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": 'get_resource(resource_name="x86-print-this-1500-simpoints")', + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint", + ], + }, + ] file = tempfile.NamedTemporaryFile(mode="w", delete=False) - file.write(file_contents) + file.write(json.dumps(file_contents)) file.close() cls.file_path = file.name @@ -100,14 +155,16 @@ class ResourceDownloaderTestSuite(unittest.TestCase): "create_temp_resources_json" has been loaded correctly into a Python dictionary. """ - self.assertTrue("resources" in json) - self.assertEquals(2, len(json["resources"])) - self.assertTrue("name" in json["resources"][0]) - self.assertEquals("riscv-disk-img", json["resources"][0]["name"]) - self.assertTrue("name" in json["resources"][1]) - self.assertEquals( - "riscv-lupio-busybox-img", json["resources"][1]["name"] - ) + self.assertEquals(4, len(json)) + self.assertTrue("id" in json[0]) + self.assertEquals("this-is-a-test-resource", json[0]["id"]) + self.assertEquals("binary", json[0]["category"]) + self.assertTrue("id" in json[1]) + self.assertEquals("this-is-a-test-resource", json[1]["id"]) + self.assertTrue("id" in json[2]) + self.assertEquals("test-version", json[2]["id"]) + self.assertTrue("id" in json[3]) + self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: # Tests the gem5.resources.downloader._get_resources_json_at_path() diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py new file mode 100644 index 0000000000..791d96c1f1 --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -0,0 +1,192 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import io +import contextlib +from pathlib import Path + +from gem5.resources.resource import * + +from gem5.resources.looppoint import ( + LooppointCsvLoader, + LooppointJsonLoader, +) + +from gem5.isas import ISA + +from _m5 import core + +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch + +mock_json_path = Path(__file__).parent / "refs/obtain-resource.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), +) +class TestObtainResourcesCheck(unittest.TestCase): + @classmethod + def setUpClass(cls): + """Prior to running the suite we set the resource directory to + "ref/resource-specialization.json" + """ + os.environ["GEM5_RESOURCE_JSON"] = os.path.join( + os.path.realpath(os.path.dirname(__file__)), + "refs", + "obtain-resource.json", + ) + + @classmethod + def tearDownClass(cls) -> None: + """After running the suite we unset the gem5-resource JSON file, as to + not interfere with others tests. + """ + del os.environ["GEM5_RESOURCE_JSON"] + + def get_resource_dir(cls) -> str: + """To ensure the resources are cached to the same directory as all + other tests, this function returns the location of the testing + directories "resources" directory. + """ + return os.path.join( + os.path.realpath(os.path.dirname(__file__)), + os.pardir, + os.pardir, + os.pardir, + "gem5", + "resources", + ) + + def test_obtain_resources_no_version(self): + """Test that the resource loader returns latest version compatible with that version of gem5 when no version is specified.""" + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + ) + self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals("test description", resource.get_description()) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_compatible(self): + gem5Version = core.gem5Version + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + # self.assertIn(gem5Version, resource.get_gem5_versions()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_with_version_incompatible(self): + resource = None + f = io.StringIO() + with contextlib.redirect_stderr(f): + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertTrue( + f"warn: Resource test-binary-resource with version 1.5.0 is not known to be compatible with gem5 version {core.gem5Version}. " + "This may cause problems with your simulation. This resource's compatibility with different gem5 versions can be found here: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in f.getvalue() + ) + + resource = obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="1.5.0", + ) + self.assertEquals("1.5.0", resource.get_resource_version()) + self.assertIsInstance(resource, BinaryResource) + self.assertEquals( + "test description for 1.5.0", resource.get_description() + ) + self.assertEquals("src/test-source", resource.get_source()) + self.assertEquals(ISA.ARM, resource.get_architecture()) + + def test_obtain_resources_no_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_id(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="invalid-id", + resource_directory=self.get_resource_dir(), + resource_version="1.7.0", + ) + self.assertTrue( + "Resource with ID 'invalid-id' not found." + in str(context.exception) + ) + + def test_obtain_resources_with_version_invalid_version(self): + with self.assertRaises(Exception) as context: + obtain_resource( + resource_id="test-binary-resource", + resource_directory=self.get_resource_dir(), + resource_version="3.0.0", + ) + print("context.exception: ", context.exception) + print(str(context.exception)) + self.assertTrue( + f"Resource test-binary-resource with version '3.0.0'" + " not found.\nResource versions can be found at: " + f"https://resources.gem5.org/resources/test-binary-resource/versions" + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py new file mode 100644 index 0000000000..8f6674ff0d --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import tempfile +import os +from typing import Dict + +from gem5.resources.downloader import ( + get_resources_json_obj, +) + + +class ResourceDownloadTestSuite(unittest.TestCase): + """Test cases for gem5.resources.downloader""" + + @classmethod + def setUpClass(cls) -> str: + pass + + def get_resource_json_by_id(self) -> None: + """Get a resource by its id""" + resources = get_resources_json_obj("test-version") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "2.0.0") + + def get_resource_json_invalid_id(self) -> None: + """Should throw an exception when trying to get a resource that doesn't exist""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("this-resource-doesnt-exist") + self.assertTrue( + f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" + in str(context.exception) + ) + + def get_resource_json_by_id_and_version(self) -> None: + """Get a resource by its id and version""" + resources = get_resources_json_obj("test-version", "1.0.0") + self.assertEqual(resources["id"], "test-version") + self.assertEqual(resources["resource_version"], "1.0.0") + + def get_resource_json_by_id_and_invalid_version(self) -> None: + """Get a resource by its id and an invalid version (does not exist)""" + with self.assertRaises(Exception) as context: + get_resources_json_obj("test-version", "3.0.0") + self.assertTrue( + f"Specified Version 3.0.0 does not exist for the resource 'test-version'." + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 660bf5f38f..5c22a7341e 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -37,7 +37,25 @@ from gem5.resources.looppoint import ( from gem5.isas import ISA +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +mock_json_path = Path(__file__).parent / "refs/resource-specialization.json" + +mock_config_json = { + "sources": { + "baba": { + "url": mock_json_path, + "isMongo": False, + } + }, +} + + +@patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json), +) class ResourceSpecializationSuite(unittest.TestCase): """This suite tests that `gem5.resource.resource` casts to the correct `AbstractResource` specialization when using the `obtain_resource` @@ -79,14 +97,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_binary_resource(self) -> None: """Tests the loading of of a BinaryResource""" resource = obtain_resource( - resource_name="binary-example", + resource_id="binary-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BinaryResource) self.assertEquals( - "binary-example documentation.", resource.get_documentation() + "binary-example documentation.", resource.get_description() ) self.assertEquals("src/simple", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -94,14 +112,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_kernel_resource(self) -> None: """Tests the loading of a KernelResource.""" resource = obtain_resource( - resource_name="kernel-example", + resource_id="kernel-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, KernelResource) self.assertEquals( - "kernel-example documentation.", resource.get_documentation() + "kernel-example documentation.", resource.get_description() ) self.assertEquals("src/linux-kernel", resource.get_source()) self.assertEquals(ISA.RISCV, resource.get_architecture()) @@ -109,14 +127,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_bootloader_resource(self) -> None: """Tests the loading of a BootloaderResource.""" resource = obtain_resource( - resource_name="bootloader-example", + resource_id="bootloader-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, BootloaderResource) self.assertEquals( - "bootloader documentation.", resource.get_documentation() + "bootloader documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_architecture()) @@ -124,14 +142,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_disk_image_resource(self) -> None: """Tests the loading of a DiskImageResource.""" resource = obtain_resource( - resource_name="disk-image-example", + resource_id="disk-image-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DiskImageResource) self.assertEquals( - "disk-image documentation.", resource.get_documentation() + "disk-image documentation.", resource.get_description() ) self.assertEquals("src/x86-ubuntu", resource.get_source()) self.assertEquals("1", resource.get_root_partition()) @@ -139,40 +157,40 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_checkpoint_resource(self) -> None: """Tests the loading of a CheckpointResource.""" resource = obtain_resource( - resource_name="checkpoint-example", + resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, CheckpointResource) self.assertEquals( - "checkpoint-example documentation.", resource.get_documentation() + "checkpoint-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) def test_git_resource(self) -> None: """Tests the loading of a GitResource.""" resource = obtain_resource( - resource_name="git-example", + resource_id="git-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, GitResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_simpoint_directory_resource(self) -> None: """Tests the loading of a Simpoint directory resource.""" resource = obtain_resource( - resource_name="simpoint-directory-example", + resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointDirectoryResource) self.assertEquals( - "simpoint directory documentation.", resource.get_documentation() + "simpoint directory documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -199,14 +217,14 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_simpoint_resource(self) -> None: """Tests the loading of a Simpoint resource.""" resource = obtain_resource( - resource_name="simpoint-example", + resource_id="simpoint-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, SimpointResource) self.assertEquals( - "simpoint documentation.", resource.get_documentation() + "simpoint documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) self.assertIsNone(resource.get_local_path()) @@ -219,26 +237,27 @@ class ResourceSpecializationSuite(unittest.TestCase): def test_file_resource(self) -> None: """Tests the loading of a FileResource.""" resource = obtain_resource( - resource_name="file-example", + resource_id="file-example", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, FileResource) - self.assertIsNone(resource.get_documentation()) + self.assertIsNone(resource.get_description()) self.assertIsNone(resource.get_source()) def test_directory_resource(self) -> None: """Tests the loading of a DirectoryResource.""" resource = obtain_resource( - resource_name="directory-example", + resource_id="directory-example", resource_directory=self.get_resource_dir(), ) self.assertIsInstance(resource, DirectoryResource) self.assertEquals( - "directory-example documentation.", resource.get_documentation() + "directory-example documentation.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -247,7 +266,7 @@ class ResourceSpecializationSuite(unittest.TestCase): pinpoints csv file.""" resource = obtain_resource( - resource_name="looppoint-pinpoint-csv-resource", + resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), ) @@ -258,7 +277,7 @@ class ResourceSpecializationSuite(unittest.TestCase): self.assertIsInstance(resource, LooppointCsvLoader) self.assertEquals( - "A looppoint pinpoints csv file.", resource.get_documentation() + "A looppoint pinpoints csv file.", resource.get_description() ) self.assertIsNone(resource.get_source()) @@ -267,8 +286,9 @@ class ResourceSpecializationSuite(unittest.TestCase): Looppoint JSON file.""" resource = obtain_resource( - resource_name="looppoint-json-restore-resource-region-1", + resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), + resource_version="1.0.0", ) self.assertIsInstance(resource, LooppointJsonResource) @@ -278,6 +298,6 @@ class ResourceSpecializationSuite(unittest.TestCase): self.assertTrue("1" in resource.get_regions()) self.assertEquals( - "A looppoint json file resource.", resource.get_documentation() + "A looppoint json file resource.", resource.get_description() ) self.assertIsNone(resource.get_source()) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index 2bc31f5a3f..b898faeb79 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 The Regents of the University of California +# Copyright (c) 2023 The Regents of the University of California # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -36,6 +36,29 @@ from gem5.resources.resource import ( from typing import Dict +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch +from pathlib import Path + +mock_config_json1 = { + "sources": { + "baba": { + "url": Path(__file__).parent + / "refs/workload-checks-custom-workload.json", + "isMongo": False, + } + }, +} + +mock_config_json2 = { + "sources": { + "baba": { + "url": Path(__file__).parent / "refs/workload-checks.json", + "isMongo": False, + } + }, +} + class CustomWorkloadTestSuite(unittest.TestCase): """ @@ -43,8 +66,11 @@ class CustomWorkloadTestSuite(unittest.TestCase): """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json1), + ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -114,8 +140,7 @@ class CustomWorkloadTestSuite(unittest.TestCase): "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old value. - self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -124,8 +149,11 @@ class WorkloadTestSuite(unittest.TestCase): """ @classmethod + @patch( + "gem5.resources.client.clientwrapper", + ClientWrapper(mock_config_json2), + ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( os.path.realpath(os.path.dirname(__file__)), "refs", @@ -157,9 +185,9 @@ class WorkloadTestSuite(unittest.TestCase): self.assertTrue("kernel" in parameters) self.assertTrue(isinstance(parameters["kernel"], BinaryResource)) - self.assertTrue("disk_image" in parameters) + self.assertTrue("disk-image" in parameters) self.assertTrue( - isinstance(parameters["disk_image"], DiskImageResource) + isinstance(parameters["disk-image"], DiskImageResource) ) self.assertTrue("readfile_contents" in parameters) diff --git a/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json new file mode 100644 index 0000000000..fe87cc8c28 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json @@ -0,0 +1,30 @@ +[ + { + "category": "file", + "id": "test-duplicate", + "description": "test", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/mongo_mock.json b/tests/pyunit/stdlib/resources/refs/mongo_mock.json new file mode 100644 index 0000000000..b6376cc5e4 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/mongo_mock.json @@ -0,0 +1,56 @@ +[ + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json new file mode 100644 index 0000000000..fac95e106a --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -0,0 +1,59 @@ +[ + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.5.0", + "gem5_versions": [ + "25.0" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "2.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description v1.7.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.7.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "test-binary-resource", + "description": "test description for 1.5.0", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/test-source", + "resource_version": "1.5.0", + "gem5_versions": [ + "21.1", + "22.1" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index c4d5eb4714..1129f1bd05 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -1,132 +1,181 @@ - -{ - "version" : "develop", - "url_base" : "http://dist.gem5.org/dist/v22-1", - "previous-versions" : { - "develop" : "https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/develop/resources.json?format=TEXT", - "21.2" : "http://resources.gem5.org/prev-resources-json/resources-21-2.json" +[ + { + "category": "kernel", + "id": "kernel-example", + "description": "kernel-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "60a53c7d47d7057436bf4b9df707a841", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.4.49", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] }, - "resources": [ - { - "type" : "kernel", - "name" : "kernel-example", - "documentation" : "kernel-example documentation.", - "architecture" : "RISCV", - "is_zipped" : false, - "md5sum" : "60a53c7d47d7057436bf4b9df707a841", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.4.49", - "source" : "src/linux-kernel" - }, - { - "type" : "disk-image", - "name" : "disk-image-example", - "documentation" : "disk-image documentation.", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" - }, - { - "type" : "binary", - "name" : "binary-example", - "documentation" : "binary-example documentation.", - "architecture" : "ARM", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static", - "source" : "src/simple" - }, - { - "type" : "bootloader", - "name" : "bootloader-example", - "documentation" : "bootloader documentation.", - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url" : "{url_base}/test-progs/hello/bin/arm/linux/hello64-static" - }, - { - "type" : "checkpoint", - "name" : "checkpoint-example", - "documentation" : "checkpoint-example documentation.", - "architecture": "RISCV", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "git", - "name" : "git-example", - "documentation" : null, - "is_zipped" : false, - "is_tar_archive" : true, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type" : "file", - "name" : "file-example", - "documentation" : null, - "is_zipped" : false, - "md5sum" : "71b2cb004fe2cda4556f0b1a38638af6", - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar", - "source" : null - }, - { - "type" : "directory", - "name" : "directory-example", - "documentation" : "directory-example documentation.", - "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - }, - { - "type": "simpoint-directory", - "name": "simpoint-directory-example", - "documentation": "simpoint directory documentation.", - "is_zipped" : false, - "md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb", - "source" : null, - "is_tar_archive" : true, - "url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar", - "simpoint_interval": 1000000, - "warmup_interval": 1000000, - "simpoint_file": "simpoint.simpt", - "weight_file": "simpoint.weight", - "workload_name": "Example Workload" - }, - { - "type": "simpoint", - "name": "simpoint-example", - "documentation": "simpoint documentation.", - "simpoint_interval": 1000000, - "warmup_interval": 23445, - "simpoint_list" : [2,3,4,15], - "weight_list" : [0.1, 0.2, 0.4, 0.3] - }, - { - "type": "looppoint-pinpoint-csv", - "name": "looppoint-pinpoint-csv-resource", - "documentation" : "A looppoint pinpoints csv file.", - "is_zipped" : false, - "md5sum" : "199ab22dd463dc70ee2d034bfe045082", - "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", - "source" : null - }, - { - "type": "looppoint-json", - "name": "looppoint-json-restore-resource-region-1", - "documentation" : "A looppoint json file resource.", - "is_zipped" : false, - "region_id" : "1", - "md5sum" : "a71ed64908b082ea619b26b940a643c1", - "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", - "source" : null - } - ] -} + { + "category": "disk-image", + "id": "disk-image-example", + "description": "disk-image documentation.", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "binary", + "id": "binary-example", + "description": "binary-example documentation.", + "architecture": "ARM", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "bootloader", + "id": "bootloader-example", + "description": "bootloader documentation.", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "checkpoint", + "id": "checkpoint-example", + "description": "checkpoint-example documentation.", + "architecture": "RISCV", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "git", + "id": "git-example", + "description": null, + "is_zipped": false, + "is_tar_archive": true, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "file", + "id": "file-example", + "description": null, + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "directory", + "id": "directory-example", + "description": "directory-example documentation.", + "is_zipped": false, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint-directory", + "id": "simpoint-directory-example", + "description": "simpoint directory documentation.", + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "source": null, + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "simpoint", + "id": "simpoint-example", + "description": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list": [ + 2, + 3, + 4, + 15 + ], + "weight_list": [ + 0.1, + 0.2, + 0.4, + 0.3 + ], + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-pinpoint-csv", + "id": "looppoint-pinpoint-csv-resource", + "description": "A looppoint pinpoints csv file.", + "is_zipped": false, + "md5sum": "199ab22dd463dc70ee2d034bfe045082", + "url": "http://dist.gem5.org/dist/develop/pinpoints/x86-matrix-multiply-omp-100-8-global-pinpoints-20230127", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "looppoint-json", + "id": "looppoint-json-restore-resource-region-1", + "description": "A looppoint json file resource.", + "is_zipped": false, + "region_id": "1", + "md5sum": "a71ed64908b082ea619b26b940a643c1", + "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", + "source": null, + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json new file mode 100644 index 0000000000..812caeff43 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -0,0 +1,329 @@ +[ + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest", + "testing", + "riscv", + "testing", + "new" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "1.1.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "binary", + "id": "this-is-a-test-resource", + "description": "This is a test resource but double newer", + "architecture": "X86", + "size": 13816, + "tags": [ + "asmtest" + ], + "is_zipped": false, + "md5sum": "4e70a98b6976969deffff91eed17fba1", + "source": "src/asmtest", + "url": "http://dist.gem5.org/dist/develop/test-progs/asmtest/bin/rv64mi-p-sbreak", + "code_examples": [], + "license": " BSD-3-Clause", + "author": [], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.1" + ], + "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" + }, + { + "category": "simpoint", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-version", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "test-duplicate", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "aa", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "asfsaf", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "simpoint-resource", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "bat43f34fman", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "file", + "id": "adadadas", + "description": "Simpoints for running the 'x86-print-this' resource with the parameters `\"print this\" 15000`. This is encapsulated in the 'x86-print-this-15000-with-simpoints' workload.", + "architecture": "X86", + "size": 10240, + "tags": [], + "is_zipped": false, + "md5sum": "3fcffe3956c8a95e3fb82e232e2b41fb", + "is_tar_archive": true, + "url": "http://dist.gem5.org/dist/develop/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "code_examples": [], + "license": "", + "author": [], + "source_url": "", + "resource_version": "0.2.0", + "gem5_versions": [ + "23.0" + ], + "workload_name": "x86-print-this-15000-with-simpoints", + "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", + "workloads": [ + "x86-print-this-15000-with-simpoints", + "x86-print-this-15000-with-simpoints-and-checkpoint" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "This is a test resource", + "architecture": "X86", + "size": 688119691, + "tags": [ + "x86", + "fullsystem" + ], + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "source": "src/x86-ubuntu", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "root_partition": "1", + "code_examples": [], + "license": "", + "author": [ + "Ayaz Akram" + ], + "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "resource_version": "2.0.0", + "gem5_versions": [ + "23.0" + ], + "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json index a9dd2aaa46..8349b92658 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json @@ -1,17 +1,16 @@ -{ - "version" : null, - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "binary", - "name" : "x86-hello64-static", - "documentation" : "A 'Hello World!' binary.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "dbf120338b37153e3334603970cebd8c", - "url" : "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", - "source" : "src/simple" - } - ] -} +[ + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index 4f7e76bfb5..d19396bf8c 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -1,40 +1,48 @@ -{ - "url_base" : "http://dist.gem5.org/dist/v22-0", - "previous-versions" : {}, - "resources": [ - { - "type" : "kernel", - "name" : "x86-linux-kernel-5.2.3", - "documentation" : "The linux kernel (v5.2.3), compiled to X86.", - "architecture" : "X86", - "is_zipped" : false, - "md5sum" : "4838c99b77d33c8307b939c16624e4ac", - "url" : "{url_base}/kernels/x86/static/vmlinux-5.2.3", - "source" : "src/linux-kernel" +[ + { + "category": "kernel", + "id": "x86-linux-kernel-5.2.3", + "description": "The linux kernel (v5.2.3), compiled to X86.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "4838c99b77d33c8307b939c16624e4ac", + "url": "{url_base}/kernels/x86/static/vmlinux-5.2.3", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img", + "description": "A disk image containing Ubuntu 18.04 for x86..", + "architecture": "X86", + "is_zipped": true, + "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", + "url": "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + }, + { + "category": "workload", + "id": "simple-boot", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3", + "disk-image": "x86-ubuntu-18.04-img" }, - { - "type" : "disk-image", - "name" : "x86-ubuntu-18.04-img", - "documentation" : "A disk image containing Ubuntu 18.04 for x86..", - "architecture" : "X86", - "is_zipped" : true, - "md5sum" : "90e363abf0ddf22eefa2c7c5c9391c49", - "url" : "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", - "source" : "src/x86-ubuntu", - "root_partition": "1" + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" }, - { - "type" : "workload", - "name" : "simple-boot", - "documentation" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.2.3", - "disk_image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "echo 'Boot successful'; m5 exit" - } - } - ] -} + "resource_version": "1.0.0", + "gem5_versions": [ + "23.0" + ] + } +] From 4434d4897368b48e464ed70bbdd5a45d1c4a326f Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Fri, 2 Jun 2023 09:24:42 +0100 Subject: [PATCH 05/35] arch-arm: Apply FEAT_IDST to missing ID registers When FEAT_IDST got implemented [1], we forgot to add the logic for AArch64 ID registers tracking AArch32 state/capabilities [1]: https://gem5-review.googlesource.com/c/public/gem5/+/70723 Signed-off-by: Giacomo Travaglini Change-Id: I19bddf67ecc379a14f91cfede385692536982101 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71178 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Reviewed-by: Richard Cooper Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Maintainer: Jason Lowe-Power --- src/arch/arm/regs/misc.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/arch/arm/regs/misc.cc b/src/arch/arm/regs/misc.cc index f32aa7230c..f1c69cc007 100644 --- a/src/arch/arm/regs/misc.cc +++ b/src/arch/arm/regs/misc.cc @@ -3911,83 +3911,104 @@ ISA::initializeMiscRegMetadata() .mapsTo(MISCREG_MIDR); InitReg(MISCREG_MPIDR_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .mapsTo(MISCREG_MPIDR); InitReg(MISCREG_REVIDR_EL1) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid1)) .allPrivileges().exceptUserMode().writes(0); InitReg(MISCREG_ID_PFR0_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_PFR0); InitReg(MISCREG_ID_PFR1_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_PFR1); InitReg(MISCREG_ID_DFR0_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_DFR0); InitReg(MISCREG_ID_AFR0_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_AFR0); InitReg(MISCREG_ID_MMFR0_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_MMFR0); InitReg(MISCREG_ID_MMFR1_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_MMFR1); InitReg(MISCREG_ID_MMFR2_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_MMFR2); InitReg(MISCREG_ID_MMFR3_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_MMFR3); InitReg(MISCREG_ID_MMFR4_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_MMFR4); InitReg(MISCREG_ID_ISAR0_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR0); InitReg(MISCREG_ID_ISAR1_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR1); InitReg(MISCREG_ID_ISAR2_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR2); InitReg(MISCREG_ID_ISAR3_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR3); InitReg(MISCREG_ID_ISAR4_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR4); InitReg(MISCREG_ID_ISAR5_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR5); InitReg(MISCREG_ID_ISAR6_EL1) .allPrivileges().exceptUserMode().writes(0) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .mapsTo(MISCREG_ID_ISAR6); InitReg(MISCREG_MVFR0_EL1) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .allPrivileges().exceptUserMode().writes(0) .mapsTo(MISCREG_MVFR0); InitReg(MISCREG_MVFR1_EL1) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .allPrivileges().exceptUserMode().writes(0) .mapsTo(MISCREG_MVFR1); InitReg(MISCREG_MVFR2_EL1) + .faultRead(EL0, faultIdst) .faultRead(EL1, HCR_TRAP(tid3)) .allPrivileges().exceptUserMode().writes(0); InitReg(MISCREG_ID_AA64PFR0_EL1) From 3322127793e1d7fbaa9f28df14a3f5f72fa772f6 Mon Sep 17 00:00:00 2001 From: KUNAL PAI Date: Wed, 7 Jun 2023 15:30:15 -0700 Subject: [PATCH 06/35] tests: Fix bugs related to gem5 Vision This patch fixes refs under tests/pyunit/stdlib/resources. Removes instances of {url_base} in refs. Also, renames two refs: mongo_mock and mongo_dup_mock to mongo-mock and mongo-dup-mock to follow naming convention of other refs. Change-Id: If115114bc7a89764e7c546b77a93d36d6a3b5f8a Co-authored-by: Parth Shah Co-authored-by: Harshil2107 Co-authored-by: aarsli Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71378 Tested-by: kokoro Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py | 4 ++-- .../refs/{mongo_dup_mock.json => mongo-dup-mock.json} | 0 .../resources/refs/{mongo_mock.json => mongo-mock.json} | 0 .../resources/refs/workload-checks-custom-workload.json | 2 +- tests/pyunit/stdlib/resources/refs/workload-checks.json | 4 ++-- 5 files changed, 5 insertions(+), 5 deletions(-) rename tests/pyunit/stdlib/resources/refs/{mongo_dup_mock.json => mongo-dup-mock.json} (100%) rename tests/pyunit/stdlib/resources/refs/{mongo_mock.json => mongo-mock.json} (100%) diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py index 96aadf6879..344f67b8b0 100644 --- a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -82,12 +82,12 @@ mock_config_combined = { mock_json = {} -with open(Path(__file__).parent / "refs/mongo_mock.json", "r") as f: +with open(Path(__file__).parent / "refs/mongo-mock.json", "r") as f: mock_json = json.load(f) duplicate_mock_json = {} -with open(Path(__file__).parent / "refs/mongo_dup_mock.json", "r") as f: +with open(Path(__file__).parent / "refs/mongo-dup-mock.json", "r") as f: duplicate_mock_json = json.load(f) diff --git a/tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json b/tests/pyunit/stdlib/resources/refs/mongo-dup-mock.json similarity index 100% rename from tests/pyunit/stdlib/resources/refs/mongo_dup_mock.json rename to tests/pyunit/stdlib/resources/refs/mongo-dup-mock.json diff --git a/tests/pyunit/stdlib/resources/refs/mongo_mock.json b/tests/pyunit/stdlib/resources/refs/mongo-mock.json similarity index 100% rename from tests/pyunit/stdlib/resources/refs/mongo_mock.json rename to tests/pyunit/stdlib/resources/refs/mongo-mock.json diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json index 8349b92658..a7e9c9d84f 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json @@ -6,7 +6,7 @@ "architecture": "X86", "is_zipped": false, "md5sum": "dbf120338b37153e3334603970cebd8c", - "url": "{url_base}/test-progs/hello/bin/x86/linux/hello64-static", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", "source": "src/simple", "resource_version": "1.0.0", "gem5_versions": [ diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index d19396bf8c..d41001d26c 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -6,7 +6,7 @@ "architecture": "X86", "is_zipped": false, "md5sum": "4838c99b77d33c8307b939c16624e4ac", - "url": "{url_base}/kernels/x86/static/vmlinux-5.2.3", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.2.3", "source": "src/linux-kernel", "resource_version": "1.0.0", "gem5_versions": [ @@ -20,7 +20,7 @@ "architecture": "X86", "is_zipped": true, "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", - "url": "{url_base}/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", "source": "src/x86-ubuntu", "root_partition": "1", "resource_version": "1.0.0", From 8219b1961a66f786588860810135a7e405bd70c8 Mon Sep 17 00:00:00 2001 From: Roger Chang Date: Mon, 5 Jun 2023 14:32:18 +0800 Subject: [PATCH 07/35] scons: Fix grpc protobuf actions The change will fix the proto import issue and build issue with --no-duplicate-sources options, more details please reference: https://gem5-review.googlesource.com/c/public/gem5/+/64491. Change-Id: I259413f7739f89598dcd42c3f2e1e865cec3de43 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71318 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Maintainer: Jason Lowe-Power Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71338 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- src/SConscript | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SConscript b/src/SConscript index 13f08d2f5a..d26bf49cce 100644 --- a/src/SConscript +++ b/src/SConscript @@ -280,9 +280,9 @@ def protoc_grpc_emitter(target, source, env): root, ext = os.path.splitext(source[0].get_abspath()) return [root + '.grpc.pb.cc', root + '.grpc.pb.h'], source -protoc_grpc_action=MakeAction('${PROTOC} --grpc_out ${BUILDDIR} ' - '--plugin=protoc-gen-grpc=${PROTOC_GRPC} --proto_path ${BUILDDIR} ' - '${SOURCE.get_abspath()}', +protoc_grpc_action=MakeAction('${PROTOC} --grpc_out ${TARGET.dir.abspath} ' + '--plugin=protoc-gen-grpc=${PROTOC_GRPC} ' + '--proto_path ${SOURCE.dir.abspath} ${SOURCE.abspath}', Transform("PROTOC")) env.Append(BUILDERS={'GrpcProtoBufCC' : Builder( From d33c41118b1636b55a5aadb34e5724c9be673896 Mon Sep 17 00:00:00 2001 From: Xuan Hu Date: Tue, 17 Jan 2023 21:08:57 +0800 Subject: [PATCH 08/35] arch-riscv,cpu-minor: Add MinorDefaultVecFU for risc-v v-ext Change-Id: Id5c5ae5fa1901154cadeb0a4958703f3f15d491f Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/67295 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71398 Maintainer: Bobby Bruce --- src/cpu/minor/BaseMinorCPU.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/cpu/minor/BaseMinorCPU.py b/src/cpu/minor/BaseMinorCPU.py index bd27b92540..c20a310447 100644 --- a/src/cpu/minor/BaseMinorCPU.py +++ b/src/cpu/minor/BaseMinorCPU.py @@ -250,6 +250,33 @@ class MinorDefaultMiscFU(MinorFU): opLat = 1 +class MinorDefaultVecFU(MinorFU): + opClasses = minorMakeOpClassSet( + [ + "VectorUnitStrideLoad", + "VectorUnitStrideStore", + "VectorUnitStrideMaskLoad", + "VectorUnitStrideMaskStore", + "VectorStridedLoad", + "VectorStridedStore", + "VectorIndexedLoad", + "VectorIndexedStore", + "VectorUnitStrideFaultOnlyFirstLoad", + "VectorWholeRegisterLoad", + "VectorWholeRegisterStore", + "VectorIntegerArith", + "VectorFloatArith", + "VectorFloatConvert", + "VectorIntegerReduce", + "VectorFloatReduce", + "VectorMisc", + "VectorIntegerExtension", + "VectorConfig", + ] + ) + opLat = 1 + + class MinorDefaultFUPool(MinorFUPool): funcUnits = [ MinorDefaultIntFU(), @@ -260,6 +287,7 @@ class MinorDefaultFUPool(MinorFUPool): MinorDefaultPredFU(), MinorDefaultMemFU(), MinorDefaultMiscFU(), + MinorDefaultVecFU(), ] From 1ae409d627f8dc1945bb28ece10747f4744b1b9c Mon Sep 17 00:00:00 2001 From: Roger Chang Date: Sat, 3 Jun 2023 10:30:38 +0800 Subject: [PATCH 09/35] stdlib: Add U74VecFU to U74CPU This change is to elimilate the warning message from U74CPU. Change-Id: I7a5d0cd0b2955e54ed14fc1ac6f7127bd7f0604b Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71238 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Maintainer: Jason Lowe-Power Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71399 Maintainer: Bobby Bruce --- src/python/gem5/prebuilt/riscvmatched/riscvmatched_core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/python/gem5/prebuilt/riscvmatched/riscvmatched_core.py b/src/python/gem5/prebuilt/riscvmatched/riscvmatched_core.py index 48291bf670..4b8d2c1d32 100644 --- a/src/python/gem5/prebuilt/riscvmatched/riscvmatched_core.py +++ b/src/python/gem5/prebuilt/riscvmatched/riscvmatched_core.py @@ -75,6 +75,10 @@ class U74MiscFU(MinorDefaultMiscFU): pass +class U74VecFU(MinorDefaultVecFU): + pass + + class U74FUPool(MinorFUPool): funcUnits = [ U74IntFU(), @@ -86,6 +90,7 @@ class U74FUPool(MinorFUPool): U74MemReadFU(), U74MemWriteFU(), U74MiscFU(), + U74VecFU(), ] From 27492d9d1854f1fb80bd91e3f69659cad08ffd70 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Thu, 8 Jun 2023 14:23:11 -0700 Subject: [PATCH 10/35] misc: Update GitHub yaml files This introduces a few fixes to the yaml files, such as fixing the failing SystemC tests by removing an extra Docker container, declaring the Python version for our compiler tests, fixing typos in the GPU daily tests, and adjusting the timeout values for our compiler tests and unittests. Change-Id: I3b4f2220750138783717b3678b0d8676155cb2a1 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71400 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro Maintainer: Matt Sinclair Reviewed-by: Matt Sinclair --- .github/workflows/ci-tests.yaml | 15 ++++++++++++--- .github/workflows/compiler-tests.yaml | 8 ++++---- .github/workflows/daily-tests.yaml | 26 ++++++++++---------------- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 309fefce2a..16c096dc9a 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -8,9 +8,18 @@ on: types: [opened, reopened, edited] jobs: + pre-commit: + # runs on github hosted runner + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 + build-gem5: runs-on: [self-hosted, linux, x64, build] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: pre-commit # only runs if pre-commit passes outputs: artifactname: ${{ steps.name.outputs.test }} steps: @@ -29,8 +38,8 @@ jobs: unittests-all-opt: runs-on: [self-hosted, linux, x64, run] - container: - image: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: pre-commit # only runs if pre-commit passes timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -42,7 +51,7 @@ jobs: testlib-quick: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 + needs: [pre-commit, build-gem5] timeout-minutes: 360 # 6 hours steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 300f9eba36..e7e9cd2424 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -18,7 +18,7 @@ jobs: opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/${{ matrix.image }}:latest - timeout-minutes: 2880 # 48 hours + timeout-minutes: 240 # 4 hours steps: - uses: actions/checkout@v3 with: @@ -26,7 +26,7 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: scons build/ALL/gem5${{ matrix.opts }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} # Tests the two latest gcc and clang supported compilers against all gem5 compilations. latest-compilers-all-gem5-builds: @@ -37,7 +37,7 @@ jobs: opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/${{ matrix.image }}:latest - timeout-minutes: 2880 # 48 hours + timeout-minutes: 240 # 4 hours steps: - uses: actions/checkout@v3 with: @@ -45,4 +45,4 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: scons build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 91bb7d6f8a..97b41642b7 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -223,7 +223,7 @@ jobs: unittests-all-debug: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -240,7 +240,7 @@ jobs: unittests-all-fast: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -837,22 +837,16 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Build ARM/gem5.opt - uses: docker://gcr.io/gem5-test/ubuntu-22.04_min-dependencies:latest - # This is a bit of a hack so we can run a different docker container. - with: - args: scons --ignore-style --duplicate-sources -j4 build/ARM/gem5.opt + run: scons build/ARM/gem5.opt --ignore-style --duplicate-sources -j$(nproc) - name: Build ARM/libgem5_opt.so - uses: docker://gcr.io/gem5-test/ubuntu-22.04_min-dependencies:latest - # This is a bit of a hack so we can run a different docker container. - with: - args: scons --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j4 --duplicate-sources build/ARM/libgem5_opt.so + run: scons build/ARM/libgem5_opt.so --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j$(nproc) --duplicate-sources - name: Compile gem5 withing SystemC working-directory: ${{ github.workspace }}/util/systemc/gem5_within_systemc run: make - name: Run gem5 within SystemC test run: ./build/ARM/gem5.opt configs/deprecated/example/se.py -c tests/test-progs/hello/bin/arm/linux/hello - - name: bla - run: LD_LIBRARY_PATH=build/ARM/:/opt/systemc/lib-linux64/ ./gem5.opt.sc m5out/config.ini + - name: Continue gem5 within SystemC test + run: LD_LIBRARY_PATH=build/ARM/:/opt/systemc/lib-linux64/ ./util/systemc/gem5_within_systemc/gem5.opt.sc m5out/config.ini # Runs the gem5 Nighyly GPU tests. gpu-tests: @@ -872,7 +866,7 @@ jobs: uses: wei/wget@v1 with: args: -q http://dist.gem5.org/dist/develop/test-progs/square/square # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time: https://github.com/coder/sshcode/issues/102 - - name: Run Square test with GCN3_X8/gem5.opt (SE mode) + - name: Run Square test with GCN3_X86/gem5.opt (SE mode) run: | mkdir -p tests/testing-results ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c square @@ -880,7 +874,7 @@ jobs: uses: wei/wget@v1 with: args: -q http://dist.gem5.org/dist/develop/test-progs/heterosync/gcn3/allSyncPrims-1kernel # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time - - name: Run allSyncPrims-1kernel sleepMutex test with GCN3_X8/gem5.opt (SE mode) + - name: Run allSyncPrims-1kernel sleepMutex test with GCN3_X86/gem5.opt (SE mode) run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="sleepMutex 10 16 4" - - name: Run allSysncPrims-1kernel lfTreeBarrUsing test with GCN3_X8/gem5.opt (SE mode) - run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="lfTreeBarrUsing 10 16 4" + - name: Run allSyncPrims-1kernel lfTreeBarrUsing test with GCN3_X86/gem5.opt (SE mode) + run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="lfTreeBarrUniq 10 16 4" From cc564c8931d31eda9a9baf19d11cafc7138127b1 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Thu, 8 Jun 2023 14:25:06 -0700 Subject: [PATCH 11/35] misc: Add weekly tests to Github workflows This introduces the very-long tests from our weekly tests to be able to run within GitHub. Change-Id: I5f5b4c521b47272cb27640c572a869c26c04b95a Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71401 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro --- .github/workflows/weekly-tests.yaml | 178 ++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .github/workflows/weekly-tests.yaml diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml new file mode 100644 index 0000000000..48fb79f639 --- /dev/null +++ b/.github/workflows/weekly-tests.yaml @@ -0,0 +1,178 @@ +# This workflow runs all of the very-long tests within main.py + +name: Running weekly main.py +run-name: ${{ github.actor }} is running the weekly tests + + +on: + # Runs every Sunday from 7AM UTC + schedule: + - cron: '00 7 * * 6' + # Allows us to manually start workflow for testing + workflow_dispatch: + +jobs: + build-gem5: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + +# start running the very-long tests + testlib-very-long-gem5_library_example_tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: very-long gem5_library_example_tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/gem5_library_example_tests --length very-long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + + testlib-long-gem5-resources: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5] + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: very-long gem5-resources tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/gem5-resources --length very-long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-very-long-parsec-benchmarks: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: very-long x86-boot-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/x86-boot-tests --length very-long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." + + testlib-very-long-x86-boot-tests: + runs-on: [self-hosted, linux, x64, run] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: build-gem5 + timeout-minutes: 1440 # 24 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: very-long x86-boot-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/x86-boot-tests --length very-long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." From f78471fb81db40c836b89a015a61b8311b4b98fd Mon Sep 17 00:00:00 2001 From: Mahyar Samani Date: Thu, 8 Jun 2023 14:56:50 -0700 Subject: [PATCH 12/35] tests: Reducing json stat dump size. This change reduces the number of stats dumped as json in traffic_gen tests. Change-Id: I94becb2e6d5da6096271cf7893ff2b380314da06 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71402 Maintainer: Bobby Bruce Reviewed-by: Bobby Bruce Tested-by: kokoro --- tests/gem5/traffic_gen/simple_traffic_run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/gem5/traffic_gen/simple_traffic_run.py b/tests/gem5/traffic_gen/simple_traffic_run.py index 7c0f18865a..3766d7314f 100644 --- a/tests/gem5/traffic_gen/simple_traffic_run.py +++ b/tests/gem5/traffic_gen/simple_traffic_run.py @@ -209,7 +209,9 @@ print("Beginning simulation!") exit_event = m5.simulate() print(f"Exiting @ tick {m5.curTick()} because {exit_event.getCause()}.") -simstats = get_simstat(root, prepare_stats=True) +simstats = get_simstat( + [core.generator for core in generator.get_cores()], prepare_stats=True +) json_output = Path(m5.options.outdir) / "output.json" with open(json_output, "w") as stats_file: simstats.dump(stats_file, indent=2) From 2e2f869901f3447bc39c687e8c16e18ed798b196 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Fri, 9 Jun 2023 15:54:32 -0700 Subject: [PATCH 13/35] misc: Adjust timeout for compiler tests This updates the timeout for the compiler tests to run per step as opposed to by job, as well as increases this limit from 4 hours to 6 hours. Change-Id: I6b0fe5561f74e450cd79205a96084dd27ccccead Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71458 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro --- .github/workflows/compiler-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index e7e9cd2424..fe0574822a 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -18,7 +18,6 @@ jobs: opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/${{ matrix.image }}:latest - timeout-minutes: 240 # 4 hours steps: - uses: actions/checkout@v3 with: @@ -27,6 +26,7 @@ jobs: ref: develop - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} + timeout-minutes: 300 # 6 hours # Tests the two latest gcc and clang supported compilers against all gem5 compilations. latest-compilers-all-gem5-builds: @@ -37,7 +37,6 @@ jobs: opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/${{ matrix.image }}:latest - timeout-minutes: 240 # 4 hours steps: - uses: actions/checkout@v3 with: @@ -46,3 +45,4 @@ jobs: ref: develop - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} + timeout-minutes: 300 # 6 hours From 9c1603119c9e48ded2d63d46fb2a9d3fcfb53910 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Mon, 12 Jun 2023 14:37:51 -0700 Subject: [PATCH 14/35] misc: Increase timeout for GitHub Actions This increases the timeout for our compiler tests, and for the very-long runs in our weekly tests. Change-Id: I027d2e98ce5596430ffcb20ef484e1cd7d4f644b Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71501 Reviewed-by: Bobby Bruce Tested-by: kokoro Maintainer: Bobby Bruce --- .github/workflows/compiler-tests.yaml | 4 ++-- .github/workflows/weekly-tests.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index fe0574822a..8720e6d184 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -26,7 +26,7 @@ jobs: ref: develop - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} - timeout-minutes: 300 # 6 hours + timeout-minutes: 600 # 10 hours # Tests the two latest gcc and clang supported compilers against all gem5 compilations. latest-compilers-all-gem5-builds: @@ -45,4 +45,4 @@ jobs: ref: develop - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} - timeout-minutes: 300 # 6 hours + timeout-minutes: 600 # 10 hours diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 48fb79f639..64ab442233 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -40,7 +40,7 @@ jobs: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] - timeout-minutes: 1440 # 24 hours + timeout-minutes: 4320 # 3 days steps: - uses: actions/checkout@v3 with: @@ -76,7 +76,7 @@ jobs: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] - timeout-minutes: 1440 # 24 hours + timeout-minutes: 4320 # 3 days steps: - uses: actions/checkout@v3 with: @@ -111,7 +111,7 @@ jobs: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: build-gem5 - timeout-minutes: 1440 # 24 hours + timeout-minutes: 4320 # 3 days steps: - uses: actions/checkout@v3 with: @@ -146,7 +146,7 @@ jobs: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: build-gem5 - timeout-minutes: 1440 # 24 hours + timeout-minutes: 4320 # 3 days steps: - uses: actions/checkout@v3 with: From 5ca9541037378c230cbc5c15b7e1f90e77cfed5c Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 20:47:47 -0700 Subject: [PATCH 15/35] misc: Add 'synchronize' as CI Test action. Remove 'reopen' 'synchronize' will re-run the tests everytime the pull-request is updated, inclusive of rebasing. 'reopen' seems pointless as reopening does not change the pull-request functionality. Change-Id: I24c2686e19a8b0901323e9bd794142994dc2a87c Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71503 Maintainer: Bobby Bruce Reviewed-by: Jason Lowe-Power Tested-by: kokoro --- .github/workflows/ci-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 16c096dc9a..4c35c1a424 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -5,7 +5,8 @@ run-name: ${{ github.actor }} is running the pull-request continuous integration on: pull_request: - types: [opened, reopened, edited] + types: [opened, edited, synchronize] + jobs: pre-commit: From 20245469517e2aab6bb450598b5fd530329e60a3 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 20:50:36 -0700 Subject: [PATCH 16/35] misc: Update GitHub Actions Workflow names Change-Id: Iaf4cbdf7a8edd2b9ae00308ae41f6fa805ab9446 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71504 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power --- .github/workflows/compiler-tests.yaml | 2 +- .github/workflows/daily-tests.yaml | 2 +- .github/workflows/weekly-tests.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 8720e6d184..013a710066 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -1,6 +1,6 @@ # This workflow runs all of the compiler tests -name: Running main.py +name: Compiler Tests run-name: ${{ github.actor }} is running compiler tests diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 97b41642b7..2b8a15f7a5 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -1,6 +1,6 @@ # This workflow runs all of the long tests within main.py, extra tests in nightly.sh, and unittests -name: Running main.py +name: Daily Tests run-name: ${{ github.actor }} is running the daily tests diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 64ab442233..600449e648 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -1,6 +1,6 @@ # This workflow runs all of the very-long tests within main.py -name: Running weekly main.py +name: Weekly Tests run-name: ${{ github.actor }} is running the weekly tests From 55cecfc3e8fb2bdddfd96a850b0fdaca9bcdd0b3 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 21:00:28 -0700 Subject: [PATCH 17/35] misc: Remove 'run-name' from workflow yaml files This field is not necessary and was not a helpful description of the run. Change-Id: I078481920833fb36099e6c30295c0b60ac0ee3a0 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71505 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Bobby Bruce --- .github/workflows/ci-tests.yaml | 1 - .github/workflows/daily-tests.yaml | 2 -- .github/workflows/weekly-tests.yaml | 2 -- 3 files changed, 5 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 4c35c1a424..b1c5a1af31 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -1,7 +1,6 @@ # This workflow runs after a pull-request has been approved by a reviewer. name: CI Tests -run-name: ${{ github.actor }} is running the pull-request continuous integration tests on: pull_request: diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 2b8a15f7a5..5fab058b79 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -1,8 +1,6 @@ # This workflow runs all of the long tests within main.py, extra tests in nightly.sh, and unittests name: Daily Tests -run-name: ${{ github.actor }} is running the daily tests - on: # Runs every day from 7AM UTC diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 600449e648..4c3f6b55bc 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -1,8 +1,6 @@ # This workflow runs all of the very-long tests within main.py name: Weekly Tests -run-name: ${{ github.actor }} is running the weekly tests - on: # Runs every Sunday from 7AM UTC From d89ba08eb031dff1704261441a4eb21104ca7694 Mon Sep 17 00:00:00 2001 From: Jason Lowe-Power Date: Mon, 12 Jun 2023 15:31:04 -0700 Subject: [PATCH 18/35] python: Ignore -s as gem5 option This enables more compatibility with the normal python binary. This is needed to get multiprocessing to work on some systems. Change-Id: Ibb946136d153979bf54a773060010a0ae479a9d1 Signed-off-by: Jason Lowe-Power Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71518 Maintainer: Bobby Bruce Reviewed-by: Bobby Bruce Tested-by: kokoro --- src/python/m5/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/python/m5/main.py b/src/python/m5/main.py index a68279b633..4701dfa5ea 100644 --- a/src/python/m5/main.py +++ b/src/python/m5/main.py @@ -193,6 +193,13 @@ def parse_options(): callback=collect_args, ) + option( + "-s", + action="store_true", + help="IGNORED, only for compatibility with python. don't" + "add user site directory to sys.path; also PYTHONNOUSERSITE", + ) + # Statistics options group("Statistics Options") option( From f4559a703f16d98c9142bba6c51bc9d85ad684ec Mon Sep 17 00:00:00 2001 From: Ayaz Akram Date: Wed, 14 Jun 2023 11:03:33 -0700 Subject: [PATCH 19/35] configs: Fix SPEC benchmarks example scripts This small change fixes the gem5_library example scripts for SPEC benchmarks to make them compatible with the latest version of the std library. Change-Id: I3da9745f0ee6b253871e32082e135e0fa4040108 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71718 Maintainer: Jason Lowe-Power Tested-by: kokoro Reviewed-by: Jason Lowe-Power --- configs/example/gem5_library/x86-spec-cpu2006-benchmarks.py | 2 +- configs/example/gem5_library/x86-spec-cpu2017-benchmarks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/example/gem5_library/x86-spec-cpu2006-benchmarks.py b/configs/example/gem5_library/x86-spec-cpu2006-benchmarks.py index 60d93ebe77..10d5da0adb 100644 --- a/configs/example/gem5_library/x86-spec-cpu2006-benchmarks.py +++ b/configs/example/gem5_library/x86-spec-cpu2006-benchmarks.py @@ -262,7 +262,7 @@ board.set_kernel_disk_workload( kernel=Resource("x86-linux-kernel-4.19.83"), # The location of the x86 SPEC CPU 2017 image disk_image=CustomDiskImageResource( - args.image, disk_root_partition=args.partition + args.image, root_partition=args.partition ), readfile_contents=command, ) diff --git a/configs/example/gem5_library/x86-spec-cpu2017-benchmarks.py b/configs/example/gem5_library/x86-spec-cpu2017-benchmarks.py index c491c9bce9..cb5f5d19e3 100644 --- a/configs/example/gem5_library/x86-spec-cpu2017-benchmarks.py +++ b/configs/example/gem5_library/x86-spec-cpu2017-benchmarks.py @@ -278,7 +278,7 @@ board.set_kernel_disk_workload( kernel=Resource("x86-linux-kernel-4.19.83"), # The location of the x86 SPEC CPU 2017 image disk_image=CustomDiskImageResource( - args.image, disk_root_partition=args.partition + args.image, root_partition=args.partition ), readfile_contents=command, ) From 328aaa626fbbc3d664086b6dd4bde11bd71f9d5d Mon Sep 17 00:00:00 2001 From: Roger Chang Date: Tue, 13 Jun 2023 17:25:47 +0800 Subject: [PATCH 20/35] arch-riscv: Fix unexpected behavior of float operations in Mac OS The uint_fast16_t is the integer at least 16 bits size, it can be 32, 64 bits and more. Usually most of the simulations are in the x86-64 linux host, the size of uint_fast16_t is 64 bits. Therefore, there is no problem for double precision float operations and it can pass FloatMM test. However, in the Mac OS, the size of uint_fast16_t is 16 bits, it will lose the upper bits when converting float register bits to freg_t and it will generate unexpected results for FloatMM test. The change can guarantee that the size of data in freg_t is at least 64 bits and it will not lose any data from floating point to freg_t. Reference: https://developer.apple.com/documentation/kernel/uint_fast16_t https://codebrowser.dev/glibc/glibc/stdlib/stdint.h.html Change-Id: I3df6610f0903cdee0f56584d6cbdb51ac26c86c8 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71519 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- src/arch/riscv/regs/float.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/riscv/regs/float.hh b/src/arch/riscv/regs/float.hh index 1654bdb627..4809372070 100644 --- a/src/arch/riscv/regs/float.hh +++ b/src/arch/riscv/regs/float.hh @@ -105,7 +105,7 @@ static constexpr float64_t f64(freg_t r) { return r; } static constexpr freg_t freg(float16_t f) { return {boxF16(f.v)}; } static constexpr freg_t freg(float32_t f) { return {boxF32(f.v)}; } static constexpr freg_t freg(float64_t f) { return f; } -static constexpr freg_t freg(uint_fast16_t f) { return {f}; } +static constexpr freg_t freg(uint_fast64_t f) { return {f}; } namespace float_reg { From 82587ce71bbbdc80d3ef6386e07c892f309697a3 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 14:09:15 -0700 Subject: [PATCH 21/35] stdlib: Refactor gem5 Vision/gem5-resources code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch includes several changes to the gem5 tools interface to the gem5-resources infrastructure. These are: * The old download and JSON query functions have been removed from the downloader module. These functions were used for directly downloading and inspecting the resource JSON file, hosted at https://resources.gem5.org/resources. This information is now obtained via `gem5.client`. If a resources JSON file is specified as a client, it should conform to the new schema: https//resources.gem5.org/gem5-resources-schema.json. The old schema (pre-v23.0) is no longer valid. Tests have been updated to reflect this change. Those which tested these old functions have been removed. * Unused imports have been removed. * For the resource query functions, and those tasked with obtaining the resources, the parameter `gem5_version` has been added. In all cases it does the same thing: * It will filter results based on compatibility to the `gem5_version` specified. If no resources are compatible the latest version of that resource is chosen (though a warning is thrown). * By default it is set to the current gem5 version. * It is optional. If `None`, this filtering functionality is not carried out. * Tests have been updated to fix the version to “develop” so the they do not break between versions. * The `gem5_version` parameters will filter using a logic which will base compatibility on the specificity of the gem5-version specified in a resource’s data. If a resource has a compatible gem5-version of “v18.4” it will be compatible with any minor/hotfix version within the v18.4 release (this can be seen as matching on “v18.4.*.*”.) Likewise, if a resource has a compatible gem5-version of “v18.4.1” then it’s only compatible with the v18.4.1 release but any of it’s hot fix releases (“v18.4.1.*”). * The ‘list_resources’ function has been updated to use the “gem5.client” APIs to get resource information from the clients (MongoDB or a JSON file). This has been designed to remain backwards compatible to as much as is possible, though, due to schema changes, the function does search across all versions of gem5. * `get_resources` function was added to the `AbstractClient`. This is a more general function than `get_resource_by_id`. It was primarily created to handle the `list_resources` update but is a useful update to the API. The `get_resource_by_id` function has been altered to function as a wrapped to the `get_resources` function. * Removed “GEM5_RESOURCE_JSON” code has been removed. This is no longer used. * Tests have been cleaned up a little bit to be easier to read. * Some docstrings have been updated. Things that are left TODO with this code: * The client_wrapper/client/abstract_client abstractions are rather pointless. In particular the client_wrapper and client classes could be merged. * The downloader module no longer does much and should have its functions merged into other modules. * With the addition of the `get_resources` function, much of the code in the `AbstractClient` could be simplified. Change-Id: I0ce48e88b93a2b9db53d4749861fa0b5f9472053 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71506 Reviewed-by: Kunal Pai Maintainer: Jason Lowe-Power Tested-by: kokoro --- src/python/gem5/resources/client.py | 52 +++- .../resources/client_api/abstract_client.py | 56 +++- .../gem5/resources/client_api/atlasclient.py | 22 +- .../resources/client_api/client_wrapper.py | 81 +++++- .../gem5/resources/client_api/jsonclient.py | 38 ++- src/python/gem5/resources/downloader.py | 271 ++---------------- src/python/gem5/resources/resource.py | 18 +- src/python/gem5/resources/workload.py | 12 +- tests/gem5/configs/download_check.py | 66 +++-- .../resources/pyunit_client_wrapper_checks.py | 109 +++---- ...checks.py => pyunit_json_client_checks.py} | 37 +-- .../pyunit_obtain_resources_check.py | 47 +-- .../pyunit_resource_download_checks.py | 72 ----- .../pyunit_resource_specialization.py | 29 +- .../resources/pyunit_workload_checks.py | 48 +--- .../stdlib/resources/refs/mongo-mock.json | 4 +- .../resources/refs/obtain-resource.json | 8 +- .../refs/resource-specialization.json | 18 +- .../stdlib/resources/refs/resources.json | 24 +- .../refs/workload-checks-custom-workload.json | 16 -- .../resources/refs/workload-checks.json | 20 +- 21 files changed, 445 insertions(+), 603 deletions(-) rename tests/pyunit/stdlib/resources/{pyunit_downloader_checks.py => pyunit_json_client_checks.py} (87%) delete mode 100644 tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py delete mode 100644 tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json diff --git a/src/python/gem5/resources/client.py b/src/python/gem5/resources/client.py index bd473eb038..ab8262bf92 100644 --- a/src/python/gem5/resources/client.py +++ b/src/python/gem5/resources/client.py @@ -31,6 +31,7 @@ from typing import Optional, Dict, List from .client_api.client_wrapper import ClientWrapper from gem5.gem5_default_config import config from m5.util import inform +from _m5 import core def getFileContent(file_path: Path) -> Dict: @@ -49,17 +50,7 @@ def getFileContent(file_path: Path) -> Dict: clientwrapper = None -def get_resource_json_obj( - resource_id, - resource_version: Optional[str] = None, - clients: Optional[List[str]] = None, -) -> Dict: - """ - Get the resource json object from the clients wrapper - :param resource_id: The resource id - :param resource_version: The resource version - :param clients: The list of clients to query - """ +def _get_clientwrapper(): global clientwrapper if clientwrapper is None: # First check if the config file path is provided in the environment variable @@ -78,7 +69,42 @@ def get_resource_json_obj( gem5_config = config inform("Using default config") clientwrapper = ClientWrapper(gem5_config) + return clientwrapper - return clientwrapper.get_resource_json_obj_from_client( - resource_id, resource_version, clients + +def list_resources( + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, +) -> Dict[str, List[str]]: + """ + List all the resources available + + :param clients: The list of clients to query + :param gem5_version: The gem5 version of the resource to get. By default, + it is the gem5 version of the current build. If set to none, it will return + all gem5 versions of the resource. + :return: A Python Dict where the key is the resource id and the value is + a list of all the supported resource versions. + """ + return _get_clientwrapper().list_resources(clients, gem5_version) + + +def get_resource_json_obj( + resource_id, + resource_version: Optional[str] = None, + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, +) -> Dict: + """ + Get the resource json object from the clients wrapper + :param resource_id: The resource id + :param resource_version: The resource version + :param clients: The list of clients to query + :param gem5_version: The gem5 versions to filter the resources based on + compatibility. By default, it is the gem5 version of the current build. + If None, filtering based on compatibility is not performed. + """ + + return _get_clientwrapper().get_resource_json_obj_from_client( + resource_id, resource_version, clients, gem5_version ) diff --git a/src/python/gem5/resources/client_api/abstract_client.py b/src/python/gem5/resources/client_api/abstract_client.py index 74a513fc56..7f8ad6166e 100644 --- a/src/python/gem5/resources/client_api/abstract_client.py +++ b/src/python/gem5/resources/client_api/abstract_client.py @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import ABC, abstractmethod -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import urllib.parse @@ -63,9 +63,61 @@ class AbstractClient(ABC): return False @abstractmethod + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """ + :param resource_id: The ID of the Resource. Optional, if not set, all + resources will be returned. + :param resource_version: The version of the Resource. Optional, if + not set, all resource versions will be returned. Note: If `resource_id` + is not set, this parameter will be ignored. + :param gem5_version: The version of gem5. Optional, if not set, all + versions will be returned. + :return: A list of all the Resources with the given ID. + """ + raise NotImplementedError + + def filter_incompatible_resources( + self, + resources_to_filter: List[Dict[str, Any]], + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Returns a filtered list resources based on gem5 version + compatibility. + + Note: This function assumes if the minor component of + a resource's gem5_version is not specified, the resource is compatible + with all minor versions of the same major version. + Likewise, if no hot-fix component is specified, it is assumed that + the resource is compatible with all hot-fix versions of the same + minor version. + + * '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'. + * '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'. + * '22.3.2.4' would only be compatible with gem5 '22.3.2.4'. + + :param resources_to_filter: The list of resources to filter. + :param gem5_version: The gem5 version in which the filtered resources + should be compatible. If None, no filtering will be done. + : + """ + if not gem5_version: + return resources_to_filter + + filtered_resources = [] + for resource in resources_to_filter: + for version in resource["gem5_versions"]: + if gem5_version.startswith(version): + filtered_resources.append(resource) + return filtered_resources + def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: """ :param resource_id: The ID of the Resource. :return: A list of all the Resources with the given ID. """ - raise NotImplementedError + return self.get_resources(resource_id=resource_id) diff --git a/src/python/gem5/resources/client_api/atlasclient.py b/src/python/gem5/resources/client_api/atlasclient.py index 4a6e5cf691..7d2a27c3f7 100644 --- a/src/python/gem5/resources/client_api/atlasclient.py +++ b/src/python/gem5/resources/client_api/atlasclient.py @@ -64,14 +64,26 @@ class AtlasClient(AbstractClient): token = result["access_token"] return token - def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: url = f"{self.url}/action/find" data = { "dataSource": self.dataSource, "collection": self.collection, "database": self.database, - "filter": {"id": resource_id}, } + filter = {} + if resource_id: + filter["id"] = resource_id + if resource_version is not None: + filter["resource_version"] = resource_version + + if filter: + data["filter"] = filter data = json.dumps(data).encode("utf-8") headers = { @@ -88,4 +100,8 @@ class AtlasClient(AbstractClient): result = json.loads(response.read().decode("utf-8")) resources = result["documents"] - return resources + # I do this as a lazy post-processing step because I can't figure out + # how to do this via an Atlas query, which may be more efficient. + return self.filter_incompatible_resources( + resources_to_filter=resources, gem5_version=gem5_version + ) diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py index 74ee831c1f..ebf1b8ec2b 100644 --- a/src/python/gem5/resources/client_api/client_wrapper.py +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -59,6 +59,38 @@ class ClientWrapper: warn(f"Error creating client {client}: {str(e)}") return clients + def list_resources( + self, + clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, + ) -> Dict[str, List[str]]: + + clients_to_search = ( + list(self.clients.keys()) if clients is None else clients + ) + # There's some duplications of functionality here (similar code in + # `get_all_resources_by_id`. This code could be refactored to avoid + # this). + resources = [] + for client in clients_to_search: + if client not in self.clients: + raise Exception(f"Client: {client} does not exist") + try: + resources.extend( + self.clients[client].get_resources( + gem5_version=gem5_version + ) + ) + except Exception as e: + warn(f"Error getting resources from client {client}: {str(e)}") + + to_return = {} + for resource in resources: + if resource["id"] not in to_return: + to_return[resource["id"]] = [] + to_return[resource["id"]].append(resource["resource_version"]) + return to_return + def get_all_resources_by_id( self, resource_id: str, @@ -98,6 +130,7 @@ class ClientWrapper: resource_id: str, resource_version: Optional[str] = None, clients: Optional[List[str]] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> Dict: """ This function returns the resource object from the client with the @@ -106,6 +139,9 @@ class ClientWrapper: :param resource_version: The version of the resource to search for. :param clients: A list of clients to search through. If None, all clients are searched. + :param gem5_version: The gem5 version to check compatibility with. If + None, no compatibility check is performed. By default, is the current + version of gem5. :return: The resource object as a Python dictionary if found. If not found, exception is thrown. """ @@ -124,7 +160,9 @@ class ClientWrapper: else: compatible_resources = ( - self._get_resources_compatible_with_gem5_version(resources) + self._get_resources_compatible_with_gem5_version( + resources, gem5_version=gem5_version + ) ) if len(compatible_resources) == 0: resource_to_return = self._sort_resources(resources)[0] @@ -133,7 +171,10 @@ class ClientWrapper: compatible_resources )[0] - self._check_resource_version_compatibility(resource_to_return) + if gem5_version: + self._check_resource_version_compatibility( + resource_to_return, gem5_version=gem5_version + ) return resource_to_return @@ -172,16 +213,31 @@ class ClientWrapper: ) -> List: """ Returns a list of compatible resources with the current gem5 version. + + Note: This function assumes if the minor component of + a resource's gem5_version is not specified, it that the + resource is compatible all minor versions of the same major version. + Likewise, if no hot-fix component is specified, it is assumed that + the resource is compatible with all hot-fix versions of the same + minor version. + + * '20.1' would be compatible with gem5 '20.1.1.0' and '20.1.2.0'. + * '21.5.2' would be compatible with gem5 '21.5.2.0' and '21.5.2.0'. + * '22.3.2.4' would only be compatible with gem5 '22.3.2.4'. + :param resources: A list of resources to filter. :return: A list of compatible resources as Python dictionaries. - If no compatible resources are found, the original list of resources - is returned. + + **Note**: This is a big duplication of code. This functionality already + exists in the `AbstractClient` class. This code should be refactored + to avoid this duplication. """ - compatible_resources = [ - resource - for resource in resources - if gem5_version in resource["gem5_versions"] - ] + + compatible_resources = [] + for resource in resources: + for version in resource["gem5_versions"]: + if gem5_version.startswith(version): + compatible_resources.append(resource) return compatible_resources def _sort_resources(self, resources: List) -> List: @@ -213,7 +269,12 @@ class ClientWrapper: """ if not resource: return False - if gem5_version not in resource["gem5_versions"]: + if ( + gem5_version + and not self._get_resources_compatible_with_gem5_version( + [resource], gem5_version=gem5_version + ) + ): warn( f"Resource {resource['id']} with version " f"{resource['resource_version']} is not known to be compatible" diff --git a/src/python/gem5/resources/client_api/jsonclient.py b/src/python/gem5/resources/client_api/jsonclient.py index 225126e2a8..9e837131b0 100644 --- a/src/python/gem5/resources/client_api/jsonclient.py +++ b/src/python/gem5/resources/client_api/jsonclient.py @@ -58,13 +58,31 @@ class JSONClient(AbstractClient): ) self.resources = json.loads(response.read().decode("utf-8")) - def get_resources_by_id(self, resource_id: str) -> List[Dict[str, Any]]: - """ - :param resource_id: The ID of the Resource. - :return: A list of all the Resources with the given ID. - """ - return [ - resource - for resource in self.resources - if resource["id"] == resource_id - ] + def get_resources_json(self) -> List[Dict[str, Any]]: + """Returns a JSON representation of the resources.""" + return self.resources + + def get_resources( + self, + resource_id: Optional[str] = None, + resource_version: Optional[str] = None, + gem5_version: Optional[str] = None, + ) -> List[Dict[str, Any]]: + filter = self.resources # Unfiltered. + if resource_id: + filter = [ # Filter by resource_id. + resource + for resource in filter + if resource["id"] == resource_id + ] + if resource_version: + filter = [ # Filter by resource_version. + resource + for resource in filter + if resource["resource_version"] == resource_version + ] + + # Filter by gem5_version. + return self.filter_incompatible_resources( + resources_to_filter=filter, gem5_version=gem5_version + ) diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index 0781d9b15a..bb5ca84cc0 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -24,24 +24,24 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json import urllib.request import urllib.parse -import hashlib import os import shutil import gzip -import hashlib -import base64 import time import random from pathlib import Path import tarfile -from tempfile import gettempdir from urllib.error import HTTPError -from typing import List, Dict, Set, Optional +from typing import List, Optional, Dict -from .client import get_resource_json_obj +from _m5 import core + +from .client import ( + get_resource_json_obj, + list_resources as client_list_resources, +) from .md5_utils import md5_file, md5_dir from ..utils.progress_bar import tqdm, progress_hook @@ -53,188 +53,6 @@ information about resources from resources.gem5.org. """ -def _resources_json_version_required() -> str: - """ - Specifies the version of resources.json to obtain. - """ - return "develop" - - -def _get_resources_json_uri() -> str: - return "https://resources.gem5.org/resources.json" - - -def _url_validator(url): - try: - result = urllib.parse.urlparse(url) - return all([result.scheme, result.netloc, result.path]) - except: - return False - - -def _get_resources_json_at_path(path: str, use_caching: bool = True) -> Dict: - """ - Returns a resource JSON, in the form of a Python Dict. The location - of the JSON must be specified. - - If `use_caching` is True, and a URL is passed, a copy of the JSON will be - cached locally, and used for up to an hour after retrieval. - - :param path: The URL or local path of the JSON file. - :param use_caching: True if a cached file is to be used (up to an hour), - otherwise the file will be retrieved from the URL regardless. True by - default. Only valid in cases where a URL is passed. - """ - - # If a local valid path is passed, just load it. - if Path(path).is_file(): - return json.load(open(path)) - - # If it's not a local path, it should be a URL. We check this here and - # raise an Exception if it's not. - if not _url_validator(path): - raise Exception( - f"Resources location '{path}' is not a valid path or URL." - ) - - download_path = os.path.join( - gettempdir(), - f"gem5-resources-{hashlib.md5(path.encode()).hexdigest()}" - f"-{str(os.getuid())}.json", - ) - - # We apply a lock on the resources file for when it's downloaded, or - # re-downloaded, and read. This stops a corner-case from occuring where - # the file is re-downloaded while being read by another gem5 thread. - # Note the timeout is 120 so the `_download` function is given time to run - # its Truncated Exponential Backoff algorithm - # (maximum of roughly 1 minute). Typically this code will run quickly. - with FileLock(f"{download_path}.lock", timeout=120): - - # The resources.json file can change at any time, but to avoid - # excessive retrieval we cache a version locally and use it for up to - # an hour before obtaining a fresh copy. - # - # `time.time()` and `os.path.getmtime(..)` both return an unix epoch - # time in seconds. Therefore, the value of "3600" here represents an - # hour difference between the two values. `time.time()` gets the - # current time, and `os.path.getmtime()` gets the modification - # time of the file. This is the most portable solution as other ideas, - # like "file creation time", are not always the same concept between - # operating systems. - if ( - not use_caching - or not os.path.exists(download_path) - or (time.time() - os.path.getmtime(download_path)) > 3600 - ): - _download(path, download_path) - - with open(download_path) as f: - file_contents = f.read() - - try: - to_return = json.loads(file_contents) - except json.JSONDecodeError: - # This is a bit of a hack. If the URL specified exists in a Google - # Source repo (which is the case when on the gem5 develop branch) we - # retrieve the JSON in base64 format. This cannot be loaded directly as - # text. Conversion is therefore needed. - to_return = json.loads(base64.b64decode(file_contents).decode("utf-8")) - - return to_return - - -def _get_resources_json() -> Dict: - """ - Gets the Resources JSON. - - :returns: The Resources JSON (as a Python Dictionary). - """ - - path = os.getenv("GEM5_RESOURCE_JSON", _get_resources_json_uri()) - to_return = _get_resources_json_at_path(path=path) - - # If the current version pulled is not correct, look up the - # "previous-versions" field to find the correct one. - # If the resource JSON file does not have a "version" field or it's - # null/None, then we will use this resource JSON file (this is usefull for - # testing purposes). - version = _resources_json_version_required() - json_version = None if "version" not in to_return else to_return["version"] - - if json_version and json_version != version: - if version in to_return["previous-versions"].keys(): - to_return = _get_resources_json_at_path( - path=to_return["previous-versions"][version] - ) - else: - # This should never happen, but we thrown an exception to explain - # that we can't find the version. - raise Exception( - f"Version '{version}' of resources.json cannot be found." - ) - - return to_return - - -def _get_url_base() -> str: - """ - Obtains the "url_base" string from the resources.json file. - - :returns: The "url_base" string value from the resources.json file. - """ - json = _get_resources_json() - if "url_base" in json.keys(): - return json["url_base"] - return "" - - -def _get_resources( - valid_types: Set[str], resources_group: Optional[Dict] = None -) -> Dict[str, Dict]: - """ - A recursive function to get all the workload/resource of the specified type - in the resources.json file. - - :param valid_types: The type to return (i.e., "resource" or "workload). - :param resource_group: Used for recursion: The current resource group being - iterated through. - - :returns: A dictionary of artifact names to the resource JSON objects. - """ - - if resources_group is None: - resources_group = _get_resources_json()["resources"] - - to_return = {} - for resource in resources_group: - if resource["type"] in valid_types: - # If the type is valid then we add it directly to the map - # after a check that the name is unique. - if resource["name"] in to_return.keys(): - raise Exception( - f"Error: Duplicate resource with name '{resource['name']}'." - ) - to_return[resource["name"]] = resource - elif resource["type"] == "group": - # If it's a group we get recursive. We then check to see if there - # are any duplication of keys. - new_map = _get_resources( - valid_types=valid_types, resources_group=resource["contents"] - ) - intersection = set(new_map.keys()).intersection(to_return.keys()) - if len(intersection) > 0: - # Note: if this error is received it's likely an error with - # the resources.json file. The resources names need to be - # unique keyes. - raise Exception( - f"Error: Duplicate resources with names: {str(intersection)}." - ) - to_return.update(new_map) - - return to_return - - def _download(url: str, download_to: str, max_attempts: int = 6) -> None: """ Downloads a file. @@ -336,61 +154,26 @@ def _download(url: str, download_to: str, max_attempts: int = 6) -> None: ) -def list_resources() -> List[str]: +def list_resources( + clients: Optional[List] = None, gem5_version: Optional[str] = None +) -> Dict[str, List[str]]: """ - Lists all available resources by name. + Lists all available resources. Returns a dictionary where the key is the + id of the resources and the value is a list of that resource's versions. + + :param clients: A list of clients to use when listing resources. If None, + all clients will be used. None by default. + + :param gem5_version: The gem5 version to which all resources should be + compatible with. If None, compatibility of resources is not considered and + all resources will be returned. + + **Note**: This function is here for legacy reasons. The `list_resources` + function was originally stored here. In order to remain backwards + compatible, this function will call the `client_list_resources` function - :returns: A list of resources by name. """ - from .resource import _get_resource_json_type_map - - return _get_resources( - valid_types=_get_resource_json_type_map.keys() - ).keys() - - -def get_workload_json_obj(workload_name: str) -> Dict: - """ - Get a JSON object of a specified workload. - - :param workload_name: The name of the workload. - - :raises Exception: An exception is raised if the specified workload does - not exit. - """ - workload_map = _get_resources(valid_types={"workload"}) - - if workload_name not in workload_map: - raise Exception( - f"Error: Workload with name {workload_name} does not exist" - ) - - return workload_map[workload_name] - - -def get_resources_json_obj(resource_name: str) -> Dict: - """ - Get a JSON object of a specified resource. - - :param resource_name: The name of the resource. - - :returns: The JSON object (in the form of a dictionary). - - :raises Exception: An exception is raised if the specified resources does - not exist. - """ - from .resource import _get_resource_json_type_map - - resource_map = _get_resources( - valid_types=_get_resource_json_type_map.keys() - ) - - if resource_name not in resource_map: - raise Exception( - f"Error: Resource with name '{resource_name}' does not exist" - ) - - return resource_map[resource_name] + return client_list_resources(clients=clients, gem5_version=gem5_version) def get_resource( @@ -401,6 +184,7 @@ def get_resource( download_md5_mismatch: bool = True, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> None: """ Obtains a gem5 resource and stored it to a specified location. If the @@ -429,6 +213,10 @@ def get_resource( :param clients: A list of clients to use when obtaining the resource. If None, all clients will be used. None by default. + :param gem5_version: The gem5 version to use when obtaining the resource. + By default, the version of gem5 being used is used. This is used primarily + for testing purposes. + :raises Exception: An exception is thrown if a file is already present at `to_path` but it does not have the correct md5 sum. An exception will also be thrown is a directory is present at `to_path` @@ -444,6 +232,7 @@ def get_resource( resource_name, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) if os.path.exists(to_path): diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 22adf15670..bc9f4480ba 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -28,6 +28,7 @@ from abc import ABCMeta import os from pathlib import Path from m5.util import warn, fatal +from _m5 import core from .downloader import get_resource @@ -559,17 +560,15 @@ def obtain_resource( download_md5_mismatch: bool = True, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version=core.gem5Version, ) -> AbstractResource: """ This function primarily serves as a factory for resources. It will return the correct `AbstractResource` implementation based on the resource - requested, by referencing the "resource.json" file (by default, that hosted - at https://resources.gem5.org/resources.json). In addition to this, this - function will download the resource if not detected in the - `resource_directory`. + requested. :param resource_name: The name of the gem5 resource as it appears under the - "name" field in the `resource.json` file. + "id" field in the `resource.json` file. :param resource_directory: The location of the directory in which the resource is to be stored. If this parameter is not set, it will set to the environment variable `GEM5_RESOURCE_DIR`. If the environment is not @@ -582,11 +581,17 @@ def obtain_resource( Not a required parameter. None by default. :param clients: A list of clients to search for the resource. If this parameter is not set, it will default search all clients. + :param gem5_version: The gem5 version to use to filter incompatible + resource versions. By default set to the current gem5 version. If None, + this filtering is not performed. """ # Obtain the resource object entry for this resource resource_json = get_resource_json_obj( - resource_id, resource_version=resource_version, clients=clients + resource_id, + resource_version=resource_version, + clients=clients, + gem5_version=gem5_version, ) to_path = None @@ -629,6 +634,7 @@ def obtain_resource( download_md5_mismatch=download_md5_mismatch, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) # Obtain the type from the JSON. From this we will determine what subclass diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index 148ab3f35a..0798b891ab 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -27,6 +27,8 @@ from .resource import obtain_resource from .client import get_resource_json_obj +from _m5 import core + from typing import Dict, Any, List, Optional @@ -160,6 +162,7 @@ class Workload(AbstractWorkload): resource_directory: Optional[str] = None, resource_version: Optional[str] = None, clients: Optional[List] = None, + gem5_version: Optional[str] = core.gem5Version, ) -> None: """ This constructor will load the workload details from the workload with @@ -201,12 +204,17 @@ class Workload(AbstractWorkload): :param resource_directory: An optional parameter that specifies where any resources should be download and accessed from. If None, a default location will be used. None by default. + :param gem5_version: The gem5 version for the Workload to be loaded. + By default, the current gem5 version is used. This will filter + resources which are incompatible with the current gem5 version. If + None, no filtering will be done. """ workload_json = get_resource_json_obj( workload_name, resource_version=resource_version, clients=clients, + gem5_version=gem5_version, ) func = workload_json["function"] @@ -219,7 +227,9 @@ class Workload(AbstractWorkload): value = workload_json["resources"][key] assert isinstance(value, str) params[key] = obtain_resource( - value, resource_directory=resource_directory + value, + resource_directory=resource_directory, + gem5_version=gem5_version, ) if "additional_params" in workload_json: diff --git a/tests/gem5/configs/download_check.py b/tests/gem5/configs/download_check.py index decc62c2d7..2180f4f26a 100644 --- a/tests/gem5/configs/download_check.py +++ b/tests/gem5/configs/download_check.py @@ -26,10 +26,11 @@ from gem5.resources.downloader import ( list_resources, - get_resources_json_obj, get_resource, ) +from gem5.resources.client import get_resource_json_obj + from gem5.resources.md5_utils import md5 import os @@ -51,6 +52,15 @@ parser.add_argument( "checked", ) +parser.add_argument( + "--gem5-version", + type=str, + required=False, + help="The gem5 version to check the resources against. Resources not " + "compatible with this version will be ignored. If not set, no " + "compatibility tests are performed.", +) + parser.add_argument( "--download-directory", type=str, @@ -67,39 +77,59 @@ if not Path(args.download_directory).exists(): ids = args.ids +resource_list = list_resources(gem5_version=args.gem5_version) if len(ids) == 0: - ids = list_resources() + ids = resource_list # We log all the errors as they occur then dump them at the end. This means we # can be aware of all download errors in a single failure. errors = str() for id in ids: - if id not in list_resources(): + if id not in resource_list: errors += ( f"Resource with ID '{id}' not found in " + f"`list_resources()`.{os.linesep}" ) continue - resource_json = get_resources_json_obj(id) - download_path = os.path.join(args.download_directory, id) - try: - get_resource(resource_name=id, to_path=download_path) - except Exception as e: - errors += f"Failure to download resource '{id}'.{os.linesep}" - errors += f"Exception message:{os.linesep}{str(e)}" - errors += f"{os.linesep}{os.linesep}" - continue + for resource_version in ids[id]: - if md5(Path(download_path)) != resource_json["md5sum"]: - errors += ( - f"Downloaded resource '{id}' md5 " - + f"({md5(Path(download_path))}) differs to that in the " - + f"JSON ({resource_json['md5sum']}).{os.linesep}" + resource_json = get_resource_json_obj( + resource_id=id, + resource_version=resource_version, + gem5_version=args.gem5_version, ) + if resource_json["category"] == "workload": + # Workloads are not downloaded as part of this test. + continue + download_path = os.path.join( + args.download_directory, f"{id}-v{resource_version}" + ) + try: + get_resource( + resource_name=id, + resource_version=resource_version, + gem5_version=args.gem5_version, + to_path=download_path, + ) + except Exception as e: + errors += ( + f"Failure to download resource '{id}', " + + f"v{resource_version}.{os.linesep}" + ) + errors += f"Exception message:{os.linesep}{str(e)}" + errors += f"{os.linesep}{os.linesep}" + continue - # Remove the downloaded resource. + if md5(Path(download_path)) != resource_json["md5sum"]: + errors += ( + f"Downloaded resource '{id}' md5 " + + f"({md5(Path(download_path))}) differs to that recorded in " + + f" gem5-resources ({resource_json['md5sum']}).{os.linesep}" + ) + + # Remove the downloaded resource. shutil.rmtree(download_path, ignore_errors=True) # If errors exist, raise an exception highlighting them. diff --git a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py index 344f67b8b0..f190b1ed5f 100644 --- a/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_client_wrapper_checks.py @@ -25,13 +25,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest -from gem5.isas import ISA from gem5.resources.client import get_resource_json_obj -import gem5.resources.client from gem5.resources.client_api.client_wrapper import ClientWrapper -from typing import Dict from unittest.mock import patch -from unittest import mock import json from urllib.error import HTTPError import io @@ -62,23 +58,8 @@ mock_config_mongo = { }, } -mock_config_combined = { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - }, - "baba": { - "url": mock_json_path, - "isMongo": False, - }, - }, -} +mock_config_combined = mock_config_mongo +mock_config_combined["sources"]["baba"] = mock_config_json["sources"]["baba"] mock_json = {} @@ -145,12 +126,12 @@ class ClientWrapperTestSuite(unittest.TestCase): def test_get_resource_json_obj(self): # Test that the resource object is correctly returned resource = "this-is-a-test-resource" - resource = get_resource_json_obj(resource) + resource = get_resource_json_obj(resource, gem5_version="develop") self.assertEqual(resource["id"], "this-is-a-test-resource") - self.assertEqual(resource["resource_version"], "2.0.0") + self.assertEqual(resource["resource_version"], "1.1.0") self.assertEqual(resource["category"], "binary") self.assertEqual( - resource["description"], "This is a test resource but double newer" + resource["description"], "This is a test resource but newer" ) self.assertEqual( resource["source_url"], @@ -167,7 +148,9 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id = "test-id" client = "invalid" with self.assertRaises(Exception) as context: - get_resource_json_obj(resource_id, clients=[client]) + get_resource_json_obj( + resource_id, clients=[client], gem5_version="develop" + ) self.assertTrue( f"Client: {client} does not exist" in str(context.exception) ) @@ -181,7 +164,9 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id = "this-is-a-test-resource" resource_version = "1.0.0" resource = get_resource_json_obj( - resource_id, resource_version=resource_version + resource_id, + resource_version=resource_version, + gem5_version="develop", ) self.assertEqual(resource["id"], "this-is-a-test-resource") self.assertEqual(resource["resource_version"], "1.0.0") @@ -200,17 +185,18 @@ class ClientWrapperTestSuite(unittest.TestCase): @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_get_resource_json_obj_1(self, mock_get): resource = "x86-ubuntu-18.04-img" - resource = get_resource_json_obj(resource) + resource = get_resource_json_obj(resource, gem5_version="develop") self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") - self.assertEqual(resource["resource_version"], "1.1.0") + self.assertEqual(resource["resource_version"], "2.0.0") self.assertEqual(resource["category"], "disk-image") self.assertEqual( resource["description"], - "A disk image containing Ubuntu 18.04 for x86. This image will run an `m5 readfile` instruction after booting. If no script file is specified an `m5 exit` instruction will be executed.", + "This is a test resource", ) self.assertEqual( resource["source_url"], - "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "https://github.com/gem5/gem5-resources/tree/develop/" + "src/x86-ubuntu", ) self.assertEqual(resource["architecture"], "X86") @@ -227,6 +213,7 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id, resource_version=resource_version, clients=["gem5-resources"], + gem5_version="develop", ) self.assertEqual(resource["id"], "x86-ubuntu-18.04-img") self.assertEqual(resource["resource_version"], "1.0.0") @@ -246,7 +233,9 @@ class ClientWrapperTestSuite(unittest.TestCase): def test_get_resource_json_obj_with_id_invalid_mongodb(self, mock_get): resource_id = "invalid-id" with self.assertRaises(Exception) as context: - get_resource_json_obj(resource_id, clients=["gem5-resources"]) + get_resource_json_obj( + resource_id, clients=["gem5-resources"], gem5_version="develop" + ) self.assertTrue( "Resource with ID 'invalid-id' not found." in str(context.exception) @@ -267,12 +256,13 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id, resource_version=resource_version, clients=["gem5-resources"], + gem5_version="develop", ) self.assertTrue( f"Resource x86-ubuntu-18.04-img with version '2.5.0'" " not found.\nResource versions can be found at: " - f"https://resources.gem5.org/resources/x86-ubuntu-18.04-img/versions" - in str(context.exception) + "https://resources.gem5.org/resources/x86-ubuntu-18.04-img/" + "versions" in str(context.exception) ) @patch( @@ -286,12 +276,13 @@ class ClientWrapperTestSuite(unittest.TestCase): get_resource_json_obj( resource_id, resource_version=resource_version, + gem5_version="develop", ) self.assertTrue( - f"Resource this-is-a-test-resource with version '2.5.0'" + "Resource this-is-a-test-resource with version '2.5.0'" " not found.\nResource versions can be found at: " - f"https://resources.gem5.org/resources/this-is-a-test-resource/versions" - in str(context.exception) + "https://resources.gem5.org/resources/this-is-a-test-resource/" + "versions" in str(context.exception) ) @patch( @@ -308,11 +299,13 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id_mongo, resource_version=resource_version_mongo, clients=["gem5-resources"], + gem5_version="develop", ) resource_json = get_resource_json_obj( resource_id_json, resource_version=resource_version_json, clients=["baba"], + gem5_version="develop", ) self.assertEqual(resource_mongo["id"], "x86-ubuntu-18.04-img") self.assertEqual(resource_mongo["resource_version"], "1.0.0") @@ -322,7 +315,8 @@ class ClientWrapperTestSuite(unittest.TestCase): ) self.assertEqual( resource_mongo["source_url"], - "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", + "https://github.com/gem5/gem5-resources/tree/develop/src/" + "x86-ubuntu", ) self.assertEqual(resource_mongo["architecture"], "X86") @@ -347,6 +341,7 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id = "simpoint-resource" resource = get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertEqual(resource["id"], resource_id) self.assertEqual(resource["resource_version"], "0.2.0") @@ -371,6 +366,7 @@ class ClientWrapperTestSuite(unittest.TestCase): resource_id = "x86-ubuntu-18.04-img" resource_json = get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") @@ -378,8 +374,7 @@ class ClientWrapperTestSuite(unittest.TestCase): self.assertEqual(resource_json["category"], "disk-image") resource_json = get_resource_json_obj( - resource_id, - resource_version="1.0.0", + resource_id, resource_version="1.0.0", gem5_version="develop" ) self.assertEqual(resource_json["id"], "x86-ubuntu-18.04-img") @@ -396,6 +391,7 @@ class ClientWrapperTestSuite(unittest.TestCase): with self.assertRaises(Exception) as context: get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( f"Resource {resource_id} has multiple resources with" @@ -428,6 +424,7 @@ class ClientWrapperTestSuite(unittest.TestCase): with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" @@ -440,21 +437,7 @@ class ClientWrapperTestSuite(unittest.TestCase): @patch( "gem5.resources.client.clientwrapper", - ClientWrapper( - { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - } - }, - } - ), + ClientWrapper(mock_config_mongo), ) @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_invalid_url(self, mock_get): @@ -464,6 +447,7 @@ class ClientWrapperTestSuite(unittest.TestCase): with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" @@ -476,21 +460,7 @@ class ClientWrapperTestSuite(unittest.TestCase): @patch( "gem5.resources.client.clientwrapper", - ClientWrapper( - { - "sources": { - "gem5-resources": { - "dataSource": "gem5-vision", - "database": "gem5-vision", - "collection": "versions_test", - "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", - "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", - "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", - "isMongo": True, - } - }, - } - ), + ClientWrapper(mock_config_mongo), ) @patch("urllib.request.urlopen", side_effect=mocked_requests_post) def test_invalid_url(self, mock_get): @@ -500,6 +470,7 @@ class ClientWrapperTestSuite(unittest.TestCase): with contextlib.redirect_stderr(f): get_resource_json_obj( resource_id, + gem5_version="develop", ) self.assertTrue( "Error getting resources from client gem5-resources:" diff --git a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py b/tests/pyunit/stdlib/resources/pyunit_json_client_checks.py similarity index 87% rename from tests/pyunit/stdlib/resources/pyunit_downloader_checks.py rename to tests/pyunit/stdlib/resources/pyunit_json_client_checks.py index 19169e480e..88db3d4967 100644 --- a/tests/pyunit/stdlib/resources/pyunit_downloader_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_json_client_checks.py @@ -30,15 +30,11 @@ import os from typing import Dict import json -from gem5.resources.downloader import ( - _get_resources_json_at_path, - _get_resources_json, - _resources_json_version_required, -) +from gem5.resources.client_api.jsonclient import JSONClient -class ResourceDownloaderTestSuite(unittest.TestCase): - """Test cases for gem5.resources.downloader""" +class JSONClientTestSuite(unittest.TestCase): + """Test cases for gem5.resources.client_api.jsonclient""" @classmethod def setUpClass(cls) -> str: @@ -142,12 +138,9 @@ class ResourceDownloaderTestSuite(unittest.TestCase): file.close() cls.file_path = file.name - os.environ["GEM5_RESOURCE_JSON"] = cls.file_path - @classmethod def tearDownClass(cls) -> None: os.remove(cls.file_path) - del os.environ["GEM5_RESOURCE_JSON"] def verify_json(self, json: Dict) -> None: """ @@ -167,32 +160,22 @@ class ResourceDownloaderTestSuite(unittest.TestCase): self.assertEquals("test-version", json[3]["id"]) def test_get_resources_json_at_path(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json_at_path() - # function. + # Tests JSONClient.get_resources_json() - json = _get_resources_json_at_path(path=self.file_path) - self.verify_json(json=json) - - def test_get_resources_json(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json() function. - - json = _get_resources_json() + client = JSONClient(path=self.file_path) + json = client.get_resources_json() self.verify_json(json=json) def test_get_resources_json_invalid_url(self) -> None: - # Tests the gem5.resources.downloader._get_resources_json() function in - # case where an invalid url is passed as the URL/PATH of the - # resources.json file. + # Tests the JSONClient.get_resources_json() function in case where an + # invalid url is passed as the URL/PATH of the resources JSON file. path = "NotAURLorFilePath" - os.environ["GEM5_RESOURCE_JSON"] = path with self.assertRaises(Exception) as context: - _get_resources_json() + client = JSONClient(path=path) + json = client.get_resources_json() self.assertTrue( f"Resources location '{path}' is not a valid path or URL." in str(context.exception) ) - - # Set back to the old path - os.environ["GEM5_RESOURCE_JSON"] = self.file_path diff --git a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py index 791d96c1f1..b1eda4e6ed 100644 --- a/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py +++ b/tests/pyunit/stdlib/resources/pyunit_obtain_resources_check.py @@ -30,12 +30,7 @@ import io import contextlib from pathlib import Path -from gem5.resources.resource import * - -from gem5.resources.looppoint import ( - LooppointCsvLoader, - LooppointJsonLoader, -) +from gem5.resources.resource import obtain_resource, BinaryResource from gem5.isas import ISA @@ -61,24 +56,6 @@ mock_config_json = { new=ClientWrapper(mock_config_json), ) class TestObtainResourcesCheck(unittest.TestCase): - @classmethod - def setUpClass(cls): - """Prior to running the suite we set the resource directory to - "ref/resource-specialization.json" - """ - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "obtain-resource.json", - ) - - @classmethod - def tearDownClass(cls) -> None: - """After running the suite we unset the gem5-resource JSON file, as to - not interfere with others tests. - """ - del os.environ["GEM5_RESOURCE_JSON"] - def get_resource_dir(cls) -> str: """To ensure the resources are cached to the same directory as all other tests, this function returns the location of the testing @@ -99,26 +76,27 @@ class TestObtainResourcesCheck(unittest.TestCase): resource = obtain_resource( resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) - self.assertEquals("2.5.0", resource.get_resource_version()) + self.assertEquals("1.7.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) - # self.assertIn(gem5Version, resource.get_gem5_versions()) - self.assertEquals("test description", resource.get_description()) + self.assertEquals( + "test description v1.7.0", resource.get_description() + ) self.assertEquals("src/test-source", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) def test_obtain_resources_with_version_compatible(self): - gem5Version = core.gem5Version resource = obtain_resource( resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), - resource_version="1.7.0", + resource_version="1.5.0", + gem5_version="develop", ) - self.assertEquals("1.7.0", resource.get_resource_version()) + self.assertEquals("1.5.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) - # self.assertIn(gem5Version, resource.get_gem5_versions()) self.assertEquals( - "test description v1.7.0", resource.get_description() + "test description for 1.5.0", resource.get_description() ) self.assertEquals("src/test-source", resource.get_source()) self.assertEquals(ISA.ARM, resource.get_architecture()) @@ -143,6 +121,7 @@ class TestObtainResourcesCheck(unittest.TestCase): resource_id="test-binary-resource", resource_directory=self.get_resource_dir(), resource_version="1.5.0", + gem5_version="develop", ) self.assertEquals("1.5.0", resource.get_resource_version()) self.assertIsInstance(resource, BinaryResource) @@ -157,6 +136,7 @@ class TestObtainResourcesCheck(unittest.TestCase): obtain_resource( resource_id="invalid-id", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertTrue( "Resource with ID 'invalid-id' not found." @@ -169,6 +149,7 @@ class TestObtainResourcesCheck(unittest.TestCase): resource_id="invalid-id", resource_directory=self.get_resource_dir(), resource_version="1.7.0", + gem5_version="develop", ) self.assertTrue( "Resource with ID 'invalid-id' not found." @@ -182,8 +163,6 @@ class TestObtainResourcesCheck(unittest.TestCase): resource_directory=self.get_resource_dir(), resource_version="3.0.0", ) - print("context.exception: ", context.exception) - print(str(context.exception)) self.assertTrue( f"Resource test-binary-resource with version '3.0.0'" " not found.\nResource versions can be found at: " diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py b/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py deleted file mode 100644 index 8f6674ff0d..0000000000 --- a/tests/pyunit/stdlib/resources/pyunit_resource_download_checks.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2023 The Regents of the University of California -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer; -# redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution; -# neither the name of the copyright holders nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import unittest -import tempfile -import os -from typing import Dict - -from gem5.resources.downloader import ( - get_resources_json_obj, -) - - -class ResourceDownloadTestSuite(unittest.TestCase): - """Test cases for gem5.resources.downloader""" - - @classmethod - def setUpClass(cls) -> str: - pass - - def get_resource_json_by_id(self) -> None: - """Get a resource by its id""" - resources = get_resources_json_obj("test-version") - self.assertEqual(resources["id"], "test-version") - self.assertEqual(resources["resource_version"], "2.0.0") - - def get_resource_json_invalid_id(self) -> None: - """Should throw an exception when trying to get a resource that doesn't exist""" - with self.assertRaises(Exception) as context: - get_resources_json_obj("this-resource-doesnt-exist") - self.assertTrue( - f"Error: Resource with name 'this-resource-doesnt-exist' does not exist" - in str(context.exception) - ) - - def get_resource_json_by_id_and_version(self) -> None: - """Get a resource by its id and version""" - resources = get_resources_json_obj("test-version", "1.0.0") - self.assertEqual(resources["id"], "test-version") - self.assertEqual(resources["resource_version"], "1.0.0") - - def get_resource_json_by_id_and_invalid_version(self) -> None: - """Get a resource by its id and an invalid version (does not exist)""" - with self.assertRaises(Exception) as context: - get_resources_json_obj("test-version", "3.0.0") - self.assertTrue( - f"Specified Version 3.0.0 does not exist for the resource 'test-version'." - in str(context.exception) - ) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index 5c22a7341e..f2088db8ef 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -62,24 +62,6 @@ class ResourceSpecializationSuite(unittest.TestCase): function. """ - @classmethod - def setUpClass(cls): - """Prior to running the suite we set the resource directory to - "ref/resource-specialization.json" - """ - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "resource-specialization.json", - ) - - @classmethod - def tearDownClass(cls) -> None: - """After running the suite we unset the gem5-resource JSON file, as to - not interfere with others tests. - """ - del os.environ["GEM5_RESOURCE_JSON"] - def get_resource_dir(cls) -> str: """To ensure the resources are cached to the same directory as all other tests, this function returns the location of the testing @@ -99,6 +81,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="binary-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, BinaryResource) @@ -114,6 +97,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="kernel-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, KernelResource) @@ -129,6 +113,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="bootloader-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, BootloaderResource) @@ -144,6 +129,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="disk-image-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, DiskImageResource) @@ -159,6 +145,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="checkpoint-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, CheckpointResource) @@ -173,6 +160,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="git-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, GitResource) @@ -185,6 +173,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="simpoint-directory-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, SimpointDirectoryResource) @@ -219,6 +208,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="simpoint-example", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, SimpointResource) @@ -240,6 +230,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource_id="file-example", resource_directory=self.get_resource_dir(), resource_version="1.0.0", + gem5_version="develop", ) self.assertIsInstance(resource, FileResource) @@ -268,6 +259,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource = obtain_resource( resource_id="looppoint-pinpoint-csv-resource", resource_directory=self.get_resource_dir(), + gem5_version="develop", ) self.assertIsInstance(resource, LooppointCsvResource) @@ -289,6 +281,7 @@ class ResourceSpecializationSuite(unittest.TestCase): resource_id="looppoint-json-restore-resource-region-1", resource_directory=self.get_resource_dir(), resource_version="1.0.0", + gem5_version="develop", ) self.assertIsInstance(resource, LooppointJsonResource) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index b898faeb79..b59e09d4fe 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -40,17 +40,7 @@ from gem5.resources.client_api.client_wrapper import ClientWrapper from unittest.mock import patch from pathlib import Path -mock_config_json1 = { - "sources": { - "baba": { - "url": Path(__file__).parent - / "refs/workload-checks-custom-workload.json", - "isMongo": False, - } - }, -} - -mock_config_json2 = { +mock_config_json = { "sources": { "baba": { "url": Path(__file__).parent / "refs/workload-checks.json", @@ -68,29 +58,19 @@ class CustomWorkloadTestSuite(unittest.TestCase): @classmethod @patch( "gem5.resources.client.clientwrapper", - new=ClientWrapper(mock_config_json1), + new=ClientWrapper(mock_config_json), ) def setUpClass(cls) -> None: - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "workload-checks-custom-workload.json", - ) - cls.custom_workload = CustomWorkload( function="set_se_binary_workload", parameters={ - "binary": obtain_resource("x86-hello64-static"), + "binary": obtain_resource( + "x86-hello64-static", gem5_version="develop" + ), "arguments": ["hello", 6], }, ) - @classmethod - def tearDownClass(cls): - # Unset the environment variable so this test does not interfere with - # others. - os.environ["GEM5_RESOURCE_JSON"] - def test_get_function_str(self) -> None: # Tests `CustomResource.get_function_str` @@ -140,7 +120,8 @@ class CustomWorkloadTestSuite(unittest.TestCase): "test", self.custom_workload.get_parameters()["binary"] ) - # We set the overridden parameter back to it's old valu self.custom_workload.set_parameter("binary", old_value) + # We set the overridden parameter back to it's old value + self.custom_workload.set_parameter("binary", old_value) class WorkloadTestSuite(unittest.TestCase): @@ -151,21 +132,10 @@ class WorkloadTestSuite(unittest.TestCase): @classmethod @patch( "gem5.resources.client.clientwrapper", - ClientWrapper(mock_config_json2), + ClientWrapper(mock_config_json), ) def setUpClass(cls): - os.environ["GEM5_RESOURCE_JSON"] = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "refs", - "workload-checks.json", - ) - cls.workload = Workload("simple-boot") - - @classmethod - def tearDownClass(cls): - # Unset the environment variable so this test does not interfere with - # others. - os.environ["GEM5_RESOURCE_JSON"] + cls.workload = Workload("simple-boot", gem5_version="develop") def test_get_function_str(self) -> None: # Tests `Resource.get_function_str` diff --git a/tests/pyunit/stdlib/resources/refs/mongo-mock.json b/tests/pyunit/stdlib/resources/refs/mongo-mock.json index b6376cc5e4..e2fb058ff7 100644 --- a/tests/pyunit/stdlib/resources/refs/mongo-mock.json +++ b/tests/pyunit/stdlib/resources/refs/mongo-mock.json @@ -22,7 +22,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" }, @@ -49,7 +49,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "1.1.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" } diff --git a/tests/pyunit/stdlib/resources/refs/obtain-resource.json b/tests/pyunit/stdlib/resources/refs/obtain-resource.json index fac95e106a..9125bf4ae6 100644 --- a/tests/pyunit/stdlib/resources/refs/obtain-resource.json +++ b/tests/pyunit/stdlib/resources/refs/obtain-resource.json @@ -24,7 +24,7 @@ "source": "src/test-source", "resource_version": "2.0.0", "gem5_versions": [ - "develop" + "23.0" ] }, { @@ -38,7 +38,8 @@ "source": "src/test-source", "resource_version": "1.7.0", "gem5_versions": [ - "develop" + "develop", + "develop-2" ] }, { @@ -52,8 +53,7 @@ "source": "src/test-source", "resource_version": "1.5.0", "gem5_versions": [ - "21.1", - "22.1" + "develop" ] } ] diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index 1129f1bd05..414bf73b11 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -10,6 +10,7 @@ "source": "src/linux-kernel", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -25,6 +26,7 @@ "root_partition": "1", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -39,6 +41,7 @@ "source": "src/simple", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -51,6 +54,7 @@ "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -66,6 +70,7 @@ "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -75,10 +80,11 @@ "description": null, "is_zipped": false, "is_tar_archive": true, - "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -87,11 +93,12 @@ "id": "file-example", "description": null, "is_zipped": false, - "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "md5sum": "2efd144c11829ab18d54eae6371e120a", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -106,6 +113,7 @@ "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -125,6 +133,7 @@ "workload_name": "Example Workload", "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -148,6 +157,7 @@ ], "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -161,6 +171,7 @@ "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] }, @@ -170,11 +181,12 @@ "description": "A looppoint json file resource.", "is_zipped": false, "region_id": "1", - "md5sum": "a71ed64908b082ea619b26b940a643c1", + "md5sum": "efb85ebdf90c5cee655bf2e05ae7692a", "url": "http://dist.gem5.org/dist/develop/looppoints/x86-matrix-multiply-omp-100-8-looppoint-json-20230128", "source": null, "resource_version": "1.0.0", "gem5_versions": [ + "develop", "23.0" ] } diff --git a/tests/pyunit/stdlib/resources/refs/resources.json b/tests/pyunit/stdlib/resources/refs/resources.json index 812caeff43..56930f37d5 100644 --- a/tests/pyunit/stdlib/resources/refs/resources.json +++ b/tests/pyunit/stdlib/resources/refs/resources.json @@ -21,7 +21,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -48,7 +48,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "1.1.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -71,7 +71,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/asmtest", "resource_version": "2.0.0", "gem5_versions": [ - "23.1" + "999.1" ], "example_usage": "get_resource(resource_name=\"rv64mi-p-sbreak\")" }, @@ -94,7 +94,7 @@ "source_url": "", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -122,7 +122,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -150,7 +150,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -178,7 +178,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -206,7 +206,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -234,7 +234,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -262,7 +262,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -290,7 +290,7 @@ "source_url": "", "resource_version": "0.2.0", "gem5_versions": [ - "23.0" + "develop" ], "workload_name": "x86-print-this-15000-with-simpoints", "example_usage": "get_resource(resource_name=\"x86-print-this-1500-simpoints\")", @@ -322,7 +322,7 @@ "source_url": "https://github.com/gem5/gem5-resources/tree/develop/src/x86-ubuntu", "resource_version": "2.0.0", "gem5_versions": [ - "23.0" + "develop" ], "example_usage": "get_resource(resource_name=\"x86-ubuntu-18.04-img\")" } diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json b/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json deleted file mode 100644 index a7e9c9d84f..0000000000 --- a/tests/pyunit/stdlib/resources/refs/workload-checks-custom-workload.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "category": "binary", - "id": "x86-hello64-static", - "description": "A 'Hello World!' binary.", - "architecture": "X86", - "is_zipped": false, - "md5sum": "dbf120338b37153e3334603970cebd8c", - "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", - "source": "src/simple", - "resource_version": "1.0.0", - "gem5_versions": [ - "23.0" - ] - } -] diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index d41001d26c..dcb8577619 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -10,7 +10,7 @@ "source": "src/linux-kernel", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ] }, { @@ -25,7 +25,7 @@ "root_partition": "1", "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" ] }, { @@ -42,7 +42,21 @@ }, "resource_version": "1.0.0", "gem5_versions": [ - "23.0" + "develop" + ] + }, + { + "category": "binary", + "id": "x86-hello64-static", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" ] } ] From a63d376ecd4debd60f89fa2e0592dac6f9addae2 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 13 Jun 2023 17:42:23 -0700 Subject: [PATCH 22/35] python: Remove Python 'pipes' module This is scheduled for removal from Python in 3.13: https://docs.python.org/3/library/pipes.html. The 'shlex.quote' function can replace the 'pipes.quote' function used in "main.py". A special wrapper has been made to account for the Windows case which 'shlex.quote' doesn't handle. Change-Id: I9c84605f0ccd8468b9cab6cece6248ef8c2107f0 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71678 Maintainer: Jason Lowe-Power Tested-by: kokoro Reviewed-by: Jason Lowe-Power --- src/python/m5/main.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/python/m5/main.py b/src/python/m5/main.py index 4701dfa5ea..ddcb024f8b 100644 --- a/src/python/m5/main.py +++ b/src/python/m5/main.py @@ -496,10 +496,23 @@ def main(): % (socket.gethostname(), os.getpid()) ) - # in Python 3 pipes.quote() is moved to shlex.quote() - import pipes + def quote(arg: str) -> str: + """Quotes a string for printing in a shell. In addition to Unix, + this is designed to handle the problematic Windows cases where + 'shlex.quote' doesn't work""" - print("command line:", " ".join(map(pipes.quote, sys.argv))) + if os.name == "nt" and os.sep == "\\": + # If a Windows machine, we manually quote the string. + arg = arg.replace('"', '\\"') + if re.search("\s", args): + # We quote args which have whitespace. + arg = '"' + arg + '"' + return arg + import shlex + + return shlex.quote(arg) + + print("command line:", " ".join(map(quote, sys.argv))) print() # check to make sure we can find the listed script From b182b15f93621206c87c6c760cdfc1f5df1877cf Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 13 Jun 2023 18:14:38 -0700 Subject: [PATCH 23/35] scons,stdlib: Remove deprecated 'distutils' module The Python module 'distutils' will be removed in Python 3.12: https://docs.python.org/3/library/distutils.html This patch removed usage of 'distutils' in the gem5 code base. Change-Id: I1e3a944446149f3cd6cbf4211a1565b5f74c85a0 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71679 Tested-by: kokoro Maintainer: Jason Lowe-Power Reviewed-by: Jason Lowe-Power --- src/SConscript | 4 +-- .../resources/client_api/client_wrapper.py | 26 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/SConscript b/src/SConscript index d26bf49cce..1b4430327c 100644 --- a/src/SConscript +++ b/src/SConscript @@ -39,7 +39,7 @@ import collections import copy -import distutils.spawn +from shutil import which import itertools import os import os.path @@ -269,7 +269,7 @@ def ProtoBuf(source, tags=None, add_tags=None): '''Add a Protocol Buffer to build''' Source(source, tags, add_tags, append={'CXXFLAGS': '-Wno-array-bounds'}) -env['PROTOC_GRPC'] = distutils.spawn.find_executable('grpc_cpp_plugin') +env['PROTOC_GRPC'] = which('grpc_cpp_plugin') if env['PROTOC_GRPC']: with Configure(env) as conf: if (not env['HAVE_PKG_CONFIG'] or diff --git a/src/python/gem5/resources/client_api/client_wrapper.py b/src/python/gem5/resources/client_api/client_wrapper.py index ebf1b8ec2b..d2baabc52d 100644 --- a/src/python/gem5/resources/client_api/client_wrapper.py +++ b/src/python/gem5/resources/client_api/client_wrapper.py @@ -27,8 +27,7 @@ from .jsonclient import JSONClient from .atlasclient import AtlasClient from _m5 import core -from typing import Optional, Dict, List -from distutils.version import StrictVersion +from typing import Optional, Dict, List, Tuple import itertools from m5.util import warn @@ -247,12 +246,27 @@ class ClientWrapper: :param resources: A list of resources to sort. :return: A list of sorted resources. """ + + def sort_tuple(resource: Dict) -> Tuple: + """This is used for sorting resources by ID and version. First + the ID is sorted, then the version. In cases where the version + contains periods, it's assumed this is to separate a + "major.minor.hotfix" style versioning system. In which case, the + value separated in the most-significant position is sorted before + those less significant. If the value is a digit it is cast as an + int, otherwise, it is cast as a string, to lower-case. + """ + to_return = (resource["id"].lower(),) + for val in resource["resource_version"].split("."): + if val.isdigit(): + to_return += (int(val),) + else: + to_return += (str(val).lower(),) + return to_return + return sorted( resources, - key=lambda resource: ( - resource["id"].lower(), - StrictVersion(resource["resource_version"]), - ), + key=lambda resource: sort_tuple(resource), reverse=True, ) From d54b8f8475495213f5c2e69aadd7886c69bcd53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Armejach?= Date: Wed, 7 Jun 2023 17:01:03 +0200 Subject: [PATCH 24/35] arch-riscv: fix load reserved store conditional * According to the manual, load reservations must be cleared on a failed or a successful SC attempt. * A load reservation can be arbitrarily large. The current implementation was reserving something different than cacheBlockSize which could lead to problems if snoop addresses are cache block aligned. This patch implementation assumes a cacheBlock granularity. * Load reservations should also be cleared on faults Change-Id: I64513534710b5f269260fcb204f717801913e2f5 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71520 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- src/arch/generic/isa.hh | 1 + src/arch/riscv/faults.cc | 3 +++ src/arch/riscv/isa.cc | 29 ++++++++++++++++++----------- src/arch/riscv/isa.hh | 12 ++++++++++++ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/arch/generic/isa.hh b/src/arch/generic/isa.hh index e9e4d95d7b..58f66fc99b 100644 --- a/src/arch/generic/isa.hh +++ b/src/arch/generic/isa.hh @@ -70,6 +70,7 @@ class BaseISA : public SimObject public: virtual PCStateBase *newPCState(Addr new_inst_addr=0) const = 0; virtual void clear() {} + virtual void clearLoadReservation(ContextID cid) {} virtual RegVal readMiscRegNoEffect(RegIndex idx) const = 0; virtual RegVal readMiscReg(RegIndex idx) = 0; diff --git a/src/arch/riscv/faults.cc b/src/arch/riscv/faults.cc index 940f7107ba..8fb8f81261 100644 --- a/src/arch/riscv/faults.cc +++ b/src/arch/riscv/faults.cc @@ -153,6 +153,9 @@ RiscvFault::invoke(ThreadContext *tc, const StaticInstPtr &inst) tc->setMiscReg(MISCREG_NMIE, 0); } + // Clear load reservation address + tc->getIsaPtr()->clearLoadReservation(tc->contextId()); + // Set PC to fault handler address Addr addr = mbits(tc->readMiscReg(tvec), 63, 2); if (isInterrupt() && bits(tc->readMiscReg(tvec), 1, 0) == 1) diff --git a/src/arch/riscv/isa.cc b/src/arch/riscv/isa.cc index d744fe369b..94a8239bac 100644 --- a/src/arch/riscv/isa.cc +++ b/src/arch/riscv/isa.cc @@ -672,11 +672,6 @@ ISA::unserialize(CheckpointIn &cp) UNSERIALIZE_CONTAINER(miscRegFile); } -const int WARN_FAILURE = 10000; - -const Addr INVALID_RESERVATION_ADDR = (Addr) -1; -std::unordered_map load_reservation_addrs; - void ISA::handleLockedSnoop(PacketPtr pkt, Addr cacheBlockMask) { @@ -696,9 +691,9 @@ ISA::handleLockedRead(const RequestPtr &req) { Addr& load_reservation_addr = load_reservation_addrs[tc->contextId()]; - load_reservation_addr = req->getPaddr() & ~0xF; + load_reservation_addr = req->getPaddr(); DPRINTF(LLSC, "[cid:%d]: Reserved address %x.\n", - req->contextId(), req->getPaddr() & ~0xF); + req->contextId(), req->getPaddr()); } bool @@ -717,12 +712,13 @@ ISA::handleLockedWrite(const RequestPtr &req, Addr cacheBlockMask) lr_addr_empty ? "yes" : "no"); if (!lr_addr_empty) { DPRINTF(LLSC, "[cid:%d]: addr = %x.\n", req->contextId(), - req->getPaddr() & ~0xF); + req->getPaddr() & cacheBlockMask); DPRINTF(LLSC, "[cid:%d]: last locked addr = %x.\n", req->contextId(), - load_reservation_addr); + load_reservation_addr & cacheBlockMask); } - if (lr_addr_empty - || load_reservation_addr != ((req->getPaddr() & ~0xF))) { + if (lr_addr_empty || + (load_reservation_addr & cacheBlockMask) + != ((req->getPaddr() & cacheBlockMask))) { req->setExtraData(0); int stCondFailures = tc->readStCondFailures(); tc->setStCondFailures(++stCondFailures); @@ -730,12 +726,21 @@ ISA::handleLockedWrite(const RequestPtr &req, Addr cacheBlockMask) warn("%i: context %d: %d consecutive SC failures.\n", curTick(), tc->contextId(), stCondFailures); } + + // Must clear any reservations + load_reservation_addr = INVALID_RESERVATION_ADDR; + return false; } if (req->isUncacheable()) { req->setExtraData(2); } + // Must clear any reservations + load_reservation_addr = INVALID_RESERVATION_ADDR; + + DPRINTF(LLSC, "[cid:%d]: SC success! Current locked addr = %x.\n", + req->contextId(), load_reservation_addr & cacheBlockMask); return true; } @@ -743,6 +748,8 @@ void ISA::globalClearExclusive() { tc->getCpuPtr()->wakeup(tc->threadId()); + Addr& load_reservation_addr = load_reservation_addrs[tc->contextId()]; + load_reservation_addr = INVALID_RESERVATION_ADDR; } void diff --git a/src/arch/riscv/isa.hh b/src/arch/riscv/isa.hh index 5a2a610479..7ef5c526f5 100644 --- a/src/arch/riscv/isa.hh +++ b/src/arch/riscv/isa.hh @@ -76,6 +76,11 @@ class ISA : public BaseISA bool hpmCounterEnabled(int counter) const; + // Load reserve - store conditional monitor + const int WARN_FAILURE = 10000; + const Addr INVALID_RESERVATION_ADDR = (Addr)-1; + std::unordered_map load_reservation_addrs; + public: using Params = RiscvISAParams; @@ -87,6 +92,13 @@ class ISA : public BaseISA return new PCState(new_inst_addr, rv_type); } + void + clearLoadReservation(ContextID cid) override + { + Addr& load_reservation_addr = load_reservation_addrs[cid]; + load_reservation_addr = INVALID_RESERVATION_ADDR; + } + public: RegVal readMiscRegNoEffect(RegIndex idx) const override; RegVal readMiscReg(RegIndex idx) override; From b271367a357cbb390084884e809235d6d62b9cf5 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 27 Jun 2023 09:25:31 -0700 Subject: [PATCH 25/35] misc: Adjust compiler tests and CI tests This adds a job level timeout for the compiler tests, allows them to run weekly instead of daily, adds a workflow dispatch option, and changes the 'latests-compilers-all-gem5-builds' jobs to run only the .opt variant. It also adds a ready for review option to the CI tests to run when someone converts a draft pull request. Change-Id: Id32b5f7da90745d18de2e550bd48d32ba45fb4d9 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71725 Reviewed-by: Jason Lowe-Power Maintainer: Bobby Bruce Tested-by: kokoro --- .github/workflows/ci-tests.yaml | 2 +- .github/workflows/compiler-tests.yaml | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index b1c5a1af31..61ba4b5b53 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -4,7 +4,7 @@ name: CI Tests on: pull_request: - types: [opened, edited, synchronize] + types: [opened, edited, synchronize, ready_for_review] jobs: diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 013a710066..52569e2e01 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -5,9 +5,11 @@ run-name: ${{ github.actor }} is running compiler tests on: - # Runs every day from 7AM UTC + # Runs every Friday from 7AM UTC schedule: - - cron: '0 7 * * *' + - cron: '00 7 * * 5' + # Allows us to manually start workflow for testing + workflow_dispatch: jobs: # replication of compiler-tests.sh @@ -17,6 +19,7 @@ jobs: image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, gcc-version-7, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, clang-version-6.0, ubuntu-18.04_all-dependencies, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] + timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: - uses: actions/checkout@v3 @@ -34,8 +37,9 @@ jobs: matrix: gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base] image: [gcc-version-12, clang-version-14] - opts: [.opt, .fast] + opts: [.opt] runs-on: [self-hosted, linux, x64, run] + timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: - uses: actions/checkout@v3 From e03395b386672d56d8b03f0acaf7cb6d282c9a67 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Thu, 6 Jul 2023 16:16:12 -0700 Subject: [PATCH 26/35] misc: Check for Change-Id in GitHub Actions This adds a check to our continous integration tests that ensures that every single commit in a pull request has a Change-Id, and instructs you to add one if that check fails. This ensures all commits on GitHub are still compatible with Gerrit. Change-Id: I3fd753da5ab3ca6d6334a8e0eb574433aa1ca589 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72118 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Maintainer: Bobby Bruce Reviewed-by: Richard Cooper Reviewed-by: Bobby Bruce --- .github/workflows/ci-tests.yaml | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 61ba4b5b53..c3188b87ee 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -16,10 +16,33 @@ jobs: - uses: actions/setup-python@v3 - uses: pre-commit/action@v3.0.0 + # ensures we have a change-id in every commit, needed for gerrit + check-for-change-id: + # runs on github hosted runner + runs-on: ubuntu-latest + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + steps: + - uses: actions/github-script@v6 + env: + token: "Change-Id" + pattern: ".*" + with: + script: | + const commits = ${{ toJSON(github.event.commits) }} + for (const commit of commits) { + const id = "Change-Id: " + const message = commit.message; + if (!message.includes(id)) { + core.setFailed('One or more of the commits in this pull request is missing a Change-ID, which we require for any changes made to gem5. ' + + 'To automatically insert one, run the following:\n f=`git rev-parse --git-dir`/hooks/commit-msg ; mkdir -p $(dirname $f) ; ' + + 'curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request.') + } + } + build-gem5: runs-on: [self-hosted, linux, x64, build] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: pre-commit # only runs if pre-commit passes + needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes outputs: artifactname: ${{ steps.name.outputs.test }} steps: @@ -39,7 +62,7 @@ jobs: unittests-all-opt: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: pre-commit # only runs if pre-commit passes + needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -51,7 +74,7 @@ jobs: testlib-quick: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, build-gem5] + needs: [pre-commit, check-for-change-id, build-gem5] timeout-minutes: 360 # 6 hours steps: - uses: actions/checkout@v3 From 04b4a4705a2c0bb937172169e3634e1b3352c779 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 3 Jul 2023 10:03:57 -0700 Subject: [PATCH 27/35] stdlib: Change default gem5-resources DB collection This was set to "test_collection", which was used during development. Changing to "resources". Change-Id: I52c83c6b73f3a227fbb05dc321a4bc38210ad71c Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72158 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- src/python/gem5_default_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/gem5_default_config.py b/src/python/gem5_default_config.py index 980c883109..6b689f58d5 100644 --- a/src/python/gem5_default_config.py +++ b/src/python/gem5_default_config.py @@ -29,7 +29,7 @@ config = { "gem5-resources": { "dataSource": "gem5-vision", "database": "gem5-vision", - "collection": "versions_test", + "collection": "resources", "url": "https://data.mongodb-api.com/app/data-ejhjf/endpoint/data/v1", "authUrl": "https://realm.mongodb.com/api/client/v2.0/app/data-ejhjf/auth/providers/api-key/login", "apiKey": "OIi5bAP7xxIGK782t8ZoiD2BkBGEzMdX3upChf9zdCxHSnMoiTnjI22Yw5kOSgy9", From 4cf6695bc7bceb1505625380f0519b5a361378eb Mon Sep 17 00:00:00 2001 From: Jason Lowe-Power Date: Sun, 18 Jun 2023 14:58:24 -0400 Subject: [PATCH 28/35] misc: Add release notes for v23.0 Change-Id: I003f170339e69a445586fe0486a1db595a10683f Signed-off-by: Jason Lowe-Power --- RELEASE-NOTES.md | 118 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 931be695ba..0c158ee245 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,121 @@ +# Version 23.0 + +This release has approximately 500 contributions from 50 unique contributors. +Below we highlight key gem5 features and improvements in this release. + +## Significant API and user-facing changes + +### Major renaming of CPU stats + +The CPU stats have been renamed. +See for details. + +Now, each stage (fetch, execute, commit) have their own stat group. +Stats that are shared between the different CPU model (O3, Minor, Simple) now have the exact same names. + +**Important:** Some stat names were misleading before this change. +With this change, stats with the same names between different CPU models have the same meaning. + +### `fs.py` and `se.py` deprecated + +These scripts have not been well supported for many gem5 releases. +With gem5 23.0, we have officially deprecated these scripts. +They have been moved into the `deprecated` directory, **but they will be removed in a future release.** +As a replacement, we strongly suggest using the gem5 standard library. +See for more information. + +### Renaming of `DEBUG` guard into `GEM5_DEBUG` + +Scons no longer defines the `DEBUG` guard in debug builds, so code making using of it should use `GEM5_DEBUG` instead. + +### Other API changes + +Also, this release: + +- Removes deprecated namespaces. Namespace names were updated a couple of releases ago. This release removes the old names. +- Uses `MemberEventWrapper` in favor of `EventWrapper` for instance member functions. +- Adds an extension mechanism to `Packet` and `Request`. +- Sets x86 CPU vendor string to "HygoneGenuine" to better support GLIBC. + +## New features and improvements + +### Large improvements to gem5 resources and gem5 resources website + +We now have a new web portal for the gem5 resources: + +This web portal will allow users to browse the resources available (e.g., disk images, kernels, workloads, binaries, simpoints, etc.) to use out-of-the-box with the gem5 standard library. +You can filter based on architecture, resource type, and compatible gem5 versions. + +For each resource, there are examples of how to use the resource and pointers to examples using the resource in the gem5 codebase. + +More information can be found on gem5's website: + +We will be expanding gem5 resources with more workloads and resources over the course of the next release. +If you would like to contribute to gem5 resources by uploading your own workloads, disk images, etc., please create an issue on GitHub. + +In addition to the new gem5 Resources web portal, the gem5 Resources API has been significantly updated and improved. +There are now much simpler functions for getting resources such as `obtain_resource()` that will download the resource by name and return a reference that can be used (e.g., as a binary in `set_se_workload` function on the board). +As such the generic `Resouce` class has been deprecated and will be removed in a future release. + +Resources are now specialized for their particular category. +For example, there is now a `BinaryResource` class which will return if a user specifies a binary resource when using the `obtain_resource` function. +This allow for resource typing and for greater resource specialization. + +### Arm ISA improvements + +Architectural support for Armv9 [Scalable Matrix extension](https://developer.arm.com/documentation/ddi0616/latest) (FEAT_SME). +The implementation employs a simple renaming scheme for the Za array register in the O3 CPU, so that writes to difference tiles in the register are considered a dependency and are therefore serialized. + +The following SVE and SIMD & FP extensions have also been implemented: +* FEAT_F64MM +* FEAT_F32MM +* FEAT_DOTPROD +* FEAT_I8MM + +And more generally: + +* FEAT_TLBIOS +* FEAT_FLAGM +* FEAT_FLAGM2 +* FEAT_RNG +* FEAT_RNG_TRAP +* FEAT_EVT + +### Support for DRAMSys + +gem5 can now use DRAMSys as a DRAM backend. + +### RISC-V improvements + +This release: + +- Fully implements RISC-V scalar cryptography extensions. +- Fully implement RISC-V rv32. +- Implements PMP lock features. +- Adds general RISC-V improvements to provide better stability. + +### Standard library improvements and new components + +This release: + +- Adds MESI_Three_Level component. +- Supports ELFies and LoopPoint analysis output from Sniper. +- Supports DRAMSys in the stdlib. + +## Bugfixes and other small improvements + +This release also: + +- Removes deprecated python libraries. +- Adds a DDR5 model. +- Adds AMD GPU MI200/gfx90a support. +- Changes building so it no longer "duplicates sources" in build/ which improves support for some IDEs and code analysis. If you still need to duplicate sources you can use the `--duplicate-sources` option to `scons`. +- Enables `--debug-activate=` to use debug trace for only a single SimObject (the opposite of `--debug-ignore`). See `--debug-help` for more information. +- Adds support to exit the simulation loop based on Arm-PMU events. +- Supports Python 3.11. +- Adds the idea of a CpuCluster to gem5. + + # Version 22.1.0.0 This release has 500 contributions from 48 unique contributors and marks our second major release of 2022. From 818c2d15a229ef87e310a1dd1180980787fff3cc Mon Sep 17 00:00:00 2001 From: Matthew Poremba Date: Thu, 29 Jun 2023 16:48:48 -0700 Subject: [PATCH 29/35] configs: Create base GPUFS vega config and atomic config Move the Vega KVM script code to a common base file and add scripts for KVM and atomic. Since atomic is now possible in GPUFS this gives a way to run it without editing the current scripts. Change-Id: I094bc4d4df856563535c28c1f6d6cc045d6734cd Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71939 Tested-by: kokoro Reviewed-by: Matt Sinclair Maintainer: Matt Sinclair (cherry picked from commit 05ffa354268301821518fd430d069cd23049527d) Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72078 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- configs/example/gpufs/vega10.py | 153 +++++++++++++++++++++++++ configs/example/gpufs/vega10_atomic.py | 32 ++++++ configs/example/gpufs/vega10_kvm.py | 127 +------------------- 3 files changed, 188 insertions(+), 124 deletions(-) create mode 100644 configs/example/gpufs/vega10.py create mode 100644 configs/example/gpufs/vega10_atomic.py diff --git a/configs/example/gpufs/vega10.py b/configs/example/gpufs/vega10.py new file mode 100644 index 0000000000..9eff5a2974 --- /dev/null +++ b/configs/example/gpufs/vega10.py @@ -0,0 +1,153 @@ +# Copyright (c) 2022-2023 Advanced Micro Devices, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import m5 +import runfs +import base64 +import tempfile +import argparse +import sys +import os + +from amd import AmdGPUOptions +from common import Options +from common import GPUTLBOptions +from ruby import Ruby + + +demo_runscript_without_checkpoint = """\ +export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH +export HSA_ENABLE_INTERRUPT=0 +dmesg -n8 +dd if=/root/roms/vega10.rom of=/dev/mem bs=1k seek=768 count=128 +if [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then + echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5." + /sbin/m5 exit +fi +modprobe -v amdgpu ip_block_mask=0xff ppfeaturemask=0 dpm=0 audio=0 +echo "Running {} {}" +echo "{}" | base64 -d > myapp +chmod +x myapp +./myapp {} +/sbin/m5 exit +""" + +demo_runscript_with_checkpoint = """\ +export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH +export HSA_ENABLE_INTERRUPT=0 +dmesg -n8 +dd if=/root/roms/vega10.rom of=/dev/mem bs=1k seek=768 count=128 +if [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then + echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5." + /sbin/m5 exit +fi +modprobe -v amdgpu ip_block_mask=0xff ppfeaturemask=0 dpm=0 audio=0 +echo "Running {} {}" +echo "{}" | base64 -d > myapp +chmod +x myapp +/sbin/m5 checkpoint +./myapp {} +/sbin/m5 exit +""" + + +def addDemoOptions(parser): + parser.add_argument( + "-a", "--app", default=None, help="GPU application to run" + ) + parser.add_argument( + "-o", "--opts", default="", help="GPU application arguments" + ) + + +def runVegaGPUFS(cpu_type): + parser = argparse.ArgumentParser() + runfs.addRunFSOptions(parser) + Options.addCommonOptions(parser) + AmdGPUOptions.addAmdGPUOptions(parser) + Ruby.define_options(parser) + GPUTLBOptions.tlb_options(parser) + addDemoOptions(parser) + + # Parse now so we can override options + args = parser.parse_args() + demo_runscript = "" + + # Create temp script to run application + if args.app is None: + print(f"No application given. Use {sys.argv[0]} -a ") + sys.exit(1) + elif args.kernel is None: + print(f"No kernel path given. Use {sys.argv[0]} --kernel ") + sys.exit(1) + elif args.disk_image is None: + print(f"No disk path given. Use {sys.argv[0]} --disk-image ") + sys.exit(1) + elif args.gpu_mmio_trace is None: + print(f"No MMIO trace path. Use {sys.argv[0]} --gpu-mmio-trace ") + sys.exit(1) + elif not os.path.isfile(args.app): + print("Could not find applcation", args.app) + sys.exit(1) + + # Choose runscript Based on whether any checkpointing args are set + if args.checkpoint_dir is not None: + demo_runscript = demo_runscript_with_checkpoint + else: + demo_runscript = demo_runscript_without_checkpoint + + with open(os.path.abspath(args.app), "rb") as binfile: + encodedBin = base64.b64encode(binfile.read()).decode() + + _, tempRunscript = tempfile.mkstemp() + with open(tempRunscript, "w") as b64file: + runscriptStr = demo_runscript.format( + args.app, args.opts, encodedBin, args.opts + ) + b64file.write(runscriptStr) + + if args.second_disk == None: + args.second_disk = args.disk_image + + # Defaults for Vega10 + args.ruby = True + args.cpu_type = cpu_type + args.num_cpus = 1 + args.mem_size = "3GB" + args.dgpu = True + args.dgpu_mem_size = "16GB" + args.dgpu_start = "0GB" + args.checkpoint_restore = 0 + args.disjoint = True + args.timing_gpu = True + args.script = tempRunscript + args.dgpu_xor_low_bit = 0 + + # Run gem5 + runfs.runGpuFSSystem(args) diff --git a/configs/example/gpufs/vega10_atomic.py b/configs/example/gpufs/vega10_atomic.py new file mode 100644 index 0000000000..4ff2cc2e72 --- /dev/null +++ b/configs/example/gpufs/vega10_atomic.py @@ -0,0 +1,32 @@ +# Copyright (c) 2023 Advanced Micro Devices, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import vega10 + +vega10.runVegaGPUFS("AtomicSimpleCPU") diff --git a/configs/example/gpufs/vega10_kvm.py b/configs/example/gpufs/vega10_kvm.py index 11f9fe2f80..39dc5e0110 100644 --- a/configs/example/gpufs/vega10_kvm.py +++ b/configs/example/gpufs/vega10_kvm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Advanced Micro Devices, Inc. +# Copyright (c) 2022-2023 Advanced Micro Devices, Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -27,127 +27,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -import m5 -import runfs -import base64 -import tempfile -import argparse -import sys -import os +import vega10 -from amd import AmdGPUOptions -from common import Options -from common import GPUTLBOptions -from ruby import Ruby - - -demo_runscript_without_checkpoint = """\ -export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH -export HSA_ENABLE_INTERRUPT=0 -dmesg -n8 -dd if=/root/roms/vega10.rom of=/dev/mem bs=1k seek=768 count=128 -if [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then - echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5." - /sbin/m5 exit -fi -modprobe -v amdgpu ip_block_mask=0xff ppfeaturemask=0 dpm=0 audio=0 -echo "Running {} {}" -echo "{}" | base64 -d > myapp -chmod +x myapp -./myapp {} -/sbin/m5 exit -""" - -demo_runscript_with_checkpoint = """\ -export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH -export HSA_ENABLE_INTERRUPT=0 -dmesg -n8 -dd if=/root/roms/vega10.rom of=/dev/mem bs=1k seek=768 count=128 -if [ ! -f /lib/modules/`uname -r`/updates/dkms/amdgpu.ko ]; then - echo "ERROR: Missing DKMS package for kernel `uname -r`. Exiting gem5." - /sbin/m5 exit -fi -modprobe -v amdgpu ip_block_mask=0xff ppfeaturemask=0 dpm=0 audio=0 -echo "Running {} {}" -echo "{}" | base64 -d > myapp -chmod +x myapp -/sbin/m5 checkpoint -./myapp {} -/sbin/m5 exit -""" - - -def addDemoOptions(parser): - parser.add_argument( - "-a", "--app", default=None, help="GPU application to run" - ) - parser.add_argument( - "-o", "--opts", default="", help="GPU application arguments" - ) - - -if __name__ == "__m5_main__": - parser = argparse.ArgumentParser() - runfs.addRunFSOptions(parser) - Options.addCommonOptions(parser) - AmdGPUOptions.addAmdGPUOptions(parser) - Ruby.define_options(parser) - GPUTLBOptions.tlb_options(parser) - addDemoOptions(parser) - - # Parse now so we can override options - args = parser.parse_args() - demo_runscript = "" - - # Create temp script to run application - if args.app is None: - print(f"No application given. Use {sys.argv[0]} -a ") - sys.exit(1) - elif args.kernel is None: - print(f"No kernel path given. Use {sys.argv[0]} --kernel ") - sys.exit(1) - elif args.disk_image is None: - print(f"No disk path given. Use {sys.argv[0]} --disk-image ") - sys.exit(1) - elif args.gpu_mmio_trace is None: - print(f"No MMIO trace path. Use {sys.argv[0]} --gpu-mmio-trace ") - sys.exit(1) - elif not os.path.isfile(args.app): - print("Could not find applcation", args.app) - sys.exit(1) - - # Choose runscript Based on whether any checkpointing args are set - if args.checkpoint_dir is not None: - demo_runscript = demo_runscript_with_checkpoint - else: - demo_runscript = demo_runscript_without_checkpoint - - with open(os.path.abspath(args.app), "rb") as binfile: - encodedBin = base64.b64encode(binfile.read()).decode() - - _, tempRunscript = tempfile.mkstemp() - with open(tempRunscript, "w") as b64file: - runscriptStr = demo_runscript.format( - args.app, args.opts, encodedBin, args.opts - ) - b64file.write(runscriptStr) - - if args.second_disk == None: - args.second_disk = args.disk_image - - # Defaults for Vega10 - args.ruby = True - args.cpu_type = "X86KvmCPU" - args.num_cpus = 1 - args.mem_size = "3GB" - args.dgpu = True - args.dgpu_mem_size = "16GB" - args.dgpu_start = "0GB" - args.checkpoint_restore = 0 - args.disjoint = True - args.timing_gpu = True - args.script = tempRunscript - args.dgpu_xor_low_bit = 0 - - # Run gem5 - runfs.runGpuFSSystem(args) +vega10.runVegaGPUFS("X86KvmCPU") From 8d1722478160edc64f5069efccfaf2aaa8213208 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 3 Jul 2023 13:03:05 -0700 Subject: [PATCH 30/35] stdlib,tests: Fix download_check.py This was causing the Weekly tests to fail. The removing of the download directory should only happen at the end. Prior to this patch it was deleted and then referenced, which caused problems. Change-Id: I134782e89a13f5c3cd5c1912ad53a701d0413d16 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72159 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro --- tests/gem5/configs/download_check.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/gem5/configs/download_check.py b/tests/gem5/configs/download_check.py index 2180f4f26a..e3b06a578d 100644 --- a/tests/gem5/configs/download_check.py +++ b/tests/gem5/configs/download_check.py @@ -94,7 +94,6 @@ for id in ids: continue for resource_version in ids[id]: - resource_json = get_resource_json_obj( resource_id=id, resource_version=resource_version, @@ -129,8 +128,8 @@ for id in ids: + f" gem5-resources ({resource_json['md5sum']}).{os.linesep}" ) - # Remove the downloaded resource. - shutil.rmtree(download_path, ignore_errors=True) +# Remove the downloaded resource. +shutil.rmtree(args.download_directory, ignore_errors=True) # If errors exist, raise an exception highlighting them. if errors: From 6c997b633fb5759bd45ff947626c5228e2c67539 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 7 Jul 2023 08:46:16 -0700 Subject: [PATCH 31/35] scons: Remove -Werror for the gem5 v23.0 release While gem5 compiles on all our supported compilers, removing the -Werror flag on the stable branch ensures that, as new compilers are released with stricter warnings, gem5 remains compilable. Change-Id: Ie32a5c526a17cc584bce3b71d27b8f052caf612b Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72178 Maintainer: Bobby Bruce Reviewed-by: Bobby Bruce Tested-by: kokoro --- SConstruct | 6 ------ 1 file changed, 6 deletions(-) diff --git a/SConstruct b/SConstruct index 9b25b33783..4fe2f64366 100755 --- a/SConstruct +++ b/SConstruct @@ -447,12 +447,6 @@ for variant_path in variant_paths: conf.CheckLinkFlag( '-Wl,--thread-count=%d' % GetOption('num_jobs')) - # Treat warnings as errors but white list some warnings that we - # want to allow (e.g., deprecation warnings). - env.Append(CCFLAGS=['-Werror', - '-Wno-error=deprecated-declarations', - '-Wno-error=deprecated', - ]) else: error('\n'.join(( From 578eaead472f6bceea3876cbc85a0120a6055c9b Mon Sep 17 00:00:00 2001 From: Matthew Poremba Date: Wed, 28 Jun 2023 14:49:48 -0500 Subject: [PATCH 32/35] dev-amdgpu: Perform frame writes atomically The PCI read/write functions are atomic functions in gem5, meaning they expect a response with a latency value on the same simulation Tick. For reads to a PCI device, the response must also include a data value read from the device. The AMDGPU device has a PCI BAR which mirrors the frame buffer memory. Currently reads are done atomically, but writes are sent to a DMA device without waiting for a write completion ACK. As a result, it is possible that writes can be queued in the DMA device long enough that another read for a queued address arrives. This happens very deterministically with the AtomicSimpleCPU and causes GPUFS to break with that CPU. This change makes writes to the frame BAR atomic the same as reads. This avoids that problem and as a result the AtomicSimpleCPU can now load the driver for GPUFS simulations. Change-Id: I9a8e8b172712c78b667ebcec81a0c5d0060234db Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71898 Maintainer: Matt Sinclair Tested-by: kokoro Reviewed-by: Matt Sinclair Maintainer: Matthew Poremba Reviewed-by: Matthew Poremba (cherry picked from commit 079fc47dc202ffe7c77e1e94bb1d5e0ee38d1816) Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72079 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- src/dev/amdgpu/amdgpu_device.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/dev/amdgpu/amdgpu_device.cc b/src/dev/amdgpu/amdgpu_device.cc index 3260d058b0..d1058f1606 100644 --- a/src/dev/amdgpu/amdgpu_device.cc +++ b/src/dev/amdgpu/amdgpu_device.cc @@ -349,6 +349,22 @@ AMDGPUDevice::writeFrame(PacketPtr pkt, Addr offset) } nbio.writeFrame(pkt, offset); + + /* + * Write the value to device memory. This must be done functionally + * because this method is called by the PCIDevice::write method which + * is a non-timing write. + */ + RequestPtr req = std::make_shared(offset, pkt->getSize(), 0, + vramRequestorId()); + PacketPtr writePkt = Packet::createWrite(req); + uint8_t *dataPtr = new uint8_t[pkt->getSize()]; + std::memcpy(dataPtr, pkt->getPtr(), + pkt->getSize() * sizeof(uint8_t)); + writePkt->dataDynamic(dataPtr); + + auto system = cp->shader()->gpuCmdProc.system(); + system->getDeviceMemory(writePkt)->access(writePkt); } void @@ -489,8 +505,6 @@ AMDGPUDevice::write(PacketPtr pkt) switch (barnum) { case FRAMEBUFFER_BAR: - gpuMemMgr->writeRequest(offset, pkt->getPtr(), - pkt->getSize(), 0, nullptr); writeFrame(pkt, offset); break; case DOORBELL_BAR: From 387fc6964e27d085acaf15628045734875b7a0eb Mon Sep 17 00:00:00 2001 From: Matthew Poremba Date: Thu, 6 Jul 2023 08:13:01 -0700 Subject: [PATCH 33/35] gpu-compute,configs: Make sim exits conditional The unconditional exit event when a kernel completes that was added in c644eae2ddd34cf449a9c4476730bd29703c4dd7 is causing scripts that do not ignore unknown exit events to end simulation prematurely. One such script is the apu_se.py script used in SE mode GPU simulation. Make this exit conditional to the parameter being set to a valid value to avoid this problem. Change-Id: I1d2c082291fdbcf27390913ffdffb963ec8080dd Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72098 Reviewed-by: Jason Lowe-Power Maintainer: Matt Sinclair Maintainer: Jason Lowe-Power Reviewed-by: Matt Sinclair Tested-by: kokoro (cherry picked from commit 3756af8ed91f0860dcfb0a7111478212980e37ae) Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/72138 Maintainer: Bobby Bruce Reviewed-by: Bobby Bruce --- configs/example/gpufs/system/system.py | 3 ++- src/gpu-compute/GPU.py | 4 ++++ src/gpu-compute/dispatcher.cc | 7 ++++++- src/gpu-compute/dispatcher.hh | 2 ++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/configs/example/gpufs/system/system.py b/configs/example/gpufs/system/system.py index 263ffc0a43..471892945e 100644 --- a/configs/example/gpufs/system/system.py +++ b/configs/example/gpufs/system/system.py @@ -115,7 +115,8 @@ def makeGpuFSSystem(args): numHWQueues=args.num_hw_queues, walker=hsapp_pt_walker, ) - dispatcher = GPUDispatcher() + dispatcher_exit_events = True if args.exit_at_gpu_kernel > -1 else False + dispatcher = GPUDispatcher(kernel_exit_events=dispatcher_exit_events) cp_pt_walker = VegaPagetableWalker() gpu_cmd_proc = GPUCommandProcessor( hsapp=gpu_hsapp, dispatcher=dispatcher, walker=cp_pt_walker diff --git a/src/gpu-compute/GPU.py b/src/gpu-compute/GPU.py index c5449cc398..c64a6b791d 100644 --- a/src/gpu-compute/GPU.py +++ b/src/gpu-compute/GPU.py @@ -328,6 +328,10 @@ class GPUDispatcher(SimObject): cxx_class = "gem5::GPUDispatcher" cxx_header = "gpu-compute/dispatcher.hh" + kernel_exit_events = Param.Bool( + False, "Enable exiting sim loop after a kernel" + ) + class GPUCommandProcessor(DmaVirtDevice): type = "GPUCommandProcessor" diff --git a/src/gpu-compute/dispatcher.cc b/src/gpu-compute/dispatcher.cc index a76ba7c0be..d63c875fe5 100644 --- a/src/gpu-compute/dispatcher.cc +++ b/src/gpu-compute/dispatcher.cc @@ -50,7 +50,8 @@ GPUDispatcher::GPUDispatcher(const Params &p) : SimObject(p), shader(nullptr), gpuCmdProc(nullptr), tickEvent([this]{ exec(); }, "GPU Dispatcher tick", false, Event::CPU_Tick_Pri), - dispatchActive(false), stats(this) + dispatchActive(false), kernelExitEvents(p.kernel_exit_events), + stats(this) { schedule(&tickEvent, 0); } @@ -330,6 +331,10 @@ GPUDispatcher::notifyWgCompl(Wavefront *wf) DPRINTF(GPUWgLatency, "Kernel Complete ticks:%d kernel:%d\n", curTick(), kern_id); DPRINTF(GPUKernelInfo, "Completed kernel %d\n", kern_id); + + if (kernelExitEvents) { + exitSimLoop("GPU Kernel Completed"); + } } if (!tickEvent.scheduled()) { diff --git a/src/gpu-compute/dispatcher.hh b/src/gpu-compute/dispatcher.hh index 7699ceff7c..eafa0804b1 100644 --- a/src/gpu-compute/dispatcher.hh +++ b/src/gpu-compute/dispatcher.hh @@ -92,6 +92,8 @@ class GPUDispatcher : public SimObject std::queue doneIds; // is there a kernel in execution? bool dispatchActive; + // Enable exiting sim loop after each kernel completion + bool kernelExitEvents; protected: struct GPUDispatcherStats : public statistics::Group From e377e15c07e439b696a4a09b0179fd4553d11d0a Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Thu, 15 Jun 2023 16:08:49 -0700 Subject: [PATCH 34/35] misc: Update gem5 version to v23.0.0.0 Change-Id: Ie14b35a62a152e3d78b16bcd4c92ec9a790f1396 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71724 Tested-by: kokoro Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- src/Doxyfile | 2 +- src/base/version.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Doxyfile b/src/Doxyfile index 24d70bbc55..325040fee0 100644 --- a/src/Doxyfile +++ b/src/Doxyfile @@ -31,7 +31,7 @@ PROJECT_NAME = gem5 # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = [DEVELOP-FOR-23.0] +PROJECT_NUMBER = v23.0.0.0 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. diff --git a/src/base/version.cc b/src/base/version.cc index 8131a3197e..38f415590a 100644 --- a/src/base/version.cc +++ b/src/base/version.cc @@ -32,6 +32,6 @@ namespace gem5 /** * @ingroup api_base_utils */ -const char *gem5Version = "[DEVELOP-FOR-23.0]"; +const char *gem5Version = "23.0.0.0"; } // namespace gem5 From 20ee3b97622be2cc448de31d84dd5f29109e0ada Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 7 Jul 2023 18:21:44 -0700 Subject: [PATCH 35/35] stdlib: Remove simulator.py beta warnings This component is no longer in a beta state Change-Id: I8e309771aaa03197cf1738ad0af57c253ad58ecd --- src/python/gem5/simulate/simulator.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/python/gem5/simulate/simulator.py b/src/python/gem5/simulate/simulator.py index e27679a996..0551745b36 100644 --- a/src/python/gem5/simulate/simulator.py +++ b/src/python/gem5/simulate/simulator.py @@ -53,10 +53,6 @@ class Simulator: """ This Simulator class is used to manage the execution of a gem5 simulation. - **Warning:** The simulate package is still in a beta state. The gem5 - project does not guarantee the APIs within this package will remain - consistent in future across upcoming releases. - Example ------- Examples using the Simulator class can be found under @@ -165,12 +161,6 @@ class Simulator: """ - warn( - "The simulate package is still in a beta state. The gem5 " - "project does not guarantee the APIs within this package will " - "remain consistent across upcoming releases." - ) - # We specify a dictionary here outlining the default behavior for each # exit event. Each exit event is mapped to a generator. self._default_on_exit_dict = {