From fbaa164b6a7275760066b62fb863501c67dd2ffe Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 9 May 2023 15:30:11 -0700 Subject: [PATCH 01/55] gpu-compute: Remove use of 'std::random_shuffle' This was deprecated in C++14 and removed in C++17. This has been replaced with std::random. This has been implemented to ensure reproducible results despite (pseudo)random behavior. Change-Id: Idd52bc997547c7f8c1be88f6130adff8a37b4116 --- src/cpu/testers/gpu_ruby_test/address_manager.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/cpu/testers/gpu_ruby_test/address_manager.cc b/src/cpu/testers/gpu_ruby_test/address_manager.cc index 37f74203f7..049ba86e51 100644 --- a/src/cpu/testers/gpu_ruby_test/address_manager.cc +++ b/src/cpu/testers/gpu_ruby_test/address_manager.cc @@ -32,6 +32,8 @@ #include "cpu/testers/gpu_ruby_test/address_manager.hh" #include +#include +#include #include "base/intmath.hh" #include "base/logging.hh" @@ -58,8 +60,13 @@ AddressManager::AddressManager(int n_atomic_locs, int n_normal_locs_per_atomic) randAddressMap[i] = (Addr)((i + 128) << floorLog2(sizeof(Value))); } - // randomly shuffle randAddressMap - std::random_shuffle(randAddressMap.begin(), randAddressMap.end()); + // randomly shuffle randAddressMap. The seed is determined by the random_mt + // gem5 rng. This allows for deterministic randomization. + std::shuffle( + randAddressMap.begin(), + randAddressMap.end(), + std::default_random_engine(random_mt.random(0,UINT_MAX)) + ); // initialize atomic locations // first and last normal location per atomic location From dd7780c0b250747cb98db41e76bbd7ebd184e7ff Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 9 May 2023 15:30:45 -0700 Subject: [PATCH 02/55] dev-amdgpu: Add missing 'overrides' This causes warnings/errors in some compilers. Change-Id: I36a3548943c030d2578c2f581c8985c12eaeb0ae --- src/arch/amdgpu/vega/insts/instructions.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arch/amdgpu/vega/insts/instructions.hh b/src/arch/amdgpu/vega/insts/instructions.hh index d45a84c7b8..0e4ec04764 100644 --- a/src/arch/amdgpu/vega/insts/instructions.hh +++ b/src/arch/amdgpu/vega/insts/instructions.hh @@ -31534,8 +31534,8 @@ namespace VegaISA } // getOperandSize void execute(GPUDynInstPtr) override; - void initiateAcc(GPUDynInstPtr gpuDynInst); - void completeAcc(GPUDynInstPtr gpuDynInst); + void initiateAcc(GPUDynInstPtr gpuDynInst) override; + void completeAcc(GPUDynInstPtr gpuDynInst) override; }; // Inst_DS__DS_OR_B32 class Inst_DS__DS_XOR_B32 : public Inst_DS From 8295e84360a9cdab034b8d5f78fdf0826d5510c2 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 9 May 2023 15:39:23 -0700 Subject: [PATCH 03/55] dev: Fix Linux specific includes to be portable This allows for compilation in non-linux systems (e.g., Mac OS). Change-Id: Ib6c9406baf42db8caaad335ebc670c1905584ea2 --- src/dev/hsa/kfd_ioctl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/dev/hsa/kfd_ioctl.h b/src/dev/hsa/kfd_ioctl.h index c953787dc8..b7997c40fc 100644 --- a/src/dev/hsa/kfd_ioctl.h +++ b/src/dev/hsa/kfd_ioctl.h @@ -23,9 +23,10 @@ #ifndef KFD_IOCTL_H_INCLUDED #define KFD_IOCTL_H_INCLUDED +#include +#include + #include -#include -#include namespace gem5 { From 556955b7c9e8b4ce6e8d289244b69efd1641adb2 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 10 Jul 2023 13:42:47 -0700 Subject: [PATCH 04/55] gpu-compute: Add missing include in dispatcher.cc Due to some cherry-picking onto the release-staging branch, there was a missing "sim/sim_exit.hh" include in "src/gpu-compute/dispatcher.cc". This was causing compilation errors. This is being added to the v23.0.0 release as a hotfix. Change-Id: I1043ecf5c41ad6afc0e91311b196f4801646002f Issue-on: https://gem5.atlassian.net/browse/GEM5-1332 --- src/gpu-compute/dispatcher.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gpu-compute/dispatcher.cc b/src/gpu-compute/dispatcher.cc index d63c875fe5..7b36bce591 100644 --- a/src/gpu-compute/dispatcher.cc +++ b/src/gpu-compute/dispatcher.cc @@ -40,6 +40,7 @@ #include "gpu-compute/hsa_queue_entry.hh" #include "gpu-compute/shader.hh" #include "gpu-compute/wavefront.hh" +#include "sim/sim_exit.hh" #include "sim/syscall_emul_buf.hh" #include "sim/system.hh" From 587159069a1127e0f0d93df8229f993dce7a0937 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 10 Jul 2023 14:08:27 -0700 Subject: [PATCH 05/55] misc: Update version to v23.0.0.1 Change-Id: I3bbcfd4dd9798149b37d4a2824fe63652e29786c --- src/Doxyfile | 2 +- src/base/version.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Doxyfile b/src/Doxyfile index 325040fee0..79498c9c72 100644 --- a/src/Doxyfile +++ b/src/Doxyfile @@ -31,7 +31,7 @@ PROJECT_NAME = gem5 # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = v23.0.0.0 +PROJECT_NUMBER = v23.0.0.1 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. diff --git a/src/base/version.cc b/src/base/version.cc index 38f415590a..1069d57348 100644 --- a/src/base/version.cc +++ b/src/base/version.cc @@ -32,6 +32,6 @@ namespace gem5 /** * @ingroup api_base_utils */ -const char *gem5Version = "23.0.0.0"; +const char *gem5Version = "23.0.0.1"; } // namespace gem5 From af72b9ba580546ac12ce05bfaac3fd53fa8699f4 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 10 Jul 2023 14:13:09 -0700 Subject: [PATCH 06/55] misc: Update RELEASE-NOTES.md for v23.0.0.1 hotfix Change-Id: Ieced7f693a8cbef586324dfe7ce826da16d9a3c3 --- RELEASE-NOTES.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 0c158ee245..1a72428fc1 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,15 @@ +# Version 23.0.0.1 + +**[HOTFIX]** Fixes compilation of `GCN3_X86` and `VEGA_X85`. + +This hotfix release: + +* Removes the use of 'std::random_shuffle'. +This is a deprecated function in C++17 and has been removed in C++20. +* Adds missing 'overrides' in "src/arch/amdgpu/vega/insts/instructions.hh". +* Fixes Linux specific includes, allowing for compilation on non-linux systems. +* Adds a missing include in "src/gpu-compute/dispatcher.cc". + # Version 23.0 This release has approximately 500 contributions from 50 unique contributors. From 91d2cd738667f4e214dc08ff7aa17a41e8d6addd Mon Sep 17 00:00:00 2001 From: Yu-hsin Wang Date: Wed, 31 May 2023 11:25:07 +0800 Subject: [PATCH 07/55] fastmodel: add src include path by default We have some customized protocols in gem5 repository and they require the include path from src directory. It causes the users of those protocols need to handle the include path correctly by theirselve. This is tedious and unstable. We should add the default include path in SIMGEN command line to prevent issues. Change-Id: I2a3748646567635d131a8fb4099e02e332691e97 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71118 Reviewed-by: Wei-Han Chen Tested-by: kokoro Maintainer: Jason Lowe-Power --- src/arch/arm/fastmodel/CortexA76/x1/x1.sgproj | 2 -- src/arch/arm/fastmodel/CortexA76/x2/x2.sgproj | 2 -- src/arch/arm/fastmodel/CortexA76/x3/x3.sgproj | 2 -- src/arch/arm/fastmodel/CortexA76/x4/x4.sgproj | 2 -- src/arch/arm/fastmodel/CortexR52/x1/x1.sgproj | 2 -- src/arch/arm/fastmodel/CortexR52/x2/x2.sgproj | 2 -- src/arch/arm/fastmodel/CortexR52/x3/x3.sgproj | 2 -- src/arch/arm/fastmodel/CortexR52/x4/x4.sgproj | 2 -- src/arch/arm/fastmodel/GIC/GIC.sgproj | 2 +- src/arch/arm/fastmodel/PL330_DMAC/PL330.sgproj | 2 +- src/arch/arm/fastmodel/SConscript | 3 ++- 11 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/arch/arm/fastmodel/CortexA76/x1/x1.sgproj b/src/arch/arm/fastmodel/CortexA76/x1/x1.sgproj index f3fd0db70f..7101b499db 100644 --- a/src/arch/arm/fastmodel/CortexA76/x1/x1.sgproj +++ b/src/arch/arm/fastmodel/CortexA76/x1/x1.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexA76/x2/x2.sgproj b/src/arch/arm/fastmodel/CortexA76/x2/x2.sgproj index abec8addca..54327dc723 100644 --- a/src/arch/arm/fastmodel/CortexA76/x2/x2.sgproj +++ b/src/arch/arm/fastmodel/CortexA76/x2/x2.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexA76/x3/x3.sgproj b/src/arch/arm/fastmodel/CortexA76/x3/x3.sgproj index 666b1dc0f4..5809b31f66 100644 --- a/src/arch/arm/fastmodel/CortexA76/x3/x3.sgproj +++ b/src/arch/arm/fastmodel/CortexA76/x3/x3.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexA76/x4/x4.sgproj b/src/arch/arm/fastmodel/CortexA76/x4/x4.sgproj index e3c9063ac7..1866c98be8 100644 --- a/src/arch/arm/fastmodel/CortexA76/x4/x4.sgproj +++ b/src/arch/arm/fastmodel/CortexA76/x4/x4.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexR52/x1/x1.sgproj b/src/arch/arm/fastmodel/CortexR52/x1/x1.sgproj index 9d2a5749f6..83c0584869 100644 --- a/src/arch/arm/fastmodel/CortexR52/x1/x1.sgproj +++ b/src/arch/arm/fastmodel/CortexR52/x1/x1.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexR52/x2/x2.sgproj b/src/arch/arm/fastmodel/CortexR52/x2/x2.sgproj index e103170ed1..ab8399a419 100644 --- a/src/arch/arm/fastmodel/CortexR52/x2/x2.sgproj +++ b/src/arch/arm/fastmodel/CortexR52/x2/x2.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexR52/x3/x3.sgproj b/src/arch/arm/fastmodel/CortexR52/x3/x3.sgproj index 0c92809924..a9bfa3fa08 100644 --- a/src/arch/arm/fastmodel/CortexR52/x3/x3.sgproj +++ b/src/arch/arm/fastmodel/CortexR52/x3/x3.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/CortexR52/x4/x4.sgproj b/src/arch/arm/fastmodel/CortexR52/x4/x4.sgproj index 6a145fd1c1..dc333763cf 100644 --- a/src/arch/arm/fastmodel/CortexR52/x4/x4.sgproj +++ b/src/arch/arm/fastmodel/CortexR52/x4/x4.sgproj @@ -16,8 +16,6 @@ config "gcc" SIMGEN_COMMAND_LINE = "--num-comps-file 50"; TARGET_SYSTEMC = "1"; TARGET_SYSTEMC_AUTO = "1"; - - INCLUDE_DIRS="../../../../../"; } files { diff --git a/src/arch/arm/fastmodel/GIC/GIC.sgproj b/src/arch/arm/fastmodel/GIC/GIC.sgproj index aa5e6aec32..fd5c04a5a5 100644 --- a/src/arch/arm/fastmodel/GIC/GIC.sgproj +++ b/src/arch/arm/fastmodel/GIC/GIC.sgproj @@ -5,7 +5,7 @@ ACTIVE_CONFIG_LINUX = "gcc"; ACTIVE_CONFIG_WINDOWS = "Win64-Release-VC2015"; config "gcc" { - ADDITIONAL_COMPILER_SETTINGS = "-O3 -Wall -std=c++14 -Wno-deprecated -Wno-unused-function -I../../../../../"; + ADDITIONAL_COMPILER_SETTINGS = "-O3 -Wall -std=c++14 -Wno-deprecated -Wno-unused-function"; ADDITIONAL_LINKER_SETTINGS = "-Wl,--no-undefined"; BUILD_DIR = "./gcc"; COMPILER = "gcc-7.3"; diff --git a/src/arch/arm/fastmodel/PL330_DMAC/PL330.sgproj b/src/arch/arm/fastmodel/PL330_DMAC/PL330.sgproj index d59849c409..c972ad760c 100644 --- a/src/arch/arm/fastmodel/PL330_DMAC/PL330.sgproj +++ b/src/arch/arm/fastmodel/PL330_DMAC/PL330.sgproj @@ -5,7 +5,7 @@ ACTIVE_CONFIG_LINUX = "gcc"; ACTIVE_CONFIG_WINDOWS = "Win64-Release-VC2015"; config "gcc" { - ADDITIONAL_COMPILER_SETTINGS = "-O3 -Wall -std=c++14 -Wno-deprecated -Wno-unused-function -I../../../../../"; + ADDITIONAL_COMPILER_SETTINGS = "-O3 -Wall -std=c++14 -Wno-deprecated -Wno-unused-function"; ADDITIONAL_LINKER_SETTINGS = "-Wl,--no-undefined"; BUILD_DIR = "./gcc"; COMPILER = "gcc-7.3"; diff --git a/src/arch/arm/fastmodel/SConscript b/src/arch/arm/fastmodel/SConscript index 9d9d183516..3bc9aa5bad 100644 --- a/src/arch/arm/fastmodel/SConscript +++ b/src/arch/arm/fastmodel/SConscript @@ -377,11 +377,12 @@ class ArmFastModelComponent(object): self.rpaths = [simgen_dir, project_file_dir] self.log = gen_dir.File('build_%s.log' % tlc) self.simgen_cmd = env.subst('${CONF["SIMGEN"]} -p %s ' - '--configuration %s -b --verbose off --num-build-cpus %d %s ' + '--configuration %s -b --verbose off --num-build-cpus %d -I %s %s ' '--build-dir %s >%s') % \ (shlex.quote(project_file.srcnode().abspath), shlex.quote(config_name), GetOption('num_jobs'), + shlex.quote(Dir('#/src').srcnode().abspath), simgen_command_line, shlex.quote(simgen_dir.abspath), shlex.quote(self.log.abspath)) From f802bc0af77ca695427e491606cbaa509a620a47 Mon Sep 17 00:00:00 2001 From: Roger Chang Date: Fri, 2 Jun 2023 17:00:59 +0800 Subject: [PATCH 08/55] arch-riscv: Check FPU status for c.flwsp c.fldsp c.fswsp c.fsdsp The change adds the missing FPU checking for these instructions. Change-Id: I7f2ef89786af0d528f2029f1097cfeac6c7d65f2 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71198 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Maintainer: Jason Lowe-Power --- src/arch/riscv/isa/decoder.isa | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/arch/riscv/isa/decoder.isa b/src/arch/riscv/isa/decoder.isa index c7eefbc79c..2dcd118225 100644 --- a/src/arch/riscv/isa/decoder.isa +++ b/src/arch/riscv/isa/decoder.isa @@ -307,6 +307,11 @@ decode QUADRANT default Unknown::unknown() { CIMM1 << 5 | CIMM5<2:0> << 6; }}, {{ + STATUS status = xc->readMiscReg(MISCREG_STATUS); + if (status.fs == FPUStatus::OFF) + return std::make_shared("FPU is off", + machInst); + Fc1_bits = Mem; }}, {{ EA = rvZext(sp + offset); @@ -330,6 +335,11 @@ decode QUADRANT default Unknown::unknown() { CIMM1 << 5 | CIMM5<1:0> << 6; }}, {{ + STATUS status = xc->readMiscReg(MISCREG_STATUS); + if (status.fs == FPUStatus::OFF) + return std::make_shared("FPU is off", + machInst); + freg_t fd; fd = freg(f32(Mem_uw)); Fd_bits = fd.v; @@ -387,6 +397,11 @@ decode QUADRANT default Unknown::unknown() { offset = CIMM6<5:3> << 3 | CIMM6<2:0> << 6; }}, {{ + STATUS status = xc->readMiscReg(MISCREG_STATUS); + if (status.fs == FPUStatus::OFF) + return std::make_shared("FPU is off", + machInst); + Mem_ud = Fc2_bits; }}, {{ EA = rvZext(sp + offset); @@ -404,6 +419,11 @@ decode QUADRANT default Unknown::unknown() { offset = CIMM6<5:2> << 2 | CIMM6<1:0> << 6; }}, {{ + STATUS status = xc->readMiscReg(MISCREG_STATUS); + if (status.fs == FPUStatus::OFF) + return std::make_shared("FPU is off", + machInst); + Mem_uw = unboxF32(boxF32(Fs2_bits)); }}, {{ EA = (uint32_t)(sp_uw + offset); From a31d3416fe7327887dbad50a8298ab43cd92a384 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Tue, 30 May 2023 16:44:32 -0700 Subject: [PATCH 09/55] util: Add util for GitHub runner configuration This adds files that can be used to configure Vagrant machines that will be used to test running gem5 alongside Github Actions. Change-Id: I52b0f39b6e6044c22481f02163d5fc01eab76788 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71098 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro --- util/github-runners-vagrant/README.md | 118 ++++++++++++++++++ .../Vagrantfile-builder | 49 ++++++++ .../github-runners-vagrant/Vagrantfile-runner | 48 +++++++ .../provision_nonroot.sh | 15 +++ util/github-runners-vagrant/provision_root.sh | 43 +++++++ util/github-runners-vagrant/vm_manager.sh | 39 ++++++ 6 files changed, 312 insertions(+) create mode 100644 util/github-runners-vagrant/README.md create mode 100644 util/github-runners-vagrant/Vagrantfile-builder create mode 100644 util/github-runners-vagrant/Vagrantfile-runner create mode 100644 util/github-runners-vagrant/provision_nonroot.sh create mode 100644 util/github-runners-vagrant/provision_root.sh create mode 100755 util/github-runners-vagrant/vm_manager.sh diff --git a/util/github-runners-vagrant/README.md b/util/github-runners-vagrant/README.md new file mode 100644 index 0000000000..86b9e1da87 --- /dev/null +++ b/util/github-runners-vagrant/README.md @@ -0,0 +1,118 @@ +# Setting up a Github Actions Runner with Vagrant + +This directory provides a way to setup a Github Actions runner using Vagrant to host the runner in a Virtual machine. + +This tutorial has been written with the assumption of running on a machine with Ubuntu 22.04. +Setting up a runner on a different OS may require some changes. + +Before anything else, copy this directory, "util/github-runners-vagrant", to the root of the location on your host system you wish to setup the VMs from. +The CWD is assumed to be this directory. + +## Install Dependencies + +```sh +sudo apt install vagrant +sudo apt-get build-dep vagrant ruby-libvirt +sudo apt-get install qemu libvirt-daemon-system libvirt-clients ebtables dnsmasq-base libxslt-dev libxml2-dev libvirt-dev zlib1g-dev ruby-dev + +# Note: The vagrant-libvirt APT package does not work as intended. We must +# remove it from the system otherwise errors will occur (we will install it +# later using the vagrant plugin command). +sudo apt purge vagrant-libvirt +``` + +## Set up the Vagrantfiles for the GitHub repository + +First, generate a Personal Access Token, which you can create [here](https://github.com/settings/tokens) +Make sure to set admin permissions on this token, then replace instances of `` in the Vagrantfiles ("Vagrantfile-builder" and "Vagrant-runner") with your token. + +Next, replace instances of `` with your GitHub account name and the repository name, separated by a forward slash. +For example, if your GitHub account name is `example` and your repository name is `example-repo`, you would replace `` with `example/example-repo`. + +## Install Vagrant Plugins + +Once everything is set properly, set the `VAGRANT_HOME` environment variable to the directory in which the Vagrant files and other scripts are stored (i.e., the CWD). +For example: + +```sh +export VAGRANT_HOME=`pwd` +``` + +After this, install the relevant vagrant plugins: + +``` sh +vagrant plugin install dotenv +vagrant plugin install vagrant-libvirt +vagrant plugin install vagrant-reload +``` + +## The "builder" and "runner" VMs + +The number of CPUs and the memory size differs between the "Vagrantfile-builder" and "Vagrantfile-runner". + +In our work we have two types of machines "runners" and "builders". +Runners are single core machines with 8GB of memory, and builders are 4 core machines with 16GB of memory. +The latter is used for building gem5 binaries while the former is used for running instances of gem5. +You can expect each machine to take up approximately 60GB of disk space though VMs will consume the disk space they require. + +The "Vagrantfile-builder" file is set to create a runner machine and the "Vagrantfile-builder" file is set to create a builder machine. + +Specifying which Vagrantfile to use is done by setting the `VAGRANT_VAGRANTFILE` environment variable. + +## Creating the virtual machine + +Each VM on your host system must have a unique name. +Give the VM to be created a unique name by setting the `` variables in the Vagrantfile you wish to utilize. + +Then run: + +```sh +VAGRANT_VAGRANTFILE= vagrant up --provider=libvirt +``` + +This should automatically create your machine, as well as configure and start up a Github Actions runner. +You can check the status of the runner here: https://github.com///settings/actions/runners + +If the runner ever shows as offline, you can rerun the `vagrant up --provider=libvirt` command to make sure everything is working properly. + +If you wish to create more than one runner you must edit the `` in the Vagrant file. + +## Helper scripts + +The "vm_manager" script can be used to set up multiple builder and runner VMs. +To use this script simply modify the `NUM_RUNNERS`, `NUM_BUILDERS`, `RUNNER_PREFIX`, and `BUILDER_PREFIX` variables to the desired values. +Then run the script with: + +```sh +./vm_manager.sh +``` + +This script will create any VMs that don't already exist and ensure those that do exists are running. + +If you wish to destroy all the VMs you can run: + +```sh +./vm_manager.sh destroy +``` + +**Note:** This script assumes "VAGRANT_HOME" is set to the CWD. + +## Troubleshooting + +### The default libvirt disk image storage pool is on the wrong drive + +By default libvirt will store disk images in "/var/lib/libvirt/images". +This is not ideal as it is on a small root partition. +A solution to this is to change the default storage location. +To do so, do the following: + +```sh +virsh pool-list --all # Confirm here a "default" pool exist. We'll modify this. +virsh pool-dumpxml default >default-pool.xml # We take a dump of the default then removed it. +virsh pool-destroy default +virsh pool-undefine default +vim default-pool.xml # Change the image path to the desired path +virsh pool-define default-pool.xml # From here we re-add the default. +virsh pool-start default +virsh pool-autostart default +``` diff --git a/util/github-runners-vagrant/Vagrantfile-builder b/util/github-runners-vagrant/Vagrantfile-builder new file mode 100644 index 0000000000..242e094d19 --- /dev/null +++ b/util/github-runners-vagrant/Vagrantfile-builder @@ -0,0 +1,49 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2204" + config.vm.box_check_update = true + config.vm.define "" + config.vm.hostname = "" + # allows us to ssh into the machine, addressing the problem below + # https://www.reddit.com/r/vagrant/comments/sb7hfl/new_to_vagrant_getting_efault_warning/ + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider "libvirt" do |vb| + # Customize the amount of cpus and memory on the VM: + vb.cpus = "4".to_i + vb.memory = "16384".to_i + end + + # sets up vm + config.vm.provision :shell, path: "provision_root.sh" + config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. + config.vm.provision :reload + config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL + if [ -d ~/actions-runner ]; then + # This will be run everytime the VM is run (once created). + cd actions-runner + nohup ./run.sh & + else + # This will be run the first time the VM is created. + mkdir ~/actions-runner && cd ~/actions-runner + curl -so actions-runner-linux-x64-2.304.0.tar.gz -L \ + https://github.com/actions/runner/releases/download/v2.304.0/actions-runner-linux-x64-2.304.0.tar.gz + tar xzf ./actions-runner-linux-x64-2.304.0.tar.gz + + # configure the runner + # echo automatically sets the name of the runner, and the tags + # create a personal access token with admin permission and copy it into the curl command + echo -ne '\n\nbuild\n\n' | ./config.sh --url https://github.com/ --token $(curl -L \ + -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos//actions/runners/registration-token | jq -r '.token') + # start the runner + nohup ./run.sh & + fi + SHELL + +end diff --git a/util/github-runners-vagrant/Vagrantfile-runner b/util/github-runners-vagrant/Vagrantfile-runner new file mode 100644 index 0000000000..ab601cd9c8 --- /dev/null +++ b/util/github-runners-vagrant/Vagrantfile-runner @@ -0,0 +1,48 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2204" + config.vm.box_check_update = true + config.vm.define "" + config.vm.hostname = "" + # allows us to ssh into the machine, addressing the problem below + # https://www.reddit.com/r/vagrant/comments/sb7hfl/new_to_vagrant_getting_efault_warning/ + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider "libvirt" do |vb| + # Customize the amount of cpus and memory on the VM: + vb.cpus = "1".to_i + vb.memory = "8192".to_i + end + + # sets up vm + config.vm.provision :shell, path: "provision_root.sh" + config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. + config.vm.provision :reload + config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL + if [ -d ~/actions-runner ]; then + # This will be run everytime the VM is run (once created). + cd actions-runner + nohup ./run.sh & + else + # This will be run the first time the VM is created. + mkdir ~/actions-runner && cd ~/actions-runner + curl -so actions-runner-linux-x64-2.304.0.tar.gz -L \ + https://github.com/actions/runner/releases/download/v2.304.0/actions-runner-linux-x64-2.304.0.tar.gz + tar xzf ./actions-runner-linux-x64-2.304.0.tar.gz + + # configure the runner + # echo automatically sets the name of the runner, and the tags + # create a personal access token with admin permission and copy it into the curl command + echo -ne '\n\nrun\n\n' | ./config.sh --url https://github.com/ --token $(curl -L \ + -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer " \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos//actions/runners/registration-token | jq -r '.token') + # start the runner + nohup ./run.sh & + fi + SHELL +end diff --git a/util/github-runners-vagrant/provision_nonroot.sh b/util/github-runners-vagrant/provision_nonroot.sh new file mode 100644 index 0000000000..4465b5a192 --- /dev/null +++ b/util/github-runners-vagrant/provision_nonroot.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# fail on unset variables and command errors +set -eu -o pipefail # -x: is for debugging + +# Install deno +curl -fsSL https://deno.land/x/install/install.sh | sh +echo "export PATH=\"\${HOME}/.deno/bin:\${PATH}\"" >> ~/.profile +echo "export PATH=\"\${HOME}/.deno/bin:\${PATH}\"" >> ~/.bash_profile + +# Install docker compose +DOCKER_COMPOSE_VERSION=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.tag_name') +mkdir -p "${HOME}/.docker/cli-plugins" +curl -sL "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o "${HOME}/.docker/cli-plugins/docker-compose" +chmod +x "${HOME}/.docker/cli-plugins/docker-compose" diff --git a/util/github-runners-vagrant/provision_root.sh b/util/github-runners-vagrant/provision_root.sh new file mode 100644 index 0000000000..660064f8ca --- /dev/null +++ b/util/github-runners-vagrant/provision_root.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# fail on unset variables and command errors +set -eu -o pipefail # -x: is for debugging + +apt-get update +apt-get upgrade -y +apt-get install -y software-properties-common +add-apt-repository --yes --update ppa:git-core/ppa +apt-get install -y \ + bash \ + build-essential \ + clang-format \ + git \ + git-lfs \ + jq \ + libffi-dev \ + libssl-dev \ + nkf \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + shellcheck \ + tree \ + wget \ + yamllint \ + zstd +snap install jq + +# Install docker +apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io +groupadd docker || true +gpasswd -a vagrant docker +newgrp docker +systemctl restart docker + +# Cleanup +apt-get autoremove -y diff --git a/util/github-runners-vagrant/vm_manager.sh b/util/github-runners-vagrant/vm_manager.sh new file mode 100755 index 0000000000..1cae8d2686 --- /dev/null +++ b/util/github-runners-vagrant/vm_manager.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +NUM_RUNNERS=20 +NUM_BUILDERS=3 +RUNNER_PREFIX="$(hostname)-runner-" +BUILDER_PREFIX="$(hostname)-builder-" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +export VAGRANT_HOME=${SCRIPT_DIR} + +param="up" +if [[ $# -eq 1 ]]; then + param=$1 + if [[ "${param}" != "destroy" ]]; then + echo "Only valid parameter is 'destroy', to destroy all VMs" + exit 1 + fi +fi + + +for (( i=1; i<=NUM_RUNNERS; i++ )); do + sed -i "s/ config.vm.define.*/ config.vm.define \"${RUNNER_PREFIX}${i}\"/g" Vagrantfile-runner + sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${RUNNER_PREFIX}${i}\"/g" Vagrantfile-runner + if [[ "${param}" == "destroy" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant destroy -f + else + VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant up --provider=libvirt + fi +done + +for (( i=1; i<=NUM_BUILDERS; i++ )); do + sed -i "s/ config.vm.define.*/ config.vm.define \"${BUILDER_PREFIX}${i}\"/g" Vagrantfile-builder + sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${BUILDER_PREFIX}${i}\"/g" Vagrantfile-builder + if [[ "${param}" == "destroy" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant destroy -f + else + VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant up --provider=libvirt + fi +done From c23b15755ba3c16b52607479026c19ee8de420fb Mon Sep 17 00:00:00 2001 From: Yu-hsin Wang Date: Tue, 30 May 2023 13:26:18 +0800 Subject: [PATCH 10/55] fastmodel: only support single line literal when paring project file In python3.11, it requires the global specifier should be the first token of regex. However it's not possible when using ply library. In fastmodel case, we actually don't need to support multiline string literal. We fix this issue by just making the string literal single line. Ref: https://github.com/dabeaz/ply/issues/282 Change-Id: I746b628db7ad4c1d7834f1a1b2c1243cef68aa01 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71018 Tested-by: kokoro Maintainer: Bobby Bruce Reviewed-by: Bobby Bruce --- src/arch/arm/fastmodel/SConscript | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/arch/arm/fastmodel/SConscript b/src/arch/arm/fastmodel/SConscript index 3bc9aa5bad..7c6019e2a8 100644 --- a/src/arch/arm/fastmodel/SConscript +++ b/src/arch/arm/fastmodel/SConscript @@ -218,10 +218,9 @@ class ProjectFileParser(Grammar): t_ID = r'[A-Za-z_]\w*' def t_STRLIT(self, t): - r'(?m)"([^"])*"' + r'"([^"])*"' # strip off quotes t.value = t.value[1:-1] - t.lexer.lineno += t.value.count('\n') return t t_EQUALS = r'=' From 872dc8bf59ddf84f21e870839aa7e7b9d7b20778 Mon Sep 17 00:00:00 2001 From: Yu-hsin Wang Date: Tue, 30 May 2023 13:27:01 +0800 Subject: [PATCH 11/55] arch: set multiline re as default in isa_parser In python3.11, it requires the global specifier should be the first token of regex. However it's not possible when using ply library. Instead, we set the rules are multiline regex by default and modifies those single line rules. Ref: https://github.com/dabeaz/ply/issues/282 Change-Id: I7bdbfeb97a9dd74f45c1890a76f8cc16100e5a42 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71019 Reviewed-by: Richard Cooper Tested-by: kokoro Maintainer: Jason Lowe-Power Reviewed-by: Jason Lowe-Power --- src/arch/isa_parser/isa_parser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/arch/isa_parser/isa_parser.py b/src/arch/isa_parser/isa_parser.py index 0f29840c3b..5be50a11bf 100755 --- a/src/arch/isa_parser/isa_parser.py +++ b/src/arch/isa_parser/isa_parser.py @@ -514,6 +514,7 @@ class InstObjParams(object): class ISAParser(Grammar): def __init__(self, output_dir): super().__init__() + self.lex_kwargs["reflags"] = int(re.MULTILINE) self.output_dir = output_dir self.filename = None # for output file watermarking/scaremongering @@ -851,7 +852,7 @@ class ISAParser(Grammar): # String literal. Note that these use only single quotes, and # can span multiple lines. def t_STRLIT(self, t): - r"(?m)'([^'])+'" + r"'([^'])+'" # strip off quotes t.value = t.value[1:-1] t.lexer.lineno += t.value.count("\n") @@ -860,19 +861,19 @@ class ISAParser(Grammar): # "Code literal"... like a string literal, but delimiters are # '{{' and '}}' so they get formatted nicely under emacs c-mode def t_CODELIT(self, t): - r"(?m)\{\{([^\}]|}(?!\}))+\}\}" + r"\{\{([^\}]|}(?!\}))+\}\}" # strip off {{ & }} t.value = t.value[2:-2] t.lexer.lineno += t.value.count("\n") return t def t_CPPDIRECTIVE(self, t): - r"^\#[^\#].*\n" + r"^\#[^\#][^\n]*\n" t.lexer.lineno += t.value.count("\n") return t def t_NEWFILE(self, t): - r'^\#\#newfile\s+"[^"]*"\n' + r'^\#\#newfile\s+"[^"\n]*"\n' self.fileNameStack.push(t.lexer.lineno) t.lexer.lineno = LineTracker(t.value[11:-2]) @@ -892,7 +893,7 @@ class ISAParser(Grammar): # Comments def t_comment(self, t): - r"//.*" + r"//[^\n]*\n" # Completely ignored characters t_ignore = " \t\x0c" From b9db0976ca2116d111b3af2eb78d549df22ce0a6 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Tue, 13 Jun 2023 21:04:30 +0000 Subject: [PATCH 12/55] stdlib: Add a prebuilt MESI_Three_Level cache The cache is modeled after an AMD EPYC cache, but not exactly like AMD EPYC cache. - K cores per core complex (CCD), each core has one private split L1, and one private L2. - K cores in the same CCD share 1 slice of L3 cache, which is not a victim cache. - There can be multiple CCDs, which communicate with each other via Cross-CCD router. The Cross-CCD rounter is also connected to directory controllers and dma controllers. - All links latency are set to 1. Change-Id: Ib64248bed9155b8e48e5158ffdeebf1f2d770754 Signed-off-by: Hoa Nguyen Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71598 Maintainer: Jason Lowe-Power Tested-by: kokoro Reviewed-by: Jason Lowe-Power --- src/python/SConscript | 12 + .../caches/prebuilt/octopi_cache/__init__.py | 25 ++ .../prebuilt/octopi_cache/core_complex.py | 245 +++++++++++++++++ .../caches/prebuilt/octopi_cache/octopi.py | 257 ++++++++++++++++++ .../prebuilt/octopi_cache/octopi_network.py | 67 +++++ .../octopi_cache/ruby_network_components.py | 111 ++++++++ 6 files changed, 717 insertions(+) create mode 100644 src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/__init__.py create mode 100644 src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/core_complex.py create mode 100644 src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi.py create mode 100644 src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi_network.py create mode 100644 src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/ruby_network_components.py diff --git a/src/python/SConscript b/src/python/SConscript index f98b5700c7..ea9d3d4021 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -154,6 +154,18 @@ PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', 'gem5/components/cachehierarchies/ruby/caches/mesi_three_level/' 'l3_cache.py') +PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', + 'gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/' + 'octopi.py') +PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', + 'gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/' + 'core_complex.py') +PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', + 'gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/' + 'octopi_network.py') +PySource('gem5.components.cachehierarchies.ruby.caches.mesi_three_level', + 'gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/' + 'ruby_network_components.py') PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', 'gem5/components/cachehierarchies/ruby/caches/mi_example/__init__.py') PySource('gem5.components.cachehierarchies.ruby.caches.mi_example', diff --git a/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/__init__.py b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/__init__.py new file mode 100644 index 0000000000..b08f46b513 --- /dev/null +++ b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022-2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/core_complex.py b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/core_complex.py new file mode 100644 index 0000000000..f056d76f98 --- /dev/null +++ b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/core_complex.py @@ -0,0 +1,245 @@ +# Copyright (c) 2022-2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import List, Tuple + +from gem5.isas import ISA +from gem5.components.boards.abstract_board import AbstractBoard +from gem5.components.processors.abstract_core import AbstractCore +from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l1_cache import ( + L1Cache, +) +from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l2_cache import ( + L2Cache, +) +from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.l3_cache import ( + L3Cache, +) + +from m5.objects import SubSystem, RubySequencer + +from .ruby_network_components import ( + RubyRouter, + RubyExtLink, + RubyIntLink, + RubyNetworkComponent, +) + + +class CoreComplex(SubSystem, RubyNetworkComponent): + _core_id = 0 + _core_complex_id = 0 + + @classmethod + def _get_core_id(cls): + cls._core_id += 1 + return cls._core_id - 1 + + @classmethod + def _get_core_complex_id(cls): + cls._core_complex_id += 1 + return cls._core_complex_id - 1 + + def __init__( + self, + board: AbstractBoard, + cores: List[AbstractCore], + ruby_system, + l1i_size: str, + l1i_assoc: int, + l1d_size: str, + l1d_assoc: int, + l2_size: str, + l2_assoc: int, + l3_size: str, + l3_assoc: int, + ): + SubSystem.__init__(self=self) + RubyNetworkComponent.__init__(self=self) + + self._l1i_size = l1i_size + self._l1i_assoc = l1i_assoc + self._l1d_size = l1d_size + self._l1d_assoc = l1d_assoc + self._l2_size = l2_size + self._l2_assoc = l2_assoc + self._l3_size = l3_size + self._l3_assoc = l3_assoc + + self._board = board + self._cores = cores + self._ruby_system = ruby_system + self._cache_line_size = 64 + + self._directory_controllers = [] + + self._core_complex_id = self._get_core_complex_id() + self.main_router = RubyRouter( + self._ruby_system + ) # this will be connect to component outside the core complex + self._add_router(self.main_router) + self._create_core_complex() + + def get_main_router(self): + return self.main_router + + def _create_core_complex(self): + # Create L1 caches, L2 cache, and corresponding controllers per core + self.core_clusters = [ + self._create_core_cluster(core) for core in self._cores + ] + # Create L3 cache and its corresponding controller + self._create_shared_cache() + # Setting up one router and one external link per controller + self._create_external_links() + # Setting up L1/L2 links, L2/main links, L3/main link + self._create_internal_links() + + def _create_core_cluster(self, core: AbstractCore): + cluster = SubSystem() + core_id = self._get_core_id() + + cluster.l1_cache = L1Cache( + l1i_size=self._l1i_size, + l1i_assoc=self._l1i_assoc, + l1d_size=self._l1d_size, + l1d_assoc=self._l1d_assoc, + network=self._ruby_system.network, + core=core, + cache_line_size=self._cache_line_size, + target_isa=self._board.processor.get_isa(), + clk_domain=self._board.get_clock_domain(), + ) + cluster.l1_cache.sequencer = RubySequencer( + version=core_id, + dcache=cluster.l1_cache.Dcache, + clk_domain=cluster.l1_cache.clk_domain, + ) + + if self._board.has_io_bus(): + cluster.l1_cache.sequencer.connectIOPorts(self._board.get_io_bus()) + cluster.l1_cache.ruby_system = self._ruby_system + core.connect_icache(cluster.l1_cache.sequencer.in_ports) + core.connect_dcache(cluster.l1_cache.sequencer.in_ports) + core.connect_walker_ports( + cluster.l1_cache.sequencer.in_ports, + cluster.l1_cache.sequencer.in_ports, + ) + if self._board.get_processor().get_isa() == ISA.X86: + core.connect_interrupt( + cluster.l1_cache.sequencer.interrupt_out_port, + cluster.l1_cache.sequencer.in_ports, + ) + else: + core.connect_interrupt() + + cluster.l2_cache = L2Cache( + l2_size=self._l2_size, + l2_assoc=self._l2_assoc, + network=self._ruby_system.network, + core=core, + num_l3Caches=1, # each core complex has 1 slice of L3 Cache + cache_line_size=self._cache_line_size, + cluster_id=self._core_complex_id, + target_isa=self._board.processor.get_isa(), + clk_domain=self._board.get_clock_domain(), + ) + cluster.l2_cache.ruby_system = self._ruby_system + # L0Cache in the ruby backend is l1 cache in stdlib + # L1Cache in the ruby backend is l2 cache in stdlib + cluster.l2_cache.bufferFromL0 = cluster.l1_cache.bufferToL1 + cluster.l2_cache.bufferToL0 = cluster.l1_cache.bufferFromL1 + + return cluster + + def _create_shared_cache(self): + self.l3_cache = L3Cache( + l3_size=self._l3_size, + l3_assoc=self._l3_assoc, + network=self._ruby_system.network, + num_l3Caches=1, + cache_line_size=self._cache_line_size, + cluster_id=self._core_complex_id, + ) + self.l3_cache.ruby_system = self._ruby_system + + # This is where all routers and links are created + def _create_external_links(self): + # create a router per cache controller + # - there is one L3 per ccd + self.l3_router = RubyRouter(self._ruby_system) + self._add_router(self.l3_router) + # - there is one L1 and one L2 per cluster + for cluster in self.core_clusters: + cluster.l1_router = RubyRouter(self._ruby_system) + self._add_router(cluster.l1_router) + cluster.l2_router = RubyRouter(self._ruby_system) + self._add_router(cluster.l2_router) + + # create an ext link from a controller to a router + self.l3_router_link = RubyExtLink( + ext_node=self.l3_cache, + int_node=self.l3_router, + bandwidth_factor=64, + ) + self._add_ext_link(self.l3_router_link) + for cluster in self.core_clusters: + cluster.l1_router_link = RubyExtLink( + ext_node=cluster.l1_cache, int_node=cluster.l1_router + ) + self._add_ext_link(cluster.l1_router_link) + cluster.l2_router_link = RubyExtLink( + ext_node=cluster.l2_cache, int_node=cluster.l2_router + ) + self._add_ext_link(cluster.l2_router_link) + + def _create_internal_links(self): + # create L1/L2 links + for cluster in self.core_clusters: + l1_to_l2, l2_to_l1 = RubyIntLink.create_bidirectional_links( + cluster.l1_router, cluster.l2_router + ) + cluster.l1_to_l2_link = l1_to_l2 + cluster.l2_to_l1_link = l2_to_l1 + self._add_int_link(l1_to_l2) + self._add_int_link(l2_to_l1) + # create L2/main_router links + for cluster in self.core_clusters: + l2_to_main, main_to_l2 = RubyIntLink.create_bidirectional_links( + cluster.l2_router, self.main_router + ) + cluster.l2_to_main_link = l2_to_main + cluster.main_to_l2_link = main_to_l2 + self._add_int_link(l2_to_main) + self._add_int_link(main_to_l2) + # create L3/main_router link + l3_to_main, main_to_l3 = RubyIntLink.create_bidirectional_links( + self.l3_router, self.main_router, bandwidth_factor=64 + ) + self.l3_to_main_link = l3_to_main + self.main_to_l3_link = main_to_l3 + self._add_int_link(l3_to_main) + self._add_int_link(main_to_l3) diff --git a/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi.py b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi.py new file mode 100644 index 0000000000..9c8b93d812 --- /dev/null +++ b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi.py @@ -0,0 +1,257 @@ +# Copyright (c) 2022-2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from ...abstract_ruby_cache_hierarchy import AbstractRubyCacheHierarchy +from ....abstract_three_level_cache_hierarchy import ( + AbstractThreeLevelCacheHierarchy, +) +from ......coherence_protocol import CoherenceProtocol +from ......components.boards.abstract_board import AbstractBoard +from ......utils.requires import requires + +from ......components.cachehierarchies.ruby.caches.mesi_three_level.directory import ( + Directory, +) +from ......components.cachehierarchies.ruby.caches.mesi_three_level.dma_controller import ( + DMAController, +) + +from m5.objects import RubySystem, DMASequencer, RubyPortProxy + +from .core_complex import CoreComplex +from .octopi_network import OctopiNetwork +from .ruby_network_components import RubyRouter, RubyExtLink, RubyIntLink + +# CoreComplex sub-systems own the L1, L2, L3 controllers +# OctopiCache owns the directory controllers +# RubySystem owns the DMA Controllers +class OctopiCache( + AbstractRubyCacheHierarchy, AbstractThreeLevelCacheHierarchy +): + def __init__( + self, + l1i_size: str, + l1i_assoc: int, + l1d_size: str, + l1d_assoc: int, + l2_size: str, + l2_assoc: int, + l3_size: str, + l3_assoc: int, + num_core_complexes: int, + is_fullsystem: bool, + ): + AbstractRubyCacheHierarchy.__init__(self=self) + AbstractThreeLevelCacheHierarchy.__init__( + self=self, + l1i_size=l1i_size, + l1i_assoc=l1i_assoc, + l1d_size=l1d_size, + l1d_assoc=l1d_assoc, + l2_size=l2_size, + l2_assoc=l2_assoc, + l3_size=l3_size, + l3_assoc=l3_assoc, + ) + + self._directory_controllers = [] + self._dma_controllers = [] + self._io_controllers = [] + self._core_complexes = [] + self._num_core_complexes = num_core_complexes + self._is_fullsystem = is_fullsystem + + def incorporate_cache(self, board: AbstractBoard) -> None: + + requires( + coherence_protocol_required=CoherenceProtocol.MESI_THREE_LEVEL + ) + + cache_line_size = board.get_cache_line_size() + + self.ruby_system = RubySystem() + # MESI_Three_Level needs 3 virtual networks + self.ruby_system.number_of_virtual_networks = 3 + self.ruby_system.network = OctopiNetwork(self.ruby_system) + + # Setting up the core complex + all_cores = board.get_processor().get_cores() + num_cores_per_core_complex = len(all_cores) // self._num_core_complexes + + self.core_complexes = [ + CoreComplex( + board=board, + cores=all_cores[ + core_complex_idx + * num_cores_per_core_complex : (core_complex_idx + 1) + * num_cores_per_core_complex + ], + ruby_system=self.ruby_system, + l1i_size=self._l1i_size, + l1i_assoc=self._l1i_assoc, + l1d_size=self._l1d_size, + l1d_assoc=self._l1d_assoc, + l2_size=self._l2_size, + l2_assoc=self._l2_assoc, + l3_size=self._l3_size, + l3_assoc=self._l3_assoc, + ) + for core_complex_idx in range(self._num_core_complexes) + ] + + self.ruby_system.network.incorporate_ccds(self.core_complexes) + + self._create_directory_controllers(board) + self._create_dma_controllers(board, self.ruby_system) + + self.ruby_system.num_of_sequencers = ( + len(all_cores) + + len(self._dma_controllers) + + len(self._io_controllers) + ) + # SimpleNetwork requires .int_links and .routers to exist + # if we want to call SimpleNetwork.setup_buffers() + self.ruby_system.network.int_links = ( + self.ruby_system.network._int_links + ) + self.ruby_system.network.ext_links = ( + self.ruby_system.network._ext_links + ) + self.ruby_system.network.routers = self.ruby_system.network._routers + self.ruby_system.network.setup_buffers() + + # Set up a proxy port for the system_port. Used for load binaries and + # other functional-only things. + self.ruby_system.sys_port_proxy = RubyPortProxy() + board.connect_system_port(self.ruby_system.sys_port_proxy.in_ports) + + def _create_directory_controllers(self, board): + # Adding controllers + self.directory_controllers = [ + Directory( + self.ruby_system.network, + board.get_cache_line_size(), + addr_range, + mem_port, + ) + for addr_range, mem_port in board.get_mem_ports() + ] + for ctrl in self.directory_controllers: + ctrl.ruby_system = self.ruby_system + # Adding controller routers + self.directory_controller_routers = [ + RubyRouter(self.ruby_system.network) + for _ in range(len(self.directory_controllers)) + ] + for router in self.directory_controller_routers: + self.ruby_system.network._add_router(router) + # Adding an external link for each controller and its router + self.directory_controller_ext_links = [ + RubyExtLink(ext_node=dir_ctrl, int_node=dir_router) + for dir_ctrl, dir_router in zip( + self.directory_controllers, self.directory_controller_routers + ) + ] + for ext_link in self.directory_controller_ext_links: + self.ruby_system.network._add_ext_link(ext_link) + _directory_controller_int_links = [] + for router in self.directory_controller_routers: + int_link_1, int_link_2 = RubyIntLink.create_bidirectional_links( + router, self.ruby_system.network.cross_ccd_router + ) + _directory_controller_int_links.extend([int_link_1, int_link_2]) + self.ruby_system.network._add_int_link(int_link_1) + self.ruby_system.network._add_int_link(int_link_2) + self.directory_controller_int_links = _directory_controller_int_links + + def _create_dma_controllers(self, board, ruby_system): + # IOController for full system simulation + if self._is_fullsystem: + self.io_sequencer = DMASequencer( + version=0, ruby_system=self.ruby_system + ) + self.io_sequencer.in_ports = board.get_mem_side_coherent_io_port() + self.ruby_system.io_controller = DMAController( + dma_sequencer=self.io_sequencer, ruby_system=self.ruby_system + ) + self._io_controllers.append(self.ruby_system.io_controller) + self.io_controller_router = RubyRouter(self.ruby_system.network) + self.ruby_system.network._add_router(self.io_controller_router) + self.io_controller_ext_link = RubyExtLink( + ext_node=self._io_controllers[0], + int_node=self.io_controller_router, + ) + self.ruby_system.network._add_ext_link(self.io_controller_ext_link) + self.io_controller_int_links = ( + RubyIntLink.create_bidirectional_links( + self.io_controller_router, + self.ruby_system.network.cross_ccd_router, + ) + ) + self.ruby_system.network._add_int_link( + self.io_controller_int_links[0] + ) + self.ruby_system.network._add_int_link( + self.io_controller_int_links[1] + ) + + self._dma_controllers = [] + if board.has_dma_ports(): + self.ruby_system.dma_controllers = [ + DMAController( + dma_sequencer=DMASequencer(version=i + 1, in_ports=port), + ruby_system=self.ruby_system, + ) + for i, port in enumerate(board.get_dma_ports()) + ] + self._dma_controllers = self.ruby_system.dma_controllers + self.dma_routers = [ + RubyRouter(self.ruby_system.network) + for dma_controller in self._dma_controllers + ] + for dma_router in self.dma_routers: + self.ruby_system.network._add_router(dma_router) + self.dma_ext_links = [ + RubyExtLink(ext_node=dma_controller, int_node=dma_router) + for dma_controller, dma_router in zip( + self._dma_controllers, self.dma_routers + ) + ] + for link in self.dma_ext_links: + self.ruby_system.network._add_ext_link(link) + self.dma_int_links = [ + RubyIntLink( + dma_router, self.ruby_system.network.cross_ccd_router + ) + for dma_router in self.dma_routers + ] + [ + RubyIntLink( + self.ruby_system.network.cross_ccd_router, dma_router + ) + for dma_router in self.dma_routers + ] + for link in self.dma_int_links: + self.ruby_system.network._add_int_link(link) diff --git a/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi_network.py b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi_network.py new file mode 100644 index 0000000000..745ef826c7 --- /dev/null +++ b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/octopi_network.py @@ -0,0 +1,67 @@ +# Copyright (c) 2022-2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from m5.objects import SimpleNetwork + +from .ruby_network_components import ( + RubyNetworkComponent, + RubyRouter, + RubyIntLink, +) + +# . The Network owns all routers, all int links and all ext links that are not in CCD's. +# . The CCD subsystems are not of type RubyNetwork, so we need to copy the references of +# routers and links to OctopiNetwork._routers, ._int_links, and ._ext_links; which will +# be, in turns, copied to RubyNetwork.routers, .int_links, and .ext_links respectively. +class OctopiNetwork(SimpleNetwork, RubyNetworkComponent): + def __init__(self, ruby_system): + SimpleNetwork.__init__(self=self) + RubyNetworkComponent.__init__(self=self) + self.netifs = [] + self.ruby_system = ruby_system + self.number_of_virtual_networks = ( + ruby_system.number_of_virtual_networks + ) + + self.cross_ccd_router = RubyRouter(self) + self._add_router(self.cross_ccd_router) + + def connect_ccd_routers_to_cross_ccd_router(self, ccds): + for ccd in ccds: + int_link_1, int_link_2 = RubyIntLink.create_bidirectional_links( + self.cross_ccd_router, + ccd.get_main_router(), + bandwidth_factor=64, + ) + ccd.to_cross_ccd_router_link = int_link_1 + ccd.from_cross_ccd_router_link = int_link_2 + self._add_int_link(int_link_1) + self._add_int_link(int_link_2) + + def incorporate_ccds(self, ccds): + for ccd in ccds: + self.incorporate_ruby_subsystem(ccd) + self.connect_ccd_routers_to_cross_ccd_router(ccds) diff --git a/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/ruby_network_components.py b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/ruby_network_components.py new file mode 100644 index 0000000000..8a413ea59d --- /dev/null +++ b/src/python/gem5/components/cachehierarchies/ruby/caches/prebuilt/octopi_cache/ruby_network_components.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022-2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from m5.objects import Switch, SimpleIntLink, SimpleExtLink + + +class RubyNetworkComponent: + def __init__(self): + super().__init__() + self._routers = [] + self._ext_links = [] + self._int_links = [] + + def _add_router(self, router): + self._routers.append(router) + + def _add_ext_link(self, link): + self._ext_links.append(link) + + def _add_int_link(self, link): + self._int_links.append(link) + + def get_routers(self): + return self._routers + + def get_ext_links(self): + return self._ext_links + + def get_int_links(self): + return self._int_links + + def incorporate_ruby_subsystem(self, other_ruby_subsystem): + self._routers.extend(other_ruby_subsystem.get_routers()) + self._ext_links.extend(other_ruby_subsystem.get_ext_links()) + self._int_links.extend(other_ruby_subsystem.get_int_links()) + + +class RubyRouter(Switch): + _router_id = 0 + + @classmethod + def _get_router_id(cls): + cls._router_id += 1 + return cls._router_id - 1 + + def __init__(self, network): + super().__init__() + self.router_id = self._get_router_id() + self.virt_nets = network.number_of_virtual_networks + + +class RubyExtLink(SimpleExtLink): + _link_id = 0 + + @classmethod + def _get_link_id(cls): + cls._link_id += 1 + return cls._link_id - 1 + + def __init__(self, ext_node, int_node, bandwidth_factor=16): + super().__init__() + self.link_id = self._get_link_id() + self.ext_node = ext_node + self.int_node = int_node + self.bandwidth_factor = bandwidth_factor + + +class RubyIntLink(SimpleIntLink): + _link_id = 0 + + @classmethod + def _get_link_id(cls): + cls._link_id += 1 + return cls._link_id - 1 + + @classmethod + def create_bidirectional_links(cls, node_1, node_2, bandwidth_factor=16): + return [ + RubyIntLink(node_1, node_2, bandwidth_factor), + RubyIntLink(node_2, node_1, bandwidth_factor), + ] + + def __init__(self, src_node, dst_node, bandwidth_factor=16): + super().__init__() + self.link_id = self._get_link_id() + self.src_node = src_node + self.dst_node = dst_node + self.bandwidth_factor = bandwidth_factor From 5cc4d5dd7786ed31bb42704f183e1feead8531be Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Tue, 13 Jun 2023 21:43:39 +0000 Subject: [PATCH 13/55] configs: Add example configuration for OctopiCache Change-Id: Ia78dd63e63808ebad40052d2a7cdb67cc7179e44 Signed-off-by: Hoa Nguyen Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71618 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power --- .../caches/octopi-cache-example.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 configs/example/gem5_library/caches/octopi-cache-example.py diff --git a/configs/example/gem5_library/caches/octopi-cache-example.py b/configs/example/gem5_library/caches/octopi-cache-example.py new file mode 100644 index 0000000000..1b39a8bee5 --- /dev/null +++ b/configs/example/gem5_library/caches/octopi-cache-example.py @@ -0,0 +1,100 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +This script boots Ubuntu 20.04 with 8 timing cores in 1 CCD. + +Usage +----- + +``` +scons build/ARM_MESI_Three_Level/gem5.opt -j `nproc` +./build/ARM_MESI_Three_Level/gem5.opt \ + configs/example/gem5_library/caches/octopi-cache-example.py +``` +""" + + +from m5.objects import ArmDefaultRelease, VExpress_GEM5_Foundation + +from gem5.utils.requires import requires +from gem5.components.boards.arm_board import ArmBoard +from gem5.components.memory import DualChannelDDR4_2400 +from gem5.components.processors.simple_processor import SimpleProcessor +from gem5.components.processors.cpu_types import CPUTypes +from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.octopi import ( + OctopiCache, +) +from gem5.isas import ISA +from gem5.coherence_protocol import CoherenceProtocol +from gem5.simulate.simulator import Simulator +from gem5.resources.workload import Workload + +num_ccds = 1 # CCDs +num_cores_per_ccd = 8 # 8 cores/CCD + +# OctopiCache is built on top of gem5's MESI_Three_Level cache coherence +# protocol +requires(coherence_protocol_required=CoherenceProtocol.MESI_THREE_LEVEL) +cache_hierarchy = OctopiCache( + l1i_size="32KiB", + l1i_assoc=8, + l1d_size="32KiB", + l1d_assoc=8, + l2_size="512KiB", + l2_assoc=8, + l3_size="32MiB", + l3_assoc=16, + num_core_complexes=num_ccds, + is_fullsystem=True, +) + +memory = DualChannelDDR4_2400(size="16GB") + +# The number of cores must be consistent with +# num_core_complexes and num_cores_per_core_complexes +processor = SimpleProcessor( + cpu_type=CPUTypes.TIMING, + isa=ISA.ARM, + num_cores=num_ccds * num_cores_per_ccd, +) + +release = ArmDefaultRelease() +platform = VExpress_GEM5_Foundation() + +board = ArmBoard( + clk_freq="4GHz", + processor=processor, + memory=memory, + cache_hierarchy=cache_hierarchy, + release=release, + platform=platform, +) + +board.set_workload(Workload("arm64-ubuntu-20.04-boot")) + +simulator = Simulator(board=board) +simulator.run() From 1348f3a03eb9487e57237ee6cac0585a310d0d6c Mon Sep 17 00:00:00 2001 From: Yan Lee Date: Tue, 23 May 2023 01:07:45 -0700 Subject: [PATCH 14/55] mem: port: add TracingExtension for debug purpose TracingExtension contains a stack recording the port names passed through of the Packet. The target receiving the Packet can dump out the whole path of this Packet for the debug purpose. This mechanism can be enabled with the debug flag PortTrace. Change-Id: Ic11e708b35fdddc4f4b786d91b35fd4def08948c Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71538 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Tested-by: kokoro Reviewed-by: Yu-hsin Wang --- src/mem/SConscript | 1 + src/mem/port.cc | 24 +++++++++++++++ src/mem/port.hh | 73 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/mem/SConscript b/src/mem/SConscript index 351f24e907..6e017e0397 100644 --- a/src/mem/SConscript +++ b/src/mem/SConscript @@ -151,6 +151,7 @@ DebugFlag('MemCtrl') DebugFlag('MMU') DebugFlag('MemoryAccess') DebugFlag('PacketQueue') +DebugFlag("PortTrace") DebugFlag('ResponsePort') DebugFlag('StackDist') DebugFlag("DRAMSim2") diff --git a/src/mem/port.cc b/src/mem/port.cc index 2a253b91a3..2f754c89df 100644 --- a/src/mem/port.cc +++ b/src/mem/port.cc @@ -45,6 +45,7 @@ #include "mem/port.hh" #include "base/trace.hh" +#include "debug/PortTrace.hh" #include "debug/ResponsePort.hh" #include "sim/sim_object.hh" @@ -186,6 +187,29 @@ RequestPort::printAddr(Addr a) sendFunctional(&pkt); } +void +RequestPort::addTrace(PacketPtr pkt) const +{ + if (!gem5::debug::PortTrace || !pkt) + return; + auto ext = pkt->getExtension(); + if (!ext) { + ext = std::make_shared(); + pkt->setExtension(ext); + } + ext->add(name(), _responsePort->name()); +} + +void +RequestPort::removeTrace(PacketPtr pkt) const +{ + if (!gem5::debug::PortTrace || !pkt) + return; + auto ext = pkt->getExtension(); + panic_if(!ext, "There is no TracingExtension in the packet."); + ext->remove(); +} + /** * Response port */ diff --git a/src/mem/port.hh b/src/mem/port.hh index a3acffc427..2555706444 100644 --- a/src/mem/port.hh +++ b/src/mem/port.hh @@ -46,6 +46,10 @@ #ifndef __MEM_PORT_HH__ #define __MEM_PORT_HH__ +#include +#include +#include + #include "base/addr_range.hh" #include "mem/packet.hh" #include "mem/protocol/atomic.hh" @@ -64,6 +68,43 @@ class SlavePort; class ResponsePort; +/** + * TracingExtension is an Extension of the Packet for recording the trace + * of the Packet. The stack in the TracingExtension holds the name of the + * ports that the Packet has passed through. + */ +class TracingExtension : public gem5::Extension +{ + public: + TracingExtension() = default; + TracingExtension(const std::stack& q) { trace_ = q; } + + std::unique_ptr clone() const override + { + return std::make_unique(trace_); + } + + void + add(std::string request_port, std::string response_port) + { + trace_.push(request_port); + trace_.push(response_port); + } + + void + remove() + { + trace_.pop(); // Remove the response port name. + trace_.pop(); // Remove the request port name. + } + + bool empty() { return trace_.empty(); } + std::stack& getTrace() { return trace_; } + + private: + std::stack trace_; +}; + /** * A RequestPort is a specialisation of a Port, which * implements the default protocol for the three different level of @@ -266,6 +307,10 @@ class RequestPort: public Port, public AtomicRequestProtocol, { panic("%s was not expecting a snoop retry.\n", name()); } + + private: + void addTrace(PacketPtr pkt) const; + void removeTrace(PacketPtr pkt) const; }; class [[deprecated]] MasterPort : public RequestPort @@ -393,7 +438,11 @@ class ResponsePort : public Port, public AtomicResponseProtocol, sendTimingResp(PacketPtr pkt) { try { - return TimingResponseProtocol::sendResp(_requestPort, pkt); + _requestPort->removeTrace(pkt); + bool succ = TimingResponseProtocol::sendResp(_requestPort, pkt); + if (!succ) + _requestPort->addTrace(pkt); + return succ; } catch (UnboundPortException) { reportUnbound(); } @@ -487,7 +536,10 @@ inline Tick RequestPort::sendAtomic(PacketPtr pkt) { try { - return AtomicRequestProtocol::send(_responsePort, pkt); + addTrace(pkt); + Tick tick = AtomicRequestProtocol::send(_responsePort, pkt); + removeTrace(pkt); + return tick; } catch (UnboundPortException) { reportUnbound(); } @@ -497,8 +549,11 @@ inline Tick RequestPort::sendAtomicBackdoor(PacketPtr pkt, MemBackdoorPtr &backdoor) { try { - return AtomicRequestProtocol::sendBackdoor(_responsePort, - pkt, backdoor); + addTrace(pkt); + Tick tick = AtomicRequestProtocol::sendBackdoor(_responsePort, + pkt, backdoor); + removeTrace(pkt); + return tick; } catch (UnboundPortException) { reportUnbound(); } @@ -508,7 +563,9 @@ inline void RequestPort::sendFunctional(PacketPtr pkt) const { try { - return FunctionalRequestProtocol::send(_responsePort, pkt); + addTrace(pkt); + FunctionalRequestProtocol::send(_responsePort, pkt); + removeTrace(pkt); } catch (UnboundPortException) { reportUnbound(); } @@ -530,7 +587,11 @@ inline bool RequestPort::sendTimingReq(PacketPtr pkt) { try { - return TimingRequestProtocol::sendReq(_responsePort, pkt); + addTrace(pkt); + bool succ = TimingRequestProtocol::sendReq(_responsePort, pkt); + if (!succ) + removeTrace(pkt); + return succ; } catch (UnboundPortException) { reportUnbound(); } From 399f57fd9372c1a53c6c4e9e7fbde4ccded7bd95 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Mon, 26 Jun 2023 17:17:18 -0700 Subject: [PATCH 15/55] resources: Catch ConnectionResourceError in downloading resources This handles an error we see within GitHub Actions that occassionally occurs when downloading resources. We retry in the same way we do when handling HTTPErrors. Change-Id: I4dce5d607ccc41ad53b51e39082c486e644d815c Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71858 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power --- src/python/gem5/resources/downloader.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index bb5ca84cc0..d606f34717 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -139,6 +139,22 @@ def _download(url: str, download_to: str, max_attempts: int = 6) -> None: time.sleep((2**attempt) + random.uniform(0, 1)) else: raise e + except ConnectionResetError as e: + # This catches the ConnectionResetError we see occassionally see + # when accessing resources on GitHub Actions. It retries using a + # Truncated Exponential backoff algorithm, truncating after + # "max_attempts". If any other is retrieved we raise the error. + if e.errno == 104: + attempt += 1 + if attempt >= max_attempts: + raise Exception( + f"After {attempt} attempts, the resource json could " + "not be retrieved. OS Error Code retrieved: " + f"{e.errno}" + ) + time.sleep((2**attempt) + random.uniform(0, 1)) + else: + raise e except ValueError as e: raise Exception( "Environment variable GEM5_USE_PROXY is set to " From 16de9b5274cb846d54275bc7e510810552a9646b Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Tue, 27 Jun 2023 10:24:25 -0700 Subject: [PATCH 16/55] resources: Output error message in downloader.py This allows for the actual error message to be output in addition to the output gem5 has on ValueErrors and ImportErrors. Change-Id: Ic52f5646aa41dbf7c217ab27d142c0a18fa24c55 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71859 Tested-by: kokoro Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power --- src/python/gem5/resources/downloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/gem5/resources/downloader.py b/src/python/gem5/resources/downloader.py index d606f34717..1aeb487e61 100644 --- a/src/python/gem5/resources/downloader.py +++ b/src/python/gem5/resources/downloader.py @@ -157,12 +157,14 @@ def _download(url: str, download_to: str, max_attempts: int = 6) -> None: raise e except ValueError as e: raise Exception( + f"ValueError: {e}\n" "Environment variable GEM5_USE_PROXY is set to " f"'{use_proxy}'. The expected form is " ":'." ) except ImportError as e: raise Exception( + f"ImportError: {e}\n" "An import error has occurred. This is likely due " "the Python SOCKS client module not being " "installed. It can be installed with " From ba63c13b5ba038e87dd666db31d3df600f3b41b7 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 12:15:31 -0700 Subject: [PATCH 17/55] util: Update GitHub Runners Vagrant to overcommit memory SE mode tests were failing in some cases where the VM did not have enough memory to satisfy the constraints of the simulated system. This change ensures the VM allows overcommitting of memory. Change-Id: I1800288e16146bdae612a401c2ff282d8664892d Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71498 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Maintainer: Jason Lowe-Power --- util/github-runners-vagrant/Vagrantfile-builder | 5 +++++ util/github-runners-vagrant/Vagrantfile-runner | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/util/github-runners-vagrant/Vagrantfile-builder b/util/github-runners-vagrant/Vagrantfile-builder index 242e094d19..cd36e7894c 100644 --- a/util/github-runners-vagrant/Vagrantfile-builder +++ b/util/github-runners-vagrant/Vagrantfile-builder @@ -22,6 +22,11 @@ Vagrant.configure("2") do |config| config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. config.vm.provision :reload + config.vm.provision :shell, run: 'always', inline: <<-SHELL + # When running gem5 in SE mode we must overcommit memory. + # This is run on every startup of the VM. + /sbin/sysctl vm.overcommit_memory=1 + SHELL config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL if [ -d ~/actions-runner ]; then # This will be run everytime the VM is run (once created). diff --git a/util/github-runners-vagrant/Vagrantfile-runner b/util/github-runners-vagrant/Vagrantfile-runner index ab601cd9c8..854e5f58b1 100644 --- a/util/github-runners-vagrant/Vagrantfile-runner +++ b/util/github-runners-vagrant/Vagrantfile-runner @@ -22,6 +22,11 @@ Vagrant.configure("2") do |config| config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. config.vm.provision :reload + config.vm.provision :shell, run: 'always', inline: <<-SHELL + # When running gem5 in SE mode we must overcommit memory. + # This is run on every startup of the VM. + /sbin/sysctl vm.overcommit_memory=1 + SHELL config.vm.provision :shell, privileged: false, run: 'always', inline: <<-SHELL if [ -d ~/actions-runner ]; then # This will be run everytime the VM is run (once created). From 52ef3c3315cfa79ff9c1f20695d5ab78a10165d1 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 12:17:47 -0700 Subject: [PATCH 18/55] util: '-eq' -> '-ge' for if in vm_manager.sh A small nit-pick change that ensures that cases where the number of arguments being >1 does not result in the argument checking being skipped (NOTE: arguments after the first are never processed and are ignored). Change-Id: If7e9c16c2c3581ea95ed888586736618d1ae5f5f Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71499 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Reviewed-by: Daniel Carvalho Tested-by: kokoro --- util/github-runners-vagrant/vm_manager.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/github-runners-vagrant/vm_manager.sh b/util/github-runners-vagrant/vm_manager.sh index 1cae8d2686..0e2f4b747f 100755 --- a/util/github-runners-vagrant/vm_manager.sh +++ b/util/github-runners-vagrant/vm_manager.sh @@ -9,7 +9,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) export VAGRANT_HOME=${SCRIPT_DIR} param="up" -if [[ $# -eq 1 ]]; then +if [[ $# -ge 1 ]]; then param=$1 if [[ "${param}" != "destroy" ]]; then echo "Only valid parameter is 'destroy', to destroy all VMs" From cb12dbbad2844835571e7802f5ba6495fb394851 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 12 Jun 2023 12:19:25 -0700 Subject: [PATCH 19/55] util: Add 'shutdown' argument option to vm_manager.sh This allows for the VMs to be shutdown rather than destroyed. The can be rebooted with `./vm_manager.sh` after shutdown. Change-Id: I58329ec835af664bfb970b029e09ad16c5472015 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71500 Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce Tested-by: kokoro --- util/github-runners-vagrant/vm_manager.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/util/github-runners-vagrant/vm_manager.sh b/util/github-runners-vagrant/vm_manager.sh index 0e2f4b747f..ce46eb3fe7 100755 --- a/util/github-runners-vagrant/vm_manager.sh +++ b/util/github-runners-vagrant/vm_manager.sh @@ -11,8 +11,8 @@ export VAGRANT_HOME=${SCRIPT_DIR} param="up" if [[ $# -ge 1 ]]; then param=$1 - if [[ "${param}" != "destroy" ]]; then - echo "Only valid parameter is 'destroy', to destroy all VMs" + if [[ "${param}" != "destroy" ]] && [[ "${param}" != "shutdown" ]]; then + echo "Only valid parameters are 'destroy' and 'shutdown' to destroy all VMs or shutdown all VMs" exit 1 fi fi @@ -23,6 +23,8 @@ for (( i=1; i<=NUM_RUNNERS; i++ )); do sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${RUNNER_PREFIX}${i}\"/g" Vagrantfile-runner if [[ "${param}" == "destroy" ]]; then VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant destroy -f + elif [[ "${param}" == "shutdown" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant halt -f else VAGRANT_VAGRANTFILE=Vagrantfile-runner vagrant up --provider=libvirt fi @@ -33,6 +35,8 @@ for (( i=1; i<=NUM_BUILDERS; i++ )); do sed -i "s/ config.vm.hostname.*/ config.vm.hostname = \"${BUILDER_PREFIX}${i}\"/g" Vagrantfile-builder if [[ "${param}" == "destroy" ]]; then VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant destroy -f + elif [[ "${param}" == "shutdown" ]]; then + VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant halt -f else VAGRANT_VAGRANTFILE=Vagrantfile-builder vagrant up --provider=libvirt fi From 428bd1c5b8a284e865616e5017766c10f4d9d77d Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 13 Jun 2023 20:19:55 -0700 Subject: [PATCH 20/55] util: Add 'swapspace' daemon to runner VM. As these VMs, particularly the runners, don't have much memory, the 'swapspace' daemon allows for dynamic swap spaces to be created for when more memory is required. Change-Id: Ie8e734a8fde54e122df33dda187c6c4aafdcd006 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71680 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- util/github-runners-vagrant/Vagrantfile-builder | 3 +++ util/github-runners-vagrant/Vagrantfile-runner | 3 +++ 2 files changed, 6 insertions(+) diff --git a/util/github-runners-vagrant/Vagrantfile-builder b/util/github-runners-vagrant/Vagrantfile-builder index cd36e7894c..c0c2380b95 100644 --- a/util/github-runners-vagrant/Vagrantfile-builder +++ b/util/github-runners-vagrant/Vagrantfile-builder @@ -20,6 +20,9 @@ Vagrant.configure("2") do |config| # sets up vm config.vm.provision :shell, path: "provision_root.sh" config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # To ensure we don't run out of memory, we enable dynamic Swap Space. This is + # done via the "swapspace" daemon: https://pqxx.org/development/swapspace/ + config.vm.provision :shell, inline: "sudo apt install swapspace -y" # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. config.vm.provision :reload config.vm.provision :shell, run: 'always', inline: <<-SHELL diff --git a/util/github-runners-vagrant/Vagrantfile-runner b/util/github-runners-vagrant/Vagrantfile-runner index 854e5f58b1..eb4054ed16 100644 --- a/util/github-runners-vagrant/Vagrantfile-runner +++ b/util/github-runners-vagrant/Vagrantfile-runner @@ -20,6 +20,9 @@ Vagrant.configure("2") do |config| # sets up vm config.vm.provision :shell, path: "provision_root.sh" config.vm.provision :shell, privileged: false, path: "provision_nonroot.sh" + # To ensure we don't run out of memory, we enable dynamic Swap Space. This is + # done via the "swapspace" daemon: https://pqxx.org/development/swapspace/ + config.vm.provision :shell, inline: "sudo apt install swapspace -y" # The provision_root.sh adds the vagrant user to the docker group, so we need to reload the VM. config.vm.provision :reload config.vm.provision :shell, run: 'always', inline: <<-SHELL From a9cf1f856809ad1199845f516b53ebd4ae75e8cb Mon Sep 17 00:00:00 2001 From: Wei-Han Chen Date: Wed, 21 Jun 2023 08:20:16 +0000 Subject: [PATCH 21/55] dev: Warn when resp packet is error in dma port This CL adds a warning when the response packet is error. Change-Id: I8e94dc2b85cd1753a4d6265cfda3cd5d6325f425 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71778 Reviewed-by: Jason Lowe-Power Maintainer: Jason Lowe-Power Reviewed-by: Yu-hsin Wang Tested-by: kokoro --- src/dev/dma_device.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dev/dma_device.cc b/src/dev/dma_device.cc index 24e931ef8d..70a74eeb2f 100644 --- a/src/dev/dma_device.cc +++ b/src/dev/dma_device.cc @@ -68,6 +68,7 @@ DmaPort::handleRespPacket(PacketPtr pkt, Tick delay) { // Should always see a response with a sender state. assert(pkt->isResponse()); + warn_if(pkt->isError(), "Response pkt error."); // Get the DMA sender state. auto *state = dynamic_cast(pkt->senderState); From aba56866d4cdde13e66798261611f786bed8d1a9 Mon Sep 17 00:00:00 2001 From: Gabriel Busnot Date: Mon, 10 Oct 2022 15:51:01 +0200 Subject: [PATCH 22/55] configs: Fix default CustomMesh for use with Garnet Garnet routers do not support 0 latency switches. Use 1 instead if the network is garnet. Change-Id: I09841a01eaf413bee0a1629307ecff0ae2bda948 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/67660 Reviewed-by: Jason Lowe-Power Tested-by: kokoro Maintainer: Jason Lowe-Power --- configs/topologies/CustomMesh.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/topologies/CustomMesh.py b/configs/topologies/CustomMesh.py index c62b39a9c2..21fa2dcc13 100644 --- a/configs/topologies/CustomMesh.py +++ b/configs/topologies/CustomMesh.py @@ -171,7 +171,9 @@ class CustomMesh(SimpleTopology): def _createRNFRouter(self, mesh_router): # Create a zero-latency router bridging node controllers # and the mesh router - node_router = self._Router(router_id=len(self._routers), latency=0) + node_router = self._Router( + router_id=len(self._routers), latency=self.node_router_latency + ) self._routers.append(node_router) # connect node_router <-> mesh router @@ -270,6 +272,7 @@ class CustomMesh(SimpleTopology): self._ExtLink = ExtLink self._Router = Router + self.node_router_latency = 1 if options.network == "garnet" else 0 if hasattr(options, "router_link_latency"): self._router_link_latency = options.router_link_latency self._node_link_latency = options.node_link_latency From f4b9605babab1ed56ee2f9be8674e8450eda5b3a Mon Sep 17 00:00:00 2001 From: Gabriel Busnot Date: Tue, 2 Aug 2022 07:07:33 +0000 Subject: [PATCH 23/55] python: Fix namespaced enums params code generation The wrapper_name parameter was not properly handled. Enums were always generated in the enums namespace even if required differently by wrapper_name. Change-Id: I366846ce39dfe10effc2cc145e7772a3fd171b92 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/67662 Tested-by: kokoro Reviewed-by: Bobby Bruce Maintainer: Bobby Bruce --- build_tools/enum_cc.py | 4 ++-- src/python/m5/params.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_tools/enum_cc.py b/build_tools/enum_cc.py index 5d82b401b2..173143fc38 100644 --- a/build_tools/enum_cc.py +++ b/build_tools/enum_cc.py @@ -97,7 +97,7 @@ const char *${name}Strings[static_cast(${name}::Num_${name})] = ) else: code( - """namespace enums + """namespace ${wrapper_name} {""" ) code.indent(1) @@ -112,7 +112,7 @@ code("};") if not enum.wrapper_is_struct and not enum.is_class: code.dedent(1) - code("} // namespace enums") + code("} // namespace ${wrapper_name}") code("} // namespace gem5") diff --git a/src/python/m5/params.py b/src/python/m5/params.py index 2ca6dfcc14..2559b553d2 100644 --- a/src/python/m5/params.py +++ b/src/python/m5/params.py @@ -1560,8 +1560,8 @@ class MetaEnum(MetaParamValue): if cls.is_class: cls.cxx_type = f"{name}" else: - cls.cxx_type = f"enums::{name}" - + scope = init_dict.get("wrapper_name", "enums") + cls.cxx_type = f"{scope}::{name}" super().__init__(name, bases, init_dict) From 61e0a6b4ede3d0d5c2a76a3454621b0ec922f1bd Mon Sep 17 00:00:00 2001 From: mbabaie Date: Wed, 14 Jun 2023 12:07:11 -0700 Subject: [PATCH 24/55] misc: update gem5 links This change updates all of the gerrit links to use github. Change-Id: I2a020dafac0bd2ba99b26c6a9cd4f0c585e253f8 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71719 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- CONTRIBUTING.md | 130 ++++-------------- .../gem5-all-min-dependencies/Dockerfile | 2 +- util/gem5art/artifact/README.md | 2 +- util/gem5art/artifact/setup.py | 4 +- util/gem5art/run/setup.py | 2 +- util/gem5art/tasks/setup.py | 2 +- 6 files changed, 29 insertions(+), 113 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae771d3ffb..c3c4d617d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,8 +67,8 @@ High-level flow for submitting changes | Submit code | +-------------+ -After creating your change to gem5, you can post a review on our Gerrit -code-review site: https://gem5-review.googlesource.com. Before being able to +After creating your change to gem5, you can post a review to git +via a pull request at: https://github.com/gem5/gem5/pulls/. Before being able to submit your code to the mainline of gem5, the code is reviewed by others in the community. Additionally, the maintainer for that part of the code must sign off on it. @@ -77,13 +77,13 @@ Cloning the gem5 repo to contribute =================================== If you plan on contributing, it is strongly encouraged for you to clone the -repository directly, and checkout the `develop` branch from our gerrit instance -at https://gem5.googlesource.com/. +repository directly, and checkout the `develop` branch from our git instance +at https://github.com/gem5/gem5/. To clone the gem5 repository: ``` - git clone https://gem5.googlesource.com/public/gem5 + git clone https://github.com/gem5/gem5/ ``` By default, the stable branch is checked out. The stable branch contains the @@ -156,7 +156,7 @@ The hooks are also automatically installed when gem5 is compiled. When you run a `git commit` command the pre-commit hook will run checks on your committed code. The commit will be blocked if a check fails. -The same checks are run as part of Gerrit's CI tests (those required to obtain +The same checks are run as part of github actions CI tests (those required to obtain a Verified label, necessary for a change to be accepted to the develop branch). Therefore setting up pre-commit in your local gem5 development environment is recommended. @@ -187,8 +187,7 @@ A canonical commit message consists of three parts: less. Leave a blank line between the first short summary line and this detailed description. * Tags describing patch metadata. You are highly recommended to use - tags to acknowledge reviewers for their work. Gerrit will automatically add - most tags. + tags to acknowledge reviewers for their work. Tags are an optional mechanism to store additional metadata about a patch and acknowledge people who reported a bug or reviewed that patch. Tags are @@ -243,7 +242,7 @@ who has contributed a significant amount of code to the file). The up-to-date when you make further changes to files which already have your copyright but with older dates. -Note: If you do not follow these guidelines, the gerrit review site will +Note: If you do not follow these guidelines, the github actions will automatically reject your patch. If this happens, update your changeset descriptions to match the required style and resubmit. The following is a useful git command to update the most recent @@ -263,20 +262,18 @@ See TESTING.md for more information. Posting a review ================ -If you have not signed up for an account on the Gerrit review site -(https://gem5-review.googlesource.com), you first have to create an account. +If you have not signed up for an account on the github +(https://github.com/), you first have to create an account. Setting up an account --------------------- - 1. Go to https://gem5.googlesource.com/ - 2. Click "Sign In" in the upper right corner. Note: You will need a Google - account to contribute. - 3. After signing in, click "Generate Password" and follow the instructions. + 1. Go to https://github.com/ + 2. Click "Sign up" in the upper right corner. Submitting a change ------------------- -In gerrit, to submit a review request, you can simply push your git commits to +In github, to submit a review request, you can simply push your git commits to a special named branch. For more information on git push see https://git-scm.com/docs/git-push. @@ -284,94 +281,13 @@ There are three ways to push your changes to gerrit. Push change to gerrit review ---------------------------- +1. Fork the gem5 repository on GitHub from https://github.com/gem5/gem5/. +2. Create a new branch in your forked repository for your feature or bug fix. +3. Commit your changes to the new branch. +4. Push the branch to your forked repository. +5. Open a pull request from your branch in your forked repository to the main gem5 repository. -``` - git push origin HEAD:refs/for/develop -``` - -Assuming origin is https://gem5.googlesource.com/public/gem5 and you want to -push the changeset at HEAD, this will create a new review request on top of the -develop branch. More generally, - -``` - git push :refs/for/ -``` - -See https://gerrit-review.googlesource.com/Documentation/user-upload.html for -more information. - -Pushing your first change --------------------------- -The first time you push a change you may get the following error: - -``` - remote: ERROR: [fb1366b] missing Change-Id in commit message footer - ... -``` - -Within the error message, there is a command line you should run. For every new -clone of the git repo, you need to run the following command to automatically -insert the change id in the the commit (all on one line). - -``` - curl -Lo `git rev-parse --git-dir`/hooks/commit-msg \ - https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; \ - chmod +x `git rev-parse --git-dir`/hooks/commit-msg -``` - -If you receive the above error, simply run this command and then amend your -changeset. - -``` - git commit --amend -``` - -Push change to gerrit as a Work In Progress -------------------------------------------- - -It is acceptable to push commits as "Work In Progress" (WIP) changes within -gerrit. WIP changes are publicly visible though no one will be able to review -the changes or be directly notified they have been submitted. WIP changes can -be useful for backing up code currently under-development or for sharing -incomplete code with the wider community (i.e., the link to the gerrit change -may be shared, and others may download the change, comment on it, and track -alterations over time). - -See https://gerrit-review.googlesource.com/Documentation/intro-user.html#wip -for details on WIP gerrit changes. - -To push a change as a WIP: - -``` - git push origin HEAD:refs/for/develop%wip -``` - -Once you have pushed your change as a WIP, you can log onto [gerrit]( -https://gem5-review.googlesource.com) and view it. Once you're happy with the -change you can add reviewers which shall move your change from WIP status -to be considered for submission by the wider gem5 community. Switching from a -WIP to a regular change does not notify the gem5 community, via the gem5-dev -mailing-list, that a change has been submitted (as would occur if a change were -submitted directly for review). It is therefore important to include reviewers -and CC those who you wish to view the change (they will be notified -automatically via email). - -Push change bypassing gerrit ------------------------------ - -Only maintainers can bypass gerrit review. This should very rarely be used. - -``` - git push origin HEAD:refs/heads/develop -``` - -Other gerrit push options -------------------------- - -There are a number of options you can specify when uploading your changes to -gerrit (e.g., reviewers, labels). The gerrit documentation has more -information. -https://gerrit-review.googlesource.com/Documentation/user-upload.html +We will continue to use the “develop” branch for development, so please ensure your pull requests are for the gem5 develop branch. Pull requests to the stable branch will be blocked. Branches ======== @@ -413,8 +329,8 @@ development.** Reviewing patches ================= -Reviewing patches is done on our gerrit instance at -https://gem5-review.googlesource.com/. +Reviewing patches is done on our github instance at +https://github.com/gem5/gem5/pulls/. After logging in with your Google account, you will be able to comment, review, and push your own patches as well as review others' patches. All gem5 users are @@ -550,7 +466,7 @@ into the stable branch after two weeks, thus marking the new release. 3. The staging branch will have the full suite of gem5 tests run on it to ensure all tests pass and the to-be-released code is in a decent state. 4. If a user submits a changeset to the staging branch, it will be considered -and undergo the standard Gerrit review process. However, only alterations that +and undergo the standard github review process. However, only alterations that cannot wait until the following release will be accepted for submission into the branch (i.e., submissions to the staging branch for "last minute" inclusions to the release should be of a high priority, such as a critical bug @@ -585,7 +501,7 @@ permitted, the following steps will be taken: 1. A new branch with the prefix "hotfix-" will be created from the stable branch. Only gem5 maintainers can create branches. If a non-maintainer requires the creation of a hotfix branch then they should contact a gem5 maintainer. -2. The change shall be submitted to the hotfix branch via gerrit. Full review, +2. The change shall be submitted to the hotfix branch via github. Full review, as with any other change, will be required. 3. Once fully submitted, the hotfix branch shall be merged into both the develop and the stable branch by a gem5 maintainer. diff --git a/util/dockerfiles/gem5-all-min-dependencies/Dockerfile b/util/dockerfiles/gem5-all-min-dependencies/Dockerfile index da5613e1a1..b28b674fa7 100644 --- a/util/dockerfiles/gem5-all-min-dependencies/Dockerfile +++ b/util/dockerfiles/gem5-all-min-dependencies/Dockerfile @@ -26,7 +26,7 @@ FROM gcr.io/gem5-test/ubuntu-22.04_min-dependencies:latest as source RUN apt -y update && apt -y install git -RUN git clone -b develop https://gem5.googlesource.com/public/gem5 /gem5 +RUN git clone -b develop https://github.com/gem5/gem5/ /gem5 WORKDIR /gem5 RUN scons -j`nproc` build/ALL/gem5.fast diff --git a/util/gem5art/artifact/README.md b/util/gem5art/artifact/README.md index aae297332a..86e30abcd0 100644 --- a/util/gem5art/artifact/README.md +++ b/util/gem5art/artifact/README.md @@ -51,7 +51,7 @@ gem5_binary = Artifact.registerArtifact( inputs = [gem5_repo,], documentation = ''' Default gem5 binary compiled for the X86 ISA. - This was built from the main gem5 repo (gem5.googlesource.com) without + This was built from the main gem5 repo (https://github.com/gem5/gem5/) without any modifications. We recently updated to the current gem5 master which has a fix for memory channel address striping. ''' diff --git a/util/gem5art/artifact/setup.py b/util/gem5art/artifact/setup.py index 869603db4b..78247eb16d 100755 --- a/util/gem5art/artifact/setup.py +++ b/util/gem5art/artifact/setup.py @@ -56,8 +56,8 @@ setup( install_requires=["pymongo"], python_requires=">=3.6", project_urls={ - "Bug Reports": "https://gem5.atlassian.net/", - "Source": "https://gem5.googlesource.com/", + "Bug Reports": "https://github.com/gem5/issues/", + "Source": "https://github.com/gem5/gem5/", "Documentation": "https://www.gem5.org/documentation/gem5art", }, ) diff --git a/util/gem5art/run/setup.py b/util/gem5art/run/setup.py index d17124bd1f..1ab51b5c2d 100755 --- a/util/gem5art/run/setup.py +++ b/util/gem5art/run/setup.py @@ -57,7 +57,7 @@ setup( python_requires=">=3.6", project_urls={ "Bug Reports": "https://gem5.atlassian.net/", - "Source": "https://gem5.googlesource.com/", + "Source": "https://github.com/gem5/gem5/", "Documentation": "https://www.gem5.org/documentation/gem5art", }, scripts=["bin/gem5art-getruns"], diff --git a/util/gem5art/tasks/setup.py b/util/gem5art/tasks/setup.py index 7bcfc642ae..290c68a17a 100755 --- a/util/gem5art/tasks/setup.py +++ b/util/gem5art/tasks/setup.py @@ -58,7 +58,7 @@ setup( python_requires=">=3.6", project_urls={ "Bug Reports": "https://gem5.atlassian.net/", - "Source": "https://gem5.googlesource.com/", + "Source": "https://github.com/gem5/gem5/", "Documentation": "https://www.gem5.org/documentation/gem5art", }, ) From a2fcfb515239c12537d3d37f65b96a66681a55b3 Mon Sep 17 00:00:00 2001 From: Melissa Jost Date: Wed, 28 Jun 2023 10:16:52 -0700 Subject: [PATCH 25/55] misc: Update documentation and links for GitHub This changes mentions of googlesource and Gerrit to instead link to the gem5 GitHub repository, and updates the documentation to reflect the GitHub review process. Change-Id: I5dc1d9fcf6b96f9e5116802f938b7e3bb5b09567 Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/71878 Maintainer: Bobby Bruce Tested-by: kokoro Reviewed-by: Bobby Bruce --- CONTRIBUTING.md | 40 ++-------------- TESTING.md | 47 +++++++++++++++++-- configs/example/gem5_library/arm-hello.py | 2 +- .../riscv-hello-save-checkpoint.py | 2 +- .../gem5_library/dramsys/arm-hello-dramsys.py | 2 +- configs/example/gem5_library/riscv-fs.py | 2 +- ext/sst/README.md | 2 +- tests/weekly.sh | 2 +- 8 files changed, 54 insertions(+), 45 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3c4d617d2..b41a73a4a6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,12 +14,12 @@ several reasons to do this: * You will have the satisfaction of contributing back to the community. The main method for contributing code to gem5 is via our code review website: -https://gem5-review.googlesource.com/. This documents describes the details of +https://github.com/gem5/gem5/pulls/. This documents describes the details of how to create code changes, upload your changes, have your changes reviewed, and finally push your changes to gem5. More information can be found from the following sources: * http://gem5.org/contributing - * https://gerrit-review.googlesource.com/Documentation/index.html + * https://docs.github.com/en/pull-requests * https://git-scm.com/book @@ -277,9 +277,7 @@ In github, to submit a review request, you can simply push your git commits to a special named branch. For more information on git push see https://git-scm.com/docs/git-push. -There are three ways to push your changes to gerrit. - -Push change to gerrit review +Push changes to GitHub ---------------------------- 1. Fork the gem5 repository on GitHub from https://github.com/gem5/gem5/. 2. Create a new branch in your forked repository for your feature or bug fix. @@ -296,35 +294,7 @@ By default, contributions to gem5 should be made on the develop branch. The stable branch is maintained as a stable release branch (i.e., it can be pulled to obtain the latest official release of gem5). Creation of additional branches is generally discouraged due to their tendency to bloat git repositories with -abandoned code. However, the creation of new branches is permitted for -development of a specific feature or improvement if one or more of the -following criteria are met: - -1. The feature/improvement is likely to be of a large size, consisting of many -commits, with little logic in these commits being contributed separately. -2. The feature/improvement will be developed over a long period of time. -3. There is sufficient reason that a feature/improvement should not be part -of the next gem5 release (e.g., the change should be held within a feature -branch until ready for the next release, at which point it will be merged -into the develop branch). - -If a branch is required it can only be created by a project maintainer. -Therefore, if a gem5 contributor desires a separate branch for their work, they -should request one from the maintainer of the component the work relates to -(see MAINTAINERS for the list of maintainers and the components they are -responsible for). **The maintainer shall use their discretion to determine -whether the creation of a branch is necessary**. If approved, the maintainer -shall create the branch which the contributor may then use. - -Development on a branch within Gerrit functions in exactly the same way as -contributing to the develop branch. When contributors to a branch are -satisfied, they should create a merge commit into the develop branch. The -maintainer should then be notified that the branch they created can now be -deleted. - -**Abandonment of changes within branches may result in these branches being -removed from the repository. All branches within a repo should be under active -development.** +abandoned code. Reviewing patches ================= @@ -332,7 +302,7 @@ Reviewing patches Reviewing patches is done on our github instance at https://github.com/gem5/gem5/pulls/. -After logging in with your Google account, you will be able to comment, review, +After logging in with your GitHub account, you will be able to comment, review, and push your own patches as well as review others' patches. All gem5 users are encouraged to review patches. The only requirement to review patches is to be polite and respectful of others. diff --git a/TESTING.md b/TESTING.md index 146aeac8b1..50cc9bea3e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -18,8 +18,8 @@ To build and run all the unit tests: scons build/ALL/unittests.opt ``` -All unit tests should be run prior to posting a patch to -https://gem5-review.googlesource.com +All unit tests should be run prior to creating a pull request at +https://github.com/gem5/gem5/pulls/ To compile and run just one set of tests (e.g. those declared within `src/base/bitunion.test.cc`): @@ -60,8 +60,8 @@ cd tests ./main.py run ``` -The above is the *minumum* you should run before posting a patch to -https://gem5-review.googlesource.com +The above is the *minumum* you should run before posting a pull request to +https://github.com/gem5/gem5/pulls/ ## Running tests from multiple directories @@ -260,3 +260,42 @@ suites in parallel, supply the `-t ` flag to the run command. For example, to run up to three test suites at the same time:: ./main.py run --skip-build -t 3 + +## Running Tests within GitHub Actions + +To run these tests within GitHub Actions, we use the format of running +tests by directory as shown above in the "Running Tests from Multiple +Directories" section. These tests are run within workflow files, +which can be found in the .github directory of this repository. +You can learn more about workflows +[here](https://docs.github.com/en/actions/using-workflows/about-workflows). + +Each workflow is made up of individual jobs, where each job is a set +of tests that is executed on a runner within GitHub. In each +workflow, each version of gem5.opt needed is first built, and then +stored as an artifact for main.py to use. + +There are two sets of runners within the gem5 repository: builders and +runners. The builders have more resources available to allow for a +quicker compilation of gem5, while the runners have less, and are +meant to only run tests. + +After the gem5 artifact has been uploaded, a runner can then download +the versions needed for their tests. For example, in the daily-tests.yaml, +in order to run the multi_isa tests, you need artifacts of ARM, RISCV, +and VEGA_X86. + +## Adding Tests to GitHub Actions + +In order to add new tests to our GitHub Actions testing infastructure, +follow the format currently shown in the existing workflows. If the +new tests were added to an already existing directory (ex. A very-long +test in the gem5_library_example_tests), it will automatically be +included into the weekly testing, since weekly-tests.yaml already +contains a job for the gem5_library_example_tests. + +However, if a new directory is added to the tests, you need to manually +add a new step to the GitHub workflows. This would consist of both a +step to build whatever version of gem5 was required if it wasn't +already included in the file, as well as a step to run main.py +in the given directory after downloading the gem5 artifact. diff --git a/configs/example/gem5_library/arm-hello.py b/configs/example/gem5_library/arm-hello.py index d66eee5790..b4180f11eb 100644 --- a/configs/example/gem5_library/arm-hello.py +++ b/configs/example/gem5_library/arm-hello.py @@ -80,7 +80,7 @@ board = SimpleBoard( board.set_se_binary_workload( # The `Resource` class reads the `resources.json` file from the gem5 # resources repository: - # https://gem5.googlesource.com/public/gem5-resource. + # https://github.com/gem5/gem5-resources. # Any resource specified in this file will be automatically retrieved. # At the time of writing, this file is a WIP and does not contain all # resources. Jira ticket: https://gem5.atlassian.net/browse/GEM5-1096 diff --git a/configs/example/gem5_library/checkpoints/riscv-hello-save-checkpoint.py b/configs/example/gem5_library/checkpoints/riscv-hello-save-checkpoint.py index 159c4b76e1..439d2054fa 100644 --- a/configs/example/gem5_library/checkpoints/riscv-hello-save-checkpoint.py +++ b/configs/example/gem5_library/checkpoints/riscv-hello-save-checkpoint.py @@ -97,7 +97,7 @@ board = SimpleBoard( board.set_se_binary_workload( # The `Resource` class reads the `resources.json` file from the gem5 # resources repository: - # https://gem5.googlesource.com/public/gem5-resource. + # https://github.com/gem5/gem5-resources. # Any resource specified in this file will be automatically retrieved. # At the time of writing, this file is a WIP and does not contain all # resources. Jira ticket: https://gem5.atlassian.net/browse/GEM5-1096 diff --git a/configs/example/gem5_library/dramsys/arm-hello-dramsys.py b/configs/example/gem5_library/dramsys/arm-hello-dramsys.py index 8b25a36396..ae0f51ad40 100644 --- a/configs/example/gem5_library/dramsys/arm-hello-dramsys.py +++ b/configs/example/gem5_library/dramsys/arm-hello-dramsys.py @@ -74,7 +74,7 @@ board = SimpleBoard( board.set_se_binary_workload( # The `Resource` class reads the `resources.json` file from the gem5 # resources repository: - # https://gem5.googlesource.com/public/gem5-resource. + # https://github.com/gem5/gem5-resources. # Any resource specified in this file will be automatically retrieved. # At the time of writing, this file is a WIP and does not contain all # resources. Jira ticket: https://gem5.atlassian.net/browse/GEM5-1096 diff --git a/configs/example/gem5_library/riscv-fs.py b/configs/example/gem5_library/riscv-fs.py index e3e2bc75e1..e4dce027e3 100644 --- a/configs/example/gem5_library/riscv-fs.py +++ b/configs/example/gem5_library/riscv-fs.py @@ -27,7 +27,7 @@ """ This example runs a simple linux boot. It uses the 'riscv-disk-img' resource. It is built with the sources in `src/riscv-fs` in [gem5 resources]( -https://gem5.googlesource.com/public/gem5-resources). +https://github.com/gem5/gem5-resources). Characteristics --------------- diff --git a/ext/sst/README.md b/ext/sst/README.md index 1f37cb4c44..fb998b5e18 100644 --- a/ext/sst/README.md +++ b/ext/sst/README.md @@ -78,7 +78,7 @@ the `bbl-busybox-boot-exit` resource, which contains an m5 binary, and `m5 exit` will be called upon the booting process reaching the early userspace. More information about building a bootloader containing a Linux Kernel and a customized workload is available at -[https://gem5.googlesource.com/public/gem5-resources/+/refs/heads/stable/src/riscv-boot-exit-nodisk/]. +[https://github.com/gem5/gem5-resources/tree/stable/src/riscv-boot-exit-nodisk]. ## Running an example simulation (Arm) diff --git a/tests/weekly.sh b/tests/weekly.sh index 9c7ebdf76d..2bfd33741d 100755 --- a/tests/weekly.sh +++ b/tests/weekly.sh @@ -94,7 +94,7 @@ rm -rf ${gem5_root}/m5out coAuthorsDBLP.graph 1k_128k.gr result.out # Moreover, DNNMark builds a library and thus doesn't have a binary, so we # need to build it before we run it. # Need to pull this first because HACC's docker requires this path to exist -git clone https://gem5.googlesource.com/public/gem5-resources \ +git clone https://github.com/gem5/gem5-resources \ "${gem5_root}/gem5-resources" From 2242196f03c6f73d62077b1437929eadee8e8741 Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Thu, 13 Jul 2023 18:57:51 +0100 Subject: [PATCH 26/55] arch-arm: Fix assert fail when UQRSHL shiftAmt==0 (#75) When shiftAmt is 0 for a UQRSHL instruction, the code called bits() with incorrect arguments. This fixes a left-shift of 0 to be a NOP/mov, as required. Change-Id: Ic86ca40ac42bfb767a09e8c65a53cec56382a008 Co-authored-by: Marton Erdos --- src/arch/arm/isa/insts/neon64.isa | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/arch/arm/isa/insts/neon64.isa b/src/arch/arm/isa/insts/neon64.isa index 6608f61688..a3b79be912 100644 --- a/src/arch/arm/isa/insts/neon64.isa +++ b/src/arch/arm/isa/insts/neon64.isa @@ -3403,7 +3403,7 @@ let {{ destElem = (srcElem1 >> shiftAmt); } destElem += rBit; - } else { + } else if (shiftAmt > 0) { if (shiftAmt >= sizeof(Element) * 8) { if (srcElem1 != 0) { destElem = mask(sizeof(Element) * 8); @@ -3421,6 +3421,8 @@ let {{ destElem = srcElem1 << shiftAmt; } } + } else { + destElem = srcElem1; } FpscrQc = fpscr; ''' From 189bf66b3e488ec01e003b1813599708307b4ac8 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 14 Jul 2023 06:31:34 -0700 Subject: [PATCH 27/55] tests: Improve Pyunit tests gem5 Resources' downloads (#79) * tests: Remove large files from resource specialization tests These tests were downloading resources (but not actually using them) to ensure the `obtain_resources` function returned the correct specialization and was parsing the data correctly. As these resources were never used, this patch removes the downloading of large files in this case, replacing them with smaller binaries. Change-Id: I7b33aa6be8ec65b296b470cd50b128c084f2b71f * tests: Rename 'looppoint-json...' example specailization Appending '-example' to the end avoids any name clashes. 'looppoint-json-restore-resources-region-1' shares this ID with a real resources in gem5 Resources. Change-Id: I9853e97cb71e768c46ad173b5a497609f4acc3b2 * tests: Remove disk image download from Workload Checks This download is big and unecessary (the workload is never run as part of the test). This patch changes this test to instead download a small binary in it's palce (again, this does not matter as this is never actually run as a disk image). Change-Id: I74034ebcf5f2501917847c258570e88a8f653a5d * tests: Update IDs in Pyunit Workload checks Some of these IDs clash with real workloads/resources in gem5 Resources. To avoid any possible clashes or confusions, all the mock resources/workloads in this suite of tests has been renamed with '-example' appending on the end of the ID. Change-Id: Ifd907b2321416bf05e8c4e646024d179da2ca487 --- .../pyunit_resource_specialization.py | 2 +- .../stdlib/resources/pyunit_workload_checks.py | 2 +- .../refs/resource-specialization.json | 18 +++++++++--------- .../stdlib/resources/refs/workload-checks.json | 16 ++++++++-------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index f2088db8ef..93b83019b9 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -278,7 +278,7 @@ class ResourceSpecializationSuite(unittest.TestCase): Looppoint JSON file.""" resource = obtain_resource( - resource_id="looppoint-json-restore-resource-region-1", + resource_id="looppoint-json-restore-resource-region-1-example", resource_directory=self.get_resource_dir(), resource_version="1.0.0", gem5_version="develop", diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index b59e09d4fe..b374e7a8d3 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -65,7 +65,7 @@ class CustomWorkloadTestSuite(unittest.TestCase): function="set_se_binary_workload", parameters={ "binary": obtain_resource( - "x86-hello64-static", gem5_version="develop" + "x86-hello64-static-example", gem5_version="develop" ), "arguments": ["hello", 6], }, diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index 414bf73b11..9a92204765 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -19,9 +19,9 @@ "id": "disk-image-example", "description": "disk-image documentation.", "architecture": "X86", - "is_zipped": true, - "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", - "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "is_zipped": false, + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", "source": "src/x86-ubuntu", "root_partition": "1", "resource_version": "1.0.0", @@ -64,9 +64,9 @@ "description": "checkpoint-example documentation.", "architecture": "RISCV", "is_zipped": false, - "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "source": null, "is_tar_archive": true, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ @@ -93,8 +93,8 @@ "id": "file-example", "description": null, "is_zipped": false, - "md5sum": "2efd144c11829ab18d54eae6371e120a", - "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", + "md5sum": "71b2cb004fe2cda4556f0b1a38638af6", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/arm/linux/hello64-static", "source": null, "resource_version": "1.0.0", "gem5_versions": [ @@ -106,10 +106,10 @@ "category": "directory", "id": "directory-example", "description": "directory-example documentation.", - "is_zipped": false, - "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "source": null, + "is_zipped": false, "is_tar_archive": true, + "md5sum": "3a57c1bb1077176c4587b8a3bf4f8ace", "url": "http://dist.gem5.org/dist/develop/checkpoints/riscv-hello-example-checkpoint.tar", "resource_version": "1.0.0", "gem5_versions": [ @@ -177,7 +177,7 @@ }, { "category": "looppoint-json", - "id": "looppoint-json-restore-resource-region-1", + "id": "looppoint-json-restore-resource-region-1-example", "description": "A looppoint json file resource.", "is_zipped": false, "region_id": "1", diff --git a/tests/pyunit/stdlib/resources/refs/workload-checks.json b/tests/pyunit/stdlib/resources/refs/workload-checks.json index dcb8577619..bf954059c5 100644 --- a/tests/pyunit/stdlib/resources/refs/workload-checks.json +++ b/tests/pyunit/stdlib/resources/refs/workload-checks.json @@ -1,7 +1,7 @@ [ { "category": "kernel", - "id": "x86-linux-kernel-5.2.3", + "id": "x86-linux-kernel-5.2.3-example", "description": "The linux kernel (v5.2.3), compiled to X86.", "architecture": "X86", "is_zipped": false, @@ -15,12 +15,12 @@ }, { "category": "disk-image", - "id": "x86-ubuntu-18.04-img", + "id": "x86-ubuntu-18.04-img-example", "description": "A disk image containing Ubuntu 18.04 for x86..", "architecture": "X86", - "is_zipped": true, - "md5sum": "90e363abf0ddf22eefa2c7c5c9391c49", - "url": "http://dist.gem5.org/dist/develop/images/x86/ubuntu-18-04/x86-ubuntu.img.gz", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", "source": "src/x86-ubuntu", "root_partition": "1", "resource_version": "1.0.0", @@ -34,8 +34,8 @@ "description": "Description of workload here", "function": "set_kernel_disk_workload", "resources": { - "kernel": "x86-linux-kernel-5.2.3", - "disk-image": "x86-ubuntu-18.04-img" + "kernel": "x86-linux-kernel-5.2.3-example", + "disk-image": "x86-ubuntu-18.04-img-example" }, "additional_params": { "readfile_contents": "echo 'Boot successful'; m5 exit" @@ -47,7 +47,7 @@ }, { "category": "binary", - "id": "x86-hello64-static", + "id": "x86-hello64-static-example", "description": "A 'Hello World!' binary.", "architecture": "X86", "is_zipped": false, From 84c4451cebac98a9e17e3ac87da940b308c90a27 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 17 Jul 2023 10:42:28 -0700 Subject: [PATCH 28/55] util: Add "Improving stability" sec to github-vagrant-runner (#87) Change-Id: I9812a21523b5b29bd7f570df4f1e90dbeabea085 --- util/github-runners-vagrant/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/util/github-runners-vagrant/README.md b/util/github-runners-vagrant/README.md index 86b9e1da87..7d0f116260 100644 --- a/util/github-runners-vagrant/README.md +++ b/util/github-runners-vagrant/README.md @@ -97,6 +97,20 @@ If you wish to destroy all the VMs you can run: **Note:** This script assumes "VAGRANT_HOME" is set to the CWD. +## Improving stability + +Occasionally GitHub runner services, or VMs, go down. This is often silent and +usually only noticable from going to the GitHub repo page "settings" -> "actions" -> "runners" and observing the status. +When the VMs or the service stop working they need restarted. +To do so you can sun `./vm_manager.sh`. This will cycle through the VMs and execute a `vagrant up` command. +This does one of three things depending on the state of the VM: + +1. If the VM is down this will bring the VM back online and start the GitHub runner service. +2. If the VM is up but the GitHub runner service is down, this will start the GitHub runner service. +3. If the VM is up and the GitHub runner service is running (i.e., everything is fine) then this does nothing. + +Given there is no harm in running this command frequently, we recommend setting up a cron job to automatically execute `./vm_manager.sh` every few hours. + ## Troubleshooting ### The default libvirt disk image storage pool is on the wrong drive From b82ae1481bd10c87f98f9d52bfcc8e432ab2a856 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 17 Jul 2023 15:30:35 -0700 Subject: [PATCH 29/55] misc: Update README/README.md (#71) * misc: Update README to README.md This change converts the text-based README to markdown. This works better with modern source-control systems, most notably, GitHub. The README.md has been broken down into sections to better organize the document. This section now included expanded information on Reporting bugs and Requesting Features. Due to renaming 'README' to 'README.md', this code was generating the following for "info.py": ``` README.md = "" ``` As '.' is used to access member variables/methods in python. To fix this "infopy.oy" now replaces "." with "_". As such the generated in in "info.py" is now: ``` README_MD = "" This puts GitHub Discussions and GitHub Issues towards the top of the list. This is to incentivize their usage. Change-Id: I18018ba23493f43861544497f23ec59f1e8debe1 --------- Co-authored-by: Jason Lowe-Power --- README | 43 --------------------- README.md | 90 +++++++++++++++++++++++++++++++++++++++++++ build_tools/infopy.py | 6 ++- src/SConscript | 2 +- 4 files changed, 96 insertions(+), 45 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 5803372a98..0000000000 --- a/README +++ /dev/null @@ -1,43 +0,0 @@ -This is the gem5 simulator. - -The main website can be found at http://www.gem5.org - -A good starting point is http://www.gem5.org/about, and for -more information about building the simulator and getting started -please see http://www.gem5.org/documentation and -http://www.gem5.org/documentation/learning_gem5/introduction. - -To build gem5, you will need the following software: g++ or clang, -Python (gem5 links in the Python interpreter), SCons, zlib, m4, and lastly -protobuf if you want trace capture and playback support. Please see -http://www.gem5.org/documentation/general_docs/building for more details -concerning the minimum versions of these tools. - -Once you have all dependencies resolved, type 'scons -build//gem5.opt' where CONFIG is one of the options in build_opts like -ARM, NULL, MIPS, POWER, SPARC, X86, Garnet_standalone, etc. This will build an -optimized version of the gem5 binary (gem5.opt) with the the specified -configuration. See http://www.gem5.org/documentation/general_docs/building for -more details and options. - -The main source tree includes these subdirectories: - - build_opts: pre-made default configurations for gem5 - - build_tools: tools used internally by gem5's build process. - - configs: example simulation configuration scripts - - ext: less-common external packages needed to build gem5 - - include: include files for use in other programs - - site_scons: modular components of the build system - - src: source code of the gem5 simulator - - system: source for some optional system software for simulated systems - - tests: regression tests - - util: useful utility programs and files - -To run full-system simulations, you may need compiled system firmware, kernel -binaries and one or more disk images, depending on gem5's configuration and -what type of workload you're trying to run. Many of those resources can be -downloaded from http://resources.gem5.org, and/or from the git repository here: -https://gem5.googlesource.com/public/gem5-resources/ - -If you have questions, please send mail to gem5-users@gem5.org - -Enjoy using gem5 and please share your modifications and extensions. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..bd6db2fbd7 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# The gem5 Simulator + +This is the repository for the gem5 simulator. It contains the full source code +for the simulator and all tests and regressions. + +The gem5 simulator is a modular platform for computer-system architecture +research, encompassing system-level architecture as well as processor +microarchitecture. It is primarily used to evaluate new hardware designs, +system software changes, and compile-time and run-time system optimizations. + +The main website can be found at . + +## Getting started + +A good starting point is , and for +more information about building the simulator and getting started +please see and +. + +## Building gem5 + +To build gem5, you will need the following software: g++ or clang, +Python (gem5 links in the Python interpreter), SCons, zlib, m4, and lastly +protobuf if you want trace capture and playback support. Please see + for more details +concerning the minimum versions of these tools. + +Once you have all dependencies resolved, execute +`scons build/ALL/gem5.opt` to build an optimized version of the gem5 binary +(`gem5.opt`) containing all gem5 ISAs. If you only wish to compile gem5 to +include a single ISA, you can replace `ALL` with the name of the ISA. Valid +options include `ARM`, `NULL`, `MIPS`, `POWER`, `SPARC`, and `X86` The complete +list of options can be found in the build_opts directory. + +See https://www.gem5.org/documentation/general_docs/building for more +information on building gem5. + +## The Source Tree + +The main source tree includes these subdirectories: + +* build_opts: pre-made default configurations for gem5 +* build_tools: tools used internally by gem5's build process. +* configs: example simulation configuration scripts +* ext: less-common external packages needed to build gem5 +* include: include files for use in other programs +* site_scons: modular components of the build system +* src: source code of the gem5 simulator. The C++ source, Python wrappers, and Python standard library are found in this directory. +* system: source for some optional system software for simulated systems +* tests: regression tests +* util: useful utility programs and files + +## gem5 Resources + +To run full-system simulations, you may need compiled system firmware, kernel +binaries and one or more disk images, depending on gem5's configuration and +what type of workload you're trying to run. Many of these resources can be +obtained from . + +More information on gem5 Resources can be found at +. + +## Getting Help, Reporting bugs, and Requesting Features + +We provide a variety of channels for users and developers to get help, report +bugs, requests features, or engage in community discussions. Below +are a few of the most common we recommend using. + +* **GitHub Discussions**: A GitHub Discussions page. This can be used to start +discussions or ask questions. Available at +. +* **GitHub Issues**: A GitHub Issues page for reporting bugs or requesting +features. Available at . +* **Jira Issue Tracker**: A Jira Issue Tracker for reporting bugs or requesting +features. Available at . +* **Slack**: A Slack server with a variety of channels for the gem5 community +to engage in a variety of discussions. Please visit + to join. +* **gem5-users@gem5.org**: A mailing list for users of gem5 to ask questions +or start discussions. To join the mailing list please visit +. +* **gem5-dev@gem5.org**: A mailing list for developers of gem5 to ask questions +or start discussions. To join the mailing list please visit +. + +## Contributing to gem5 + +We hope you enjoy using gem5. When appropriate we advise charing your +contributions to the project. can help you +get started. Additional information can be found in the CONTRIBUTING.md file. diff --git a/build_tools/infopy.py b/build_tools/infopy.py index 4f15f24f98..8a4e013388 100644 --- a/build_tools/infopy.py +++ b/build_tools/infopy.py @@ -51,7 +51,11 @@ args = parser.parse_args() code = code_formatter() for source in args.files: - src = os.path.basename(source) + # We replace the "."s in the file name with underscores to make + # it a valid python identifier. With the dot, "README.md" would generate + # `README.md = "..."` which is not valid as `md` is not a property of + # `README`. + src = os.path.basename(source).replace(".", "_") with open(source, "r") as f: data = "".join(f) code("${src} = ${{repr(data)}}") diff --git a/src/SConscript b/src/SConscript index 1b4430327c..894a334e2a 100644 --- a/src/SConscript +++ b/src/SConscript @@ -615,7 +615,7 @@ PySource('m5', 'python/m5/defines.py') # Generate a file that wraps the basic top level files gem5py_env.Command('python/m5/info.py', - [ File('#/COPYING'), File('#/LICENSE'), File('#/README'), + [ File('#/COPYING'), File('#/LICENSE'), File('#/README.md'), "${GEM5PY}", "${INFOPY_PY}" ], MakeAction('"${GEM5PY}" "${INFOPY_PY}" "${TARGET}" ' '${SOURCES[:-2]}', From e810f53ebee1d95af3fc0b9725438f88224c8039 Mon Sep 17 00:00:00 2001 From: Lingkang Date: Wed, 19 Jul 2023 05:00:57 +0800 Subject: [PATCH 30/55] python: fix fatal in main.py (github #78) (#93) * python: fix fatal in main.py (github #78) Issue-On: https://github.com/gem5/gem5/issues/78 * python: fix fatal in main.py (github #78) Issue-On: https://github.com/gem5/gem5/issues/78 Change-Id: I80855b05168a067ddd7706ad9fd7e71e75bfd3b1 --------- Co-authored-by: Jason Lowe-Power --- src/python/m5/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/python/m5/main.py b/src/python/m5/main.py index ddcb024f8b..e07e9562ab 100644 --- a/src/python/m5/main.py +++ b/src/python/m5/main.py @@ -348,13 +348,14 @@ def interact(scope): def _check_tracing(): - import m5 import _m5.core + from .util import fatal + if _m5.core.TRACING_ON: return - m5.fatal("Tracing is not enabled. Compile with TRACING_ON") + fatal("Tracing is not enabled. Compile with TRACING_ON") def main(): @@ -369,7 +370,7 @@ def main(): from . import stats from . import trace - from .util import inform, fatal, panic, isInteractive + from .util import inform, panic, isInteractive from m5.util.terminal_formatter import TerminalFormatter options, arguments = parse_options() From f78fc5b2f7b995222a8d0a9ad26f464599c38718 Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Mon, 17 Jul 2023 12:35:03 +0100 Subject: [PATCH 31/55] arch-arm: Hook TLBIOS instructions to the TlbiShareable obj FEAT_TLBIOS has been introduced by a recent patch [1] which was however missing to include the outer shareable case in the Msr disambiguation switch. Which meant the TLBIOS instructions were decoded as normal MSR instructions, with no effect whatsoever on the TLBs [1]: https://gem5-review.googlesource.com/c/public/gem5/+/70567 Change-Id: I41665a4634fbe0ee8cc30dbc5d88d63103082ae9 Signed-off-by: Giacomo Travaglini --- src/arch/arm/isa/formats/aarch64.isa | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/arch/arm/isa/formats/aarch64.isa b/src/arch/arm/isa/formats/aarch64.isa index 47d509e808..68a741a831 100644 --- a/src/arch/arm/isa/formats/aarch64.isa +++ b/src/arch/arm/isa/formats/aarch64.isa @@ -545,21 +545,37 @@ namespace Aarch64 return new Tlbi64LocalHub( machInst, miscReg, rt); case MISCREG_TLBI_ALLE3IS: + case MISCREG_TLBI_ALLE3OS: case MISCREG_TLBI_ALLE2IS: + case MISCREG_TLBI_ALLE2OS: case MISCREG_TLBI_ALLE1IS: + case MISCREG_TLBI_ALLE1OS: case MISCREG_TLBI_VMALLS12E1IS: + case MISCREG_TLBI_VMALLS12E1OS: case MISCREG_TLBI_VMALLE1IS: + case MISCREG_TLBI_VMALLE1OS: case MISCREG_TLBI_VAE3IS_Xt: + case MISCREG_TLBI_VAE3OS_Xt: case MISCREG_TLBI_VALE3IS_Xt: + case MISCREG_TLBI_VALE3OS_Xt: case MISCREG_TLBI_VAE2IS_Xt: + case MISCREG_TLBI_VAE2OS_Xt: case MISCREG_TLBI_VALE2IS_Xt: + case MISCREG_TLBI_VALE2OS_Xt: case MISCREG_TLBI_VAE1IS_Xt: + case MISCREG_TLBI_VAE1OS_Xt: case MISCREG_TLBI_VALE1IS_Xt: + case MISCREG_TLBI_VALE1OS_Xt: case MISCREG_TLBI_ASIDE1IS_Xt: + case MISCREG_TLBI_ASIDE1OS_Xt: case MISCREG_TLBI_VAAE1IS_Xt: + case MISCREG_TLBI_VAAE1OS_Xt: case MISCREG_TLBI_VAALE1IS_Xt: + case MISCREG_TLBI_VAALE1OS_Xt: case MISCREG_TLBI_IPAS2E1IS_Xt: + case MISCREG_TLBI_IPAS2E1OS_Xt: case MISCREG_TLBI_IPAS2LE1IS_Xt: + case MISCREG_TLBI_IPAS2LE1OS_Xt: return new Tlbi64ShareableHub( machInst, miscReg, rt, dec.dvmEnabled); default: From fa7c00935da8338178d3166a2fae486fa95aecc0 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 26 Jul 2023 12:17:07 -0700 Subject: [PATCH 32/55] misc: Update version to v23.0.1.0 Change-Id: Icd673083f23a465205bea12407bf265e2ba6fb4a --- src/Doxyfile | 2 +- src/base/version.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Doxyfile b/src/Doxyfile index 79498c9c72..1ffbb7cce5 100644 --- a/src/Doxyfile +++ b/src/Doxyfile @@ -31,7 +31,7 @@ PROJECT_NAME = gem5 # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = v23.0.0.1 +PROJECT_NUMBER = v23.0.1.0 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. diff --git a/src/base/version.cc b/src/base/version.cc index 1069d57348..bfff67f5b6 100644 --- a/src/base/version.cc +++ b/src/base/version.cc @@ -32,6 +32,6 @@ namespace gem5 /** * @ingroup api_base_utils */ -const char *gem5Version = "23.0.0.1"; +const char *gem5Version = "23.0.1.0"; } // namespace gem5 From 919dd5efbd67ee27579014c9a4e0f67d47eb9c1f Mon Sep 17 00:00:00 2001 From: rogerchang23424 Date: Fri, 21 Jul 2023 00:35:45 +0800 Subject: [PATCH 33/55] scons: Add extra parent dir to CPPPATH if --no-duplicate-sources (#104) In the previous version of gem5, the source files of extra directories will copy to build directory for compilation. It will not be a problem if the extra directories include *.h(*.hh) from the other extra directories. After the patch applied from the change (https://gem5-review.googlesource.com/c/public/gem5/+/68758). The source files of extra directories will not copy to the build directory unless the user compiles gem5 with "--duplicate-sources". It will cause the compilation error if the code includes a header file from other repositories. For example, assume we want to compile gem5 with "foo/bar1" and "foo/bar2" repositories and they are gem5-independent. There are some header files in "foo/bar1/a.h" "foo/bar1/b.h" and "foo/bar2/d.h". If the code "foo/bar1/sample.c" tries to include the file "foo/bar2/d.h". They usually include the file by declare "#include bar2/d.h" in foo/bar1/sample.c. It can work if --duplicate-sources is specified in gem5 build because they will copy to /bar1 and /bar2 respectively, and -I is specified by default whether duplicate_sources or not. It will raise the compilation error if the user does not specify it. The change is aimed to let the situation work without duplicate-sources specified by adding parent extra directory, and adding them before the extra directories. If the --duplicate-sources specified, it will not add parent extra directories to avoid repeat include paths. Change-Id: I461e1dcb8266d785f1f38eeff77f9d515d47c03d --- src/SConscript | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/SConscript b/src/SConscript index 894a334e2a..b41921bb2f 100644 --- a/src/SConscript +++ b/src/SConscript @@ -536,6 +536,14 @@ Export('DebugFormatFlag') # the corresponding build directory to pick up generated include # files. env.Append(CPPPATH=Dir('.')) +parent_dir_set = set() + +for extra_dir in extras_dir_list: + parent_dir_set.add(str(Dir(extra_dir).Dir('..').abspath)) + +if not GetOption('duplicate_sources'): + for parent_dir in parent_dir_set: + env.Append(CPPPATH=Dir(parent_dir)) for extra_dir in extras_dir_list: env.Append(CPPPATH=Dir(extra_dir)) From 41abc6ab77b6dc24f0156aa0a9cab2d4eb1ab90e Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Thu, 20 Jul 2023 10:34:44 -0700 Subject: [PATCH 34/55] cpu-kvm: Make using perf when using KVM CPU optional (#95) * cpu-kvm: Add a variable signifying whether we are using perf Change-Id: Iaa081e364f85c863f781723b5524d267724ed0e4 Signed-off-by: Hoa Nguyen * cpu-kvm: Making it clear the functionalities are specific to KVM Change-Id: I982426f294d90655227dc15337bf73c42a260ded Signed-off-by: Hoa Nguyen * cpu-kvm: Make perf optional Change-Id: I8973c2a96575383976cea7ca3fda478f83e95c3f Signed-off-by: Hoa Nguyen * configs: Add an example config of using KVM without perf Change-Id: Ic69fa7dac4f1a2c8fe23712b0fa77b5b22c5f2df Signed-off-by: Hoa Nguyen * Apply suggestions from code review Co-authored-by: Jason Lowe-Power * misc: Add an example to the panic Change-Id: Ic1fdfb955e5d8b9ad1d4f0a2bf30fa8050deba70 Signed-off-by: Hoa Nguyen * misc: Add warning of not using perf when using KVM CPU Change-Id: I96c0832fb48c63a79773665ca6228da778ef0497 Signed-off-by: Hoa Nguyen * misc: Fix stuff Change-Id: Ib407ae7407955b695f0e0f2718324f41bb0d768f Signed-off-by: Hoa Nguyen * misc: style fix Change-Id: I7275942e43f46140fdd52c975f76abb3c81b8b0a Signed-off-by: Hoa Nguyen --------- Signed-off-by: Hoa Nguyen Co-authored-by: Jason Lowe-Power --- .../x86-ubuntu-run-with-kvm-no-perf.py | 138 ++++++++++++++++++ src/cpu/kvm/BaseKvmCPU.py | 5 + src/cpu/kvm/base.cc | 84 ++++++++--- src/cpu/kvm/base.hh | 7 +- src/cpu/kvm/perfevent.cc | 16 +- 5 files changed, 221 insertions(+), 29 deletions(-) create mode 100644 configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py diff --git a/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py new file mode 100644 index 0000000000..1c65357921 --- /dev/null +++ b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" + +This script demonstrates how to use KVM CPU without perf. +This simulation boots Ubuntu 18.04 using 2 KVM CPUs without using perf. + +Usage +----- + +``` +scons build/X86/gem5.opt -j`nproc` +./build/X86/gem5.opt configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py +``` +""" + +from gem5.utils.requires import requires +from gem5.components.boards.x86_board import X86Board +from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy import ( + MESITwoLevelCacheHierarchy, +) +from gem5.components.memory.single_channel import SingleChannelDDR4_2400 +from gem5.components.processors.simple_switchable_processor import ( + SimpleSwitchableProcessor, +) +from gem5.components.processors.cpu_types import CPUTypes +from gem5.isas import ISA +from gem5.coherence_protocol import CoherenceProtocol +from gem5.simulate.simulator import Simulator +from gem5.simulate.exit_event import ExitEvent +from gem5.resources.workload import Workload + +# This simulation requires using KVM with gem5 compiled for X86 simulation +# and with MESI_Two_Level cache coherence protocol. +requires( + isa_required=ISA.X86, + coherence_protocol_required=CoherenceProtocol.MESI_TWO_LEVEL, + kvm_required=True, +) + +from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy import ( + MESITwoLevelCacheHierarchy, +) + +cache_hierarchy = MESITwoLevelCacheHierarchy( + l1d_size="32KiB", + l1d_assoc=8, + l1i_size="32KiB", + l1i_assoc=8, + l2_size="512KiB", + l2_assoc=16, + num_l2_banks=1, +) + +# Main memory +memory = SingleChannelDDR4_2400(size="3GiB") + +# This is a switchable CPU. We first boot Ubuntu using KVM, then the guest +# will exit the simulation by calling "m5 exit" (see the `command` variable +# below, which contains the command to be run in the guest after booting). +# Upon exiting from the simulation, the Exit Event handler will switch the +# CPU type (see the ExitEvent.EXIT line below, which contains a map to +# a function to be called when an exit event happens). +processor = SimpleSwitchableProcessor( + starting_core_type=CPUTypes.KVM, + switch_core_type=CPUTypes.TIMING, + isa=ISA.X86, + num_cores=2, +) + +# Here we tell the KVM CPU (the starting CPU) not to use perf. +for proc in processor.start: + proc.core.usePerf = False + +# Here we setup the board. The X86Board allows for Full-System X86 simulations. +board = X86Board( + clk_freq="3GHz", + processor=processor, + memory=memory, + cache_hierarchy=cache_hierarchy, +) + +# Here we set the Full System workload. +# The `set_kernel_disk_workload` function for the X86Board takes a kernel, a +# disk image, and, optionally, a command to run. + +# This is the command to run after the system has booted. The first `m5 exit` +# will stop the simulation so we can switch the CPU cores from KVM to timing +# and continue the simulation to run the echo command, sleep for a second, +# then, again, call `m5 exit` to terminate the simulation. After simulation +# has ended you may inspect `m5out/system.pc.com_1.device` to see the echo +# output. +command = ( + "m5 exit;" + + "echo 'This is running on Timing CPU cores.';" + + "sleep 1;" + + "m5 exit;" +) + +workload = Workload("x86-ubuntu-18.04-boot") +workload.set_parameter("readfile_contents", command) +board.set_workload(workload) + +simulator = Simulator( + board=board, + on_exit_event={ + # Here we want override the default behavior for the first m5 exit + # exit event. Instead of exiting the simulator, we just want to + # switch the processor. The 2nd m5 exit after will revert to using + # default behavior where the simulator run will exit. + ExitEvent.EXIT: (func() for func in [processor.switch]) + }, +) +simulator.run() diff --git a/src/cpu/kvm/BaseKvmCPU.py b/src/cpu/kvm/BaseKvmCPU.py index f958e8126c..610663fa41 100644 --- a/src/cpu/kvm/BaseKvmCPU.py +++ b/src/cpu/kvm/BaseKvmCPU.py @@ -64,6 +64,11 @@ class BaseKvmCPU(BaseCPU): def support_take_over(cls): return True + usePerf = Param.Bool( + True, + "Use perf for gathering statistics from the guest and providing " + "statistic-related functionalities", + ) useCoalescedMMIO = Param.Bool(False, "Use coalesced MMIO (EXPERIMENTAL)") usePerfOverflow = Param.Bool( False, "Use perf event overflow counters (EXPERIMENTAL)" diff --git a/src/cpu/kvm/base.cc b/src/cpu/kvm/base.cc index e22e1628d2..eaa771d8cf 100644 --- a/src/cpu/kvm/base.cc +++ b/src/cpu/kvm/base.cc @@ -71,12 +71,15 @@ BaseKvmCPU::BaseKvmCPU(const BaseKvmCPUParams ¶ms) alwaysSyncTC(params.alwaysSyncTC), threadContextDirty(true), kvmStateDirty(false), + usePerf(params.usePerf), vcpuID(-1), vcpuFD(-1), vcpuMMapSize(0), _kvmRun(NULL), mmioRing(NULL), pageSize(sysconf(_SC_PAGE_SIZE)), tickEvent([this]{ tick(); }, "BaseKvmCPU tick", false, Event::CPU_Tick_Pri), activeInstPeriod(0), + hwCycles(nullptr), + hwInstructions(nullptr), perfControlledByTimer(params.usePerfOverflow), hostFactor(params.hostFactor), stats(this), ctrInsts(0) @@ -96,6 +99,22 @@ BaseKvmCPU::BaseKvmCPU(const BaseKvmCPUParams ¶ms) thread->setStatus(ThreadContext::Halted); tc = thread->getTC(); threadContexts.push_back(tc); + + if ((!usePerf) && perfControlledByTimer) { + panic("KVM: invalid combination of parameters: cannot use " + "perfControlledByTimer without usePerf\n"); + } + + // If we use perf, we create new PerfKVMCounters + if (usePerf) { + hwCycles = std::unique_ptr(new PerfKvmCounter()); + hwInstructions = std::unique_ptr(new PerfKvmCounter()); + } else { + inform("Using KVM CPU without perf. The stats related to the number " + "of cycles and instructions executed by the KVM CPU will not " + "be updated. The stats should not be used for performance " + "evaluation."); + } } BaseKvmCPU::~BaseKvmCPU() @@ -248,7 +267,7 @@ BaseKvmCPU::restartEqThread() setupCounters(); if (p.usePerfOverflow) { - runTimer.reset(new PerfKvmTimer(hwCycles, + runTimer.reset(new PerfKvmTimer(*hwCycles, KVM_KICK_SIGNAL, p.hostFactor, p.hostFreq)); @@ -424,8 +443,10 @@ BaseKvmCPU::notifyFork() vcpuFD = -1; _kvmRun = NULL; - hwInstructions.detach(); - hwCycles.detach(); + if (usePerf) { + hwInstructions->detach(); + hwCycles->detach(); + } } } @@ -690,7 +711,9 @@ BaseKvmCPU::kvmRunDrain() uint64_t BaseKvmCPU::getHostCycles() const { - return hwCycles.read(); + if (usePerf) + return hwCycles->read(); + return 0; } Tick @@ -746,21 +769,26 @@ BaseKvmCPU::kvmRun(Tick ticks) // Get hardware statistics after synchronizing contexts. The KVM // state update might affect guest cycle counters. uint64_t baseCycles(getHostCycles()); - uint64_t baseInstrs(hwInstructions.read()); + uint64_t baseInstrs = 0; + if (usePerf) { + baseInstrs = hwInstructions->read(); + } // Arm the run timer and start the cycle timer if it isn't // controlled by the overflow timer. Starting/stopping the cycle // timer automatically starts the other perf timers as they are in // the same counter group. runTimer->arm(ticks); - if (!perfControlledByTimer) - hwCycles.start(); + if (usePerf && (!perfControlledByTimer)) { + hwCycles->start(); + } ioctlRun(); runTimer->disarm(); - if (!perfControlledByTimer) - hwCycles.stop(); + if (usePerf && (!perfControlledByTimer)) { + hwCycles->stop(); + } // The control signal may have been delivered after we exited // from KVM. It will be pending in that case since it is @@ -771,7 +799,10 @@ BaseKvmCPU::kvmRun(Tick ticks) const uint64_t hostCyclesExecuted(getHostCycles() - baseCycles); const uint64_t simCyclesExecuted(hostCyclesExecuted * hostFactor); - const uint64_t instsExecuted(hwInstructions.read() - baseInstrs); + uint64_t instsExecuted = 0; + if (usePerf) { + instsExecuted = hwInstructions->read() - baseInstrs; + } ticksExecuted = runTimer->ticksFromHostCycles(hostCyclesExecuted); /* Update statistics */ @@ -1288,13 +1319,14 @@ BaseKvmCPU::setupCounters() // We might be re-attaching counters due threads being // re-initialised after fork. - if (hwCycles.attached()) - hwCycles.detach(); + if (usePerf) { + if (hwCycles->attached()) { + hwCycles->detach(); + } - hwCycles.attach(cfgCycles, - 0); // TID (0 => currentThread) - - setupInstCounter(); + hwCycles->attach(cfgCycles, 0); // TID (0 => currentThread) + setupInstCounter(); + } } bool @@ -1344,10 +1376,16 @@ BaseKvmCPU::setupInstStop() void BaseKvmCPU::setupInstCounter(uint64_t period) { + // This function is for setting up instruction counter using perf + if (!usePerf) { + return; + } + // No need to do anything if we aren't attaching for the first // time or the period isn't changing. - if (period == activeInstPeriod && hwInstructions.attached()) + if (period == activeInstPeriod && hwInstructions->attached()) { return; + } PerfKvmCounterConfig cfgInstructions(PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS); @@ -1366,15 +1404,15 @@ BaseKvmCPU::setupInstCounter(uint64_t period) // We need to detach and re-attach the counter to reliably change // sampling settings. See PerfKvmCounter::period() for details. - if (hwInstructions.attached()) - hwInstructions.detach(); - assert(hwCycles.attached()); - hwInstructions.attach(cfgInstructions, + if (hwInstructions->attached()) + hwInstructions->detach(); + assert(hwCycles->attached()); + hwInstructions->attach(cfgInstructions, 0, // TID (0 => currentThread) - hwCycles); + *hwCycles); if (period) - hwInstructions.enableSignals(KVM_KICK_SIGNAL); + hwInstructions->enableSignals(KVM_KICK_SIGNAL); activeInstPeriod = period; } diff --git a/src/cpu/kvm/base.hh b/src/cpu/kvm/base.hh index 7bbf393f9b..3cf70a0bef 100644 --- a/src/cpu/kvm/base.hh +++ b/src/cpu/kvm/base.hh @@ -653,6 +653,9 @@ class BaseKvmCPU : public BaseCPU */ bool kvmStateDirty; + /** True if using perf; False otherwise*/ + bool usePerf; + /** KVM internal ID of the vCPU */ long vcpuID; @@ -763,7 +766,7 @@ class BaseKvmCPU : public BaseCPU * PerfKvmTimer (see perfControlledByTimer) to trigger exits from * KVM. */ - PerfKvmCounter hwCycles; + std::unique_ptr hwCycles; /** * Guest instruction counter. @@ -776,7 +779,7 @@ class BaseKvmCPU : public BaseCPU * @see setupInstBreak * @see scheduleInstStop */ - PerfKvmCounter hwInstructions; + std::unique_ptr hwInstructions; /** * Does the runTimer control the performance counters? diff --git a/src/cpu/kvm/perfevent.cc b/src/cpu/kvm/perfevent.cc index f9c317da41..c5e33abf82 100644 --- a/src/cpu/kvm/perfevent.cc +++ b/src/cpu/kvm/perfevent.cc @@ -173,12 +173,20 @@ PerfKvmCounter::attach(PerfKvmCounterConfig &config, { if (errno == EACCES) { - panic("PerfKvmCounter::attach recieved error EACCESS\n" + panic("PerfKvmCounter::attach received error EACCESS.\n" " This error may be caused by a too restrictive setting\n" - " in the file '/proc/sys/kernel/perf_event_paranoid'\n" - " The default value was changed to 2 in kernel 4.6\n" + " in the file '/proc/sys/kernel/perf_event_paranoid'.\n" + " The default value was changed to 2 in kernel 4.6.\n" " A value greater than 1 prevents gem5 from making\n" - " the syscall to perf_event_open"); + " the syscall to perf_event_open.\n" + " Alternatively, you can set the usePerf flag of the KVM\n" + " CPU to False. Setting this flag to False will limit some\n" + " functionalities of KVM CPU, such as counting the number of\n" + " cycles and the number of instructions, as well as the\n" + " ability of exiting to gem5 after a certain amount of cycles\n" + " or instructions when using KVM CPU. An example can be found\n" + " here, configs/example/gem5_library/" + "x86-ubuntu-run-with-kvm-no-perf.py."); } panic("PerfKvmCounter::attach failed (%i)\n", errno); } From 4516a0059365cd7ecd308a1db17a657585d808ed Mon Sep 17 00:00:00 2001 From: Jason Lowe-Power Date: Thu, 20 Jul 2023 13:23:16 -0700 Subject: [PATCH 35/55] cpu-minor: Check pc valid before printing In https://gem5-review.googlesource.com/c/public/gem5/+/52047 inst.pc was changed from an object to a pointer. It is possible that this pointer is null (e.g., if there is an interrupt and there is a bubble). Make sure to check that it's not null before printing. I believe that other places this pointer is dereferenced without an explicit null check are safe, but I'm not certain. Change-Id: Idbe246cfdb62d4d75416d41b451fb3c076233bbc Signed-off-by: Jason Lowe-Power --- src/cpu/minor/dyn_inst.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cpu/minor/dyn_inst.cc b/src/cpu/minor/dyn_inst.cc index 68415ecd09..6ff5ed6b5e 100644 --- a/src/cpu/minor/dyn_inst.cc +++ b/src/cpu/minor/dyn_inst.cc @@ -112,6 +112,11 @@ MinorDynInst::reportData(std::ostream &os) const std::ostream & operator <<(std::ostream &os, const MinorDynInst &inst) { + if (!inst.pc) { + os << inst.id << " pc: 0x???????? (bubble)"; + return os; + } + os << inst.id << " pc: 0x" << std::hex << inst.pc->instAddr() << std::dec << " ("; @@ -169,7 +174,7 @@ MinorDynInst::minorTraceInst(const Named &named_object) const { if (isFault()) { minorInst(named_object, "id=F;%s addr=0x%x fault=\"%s\"\n", - id, pc->instAddr(), fault->name()); + id, pc ? pc->instAddr() : 0, fault->name()); } else { unsigned int num_src_regs = staticInst->numSrcRegs(); unsigned int num_dest_regs = staticInst->numDestRegs(); @@ -209,7 +214,7 @@ MinorDynInst::minorTraceInst(const Named &named_object) const minorInst(named_object, "id=%s addr=0x%x inst=\"%s\" class=%s" " flags=\"%s\"%s%s\n", - id, pc->instAddr(), + id, pc ? pc->instAddr() : 0, (staticInst->opClass() == No_OpClass ? "(invalid)" : staticInst->disassemble(0,NULL)), enums::OpClassStrings[staticInst->opClass()], From 7665c338e268736c0636400b1d3d13f169a0a61e Mon Sep 17 00:00:00 2001 From: Adwaith R Krishna Date: Fri, 21 Jul 2023 02:26:31 +0530 Subject: [PATCH 36/55] mem-garnet: Fix packet_id val in flit (#72) Change-Id: I163b5a32972783bf2e99f3383b9f86776577b727 Co-authored-by: Bobby R. Bruce --- src/mem/ruby/network/garnet/flit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/ruby/network/garnet/flit.cc b/src/mem/ruby/network/garnet/flit.cc index d31d826c93..21e6dcdc33 100644 --- a/src/mem/ruby/network/garnet/flit.cc +++ b/src/mem/ruby/network/garnet/flit.cc @@ -51,7 +51,7 @@ flit::flit(int packet_id, int id, int vc, int vnet, RouteInfo route, int size, m_enqueue_time = curTime; m_dequeue_time = curTime; m_time = curTime; - m_packet_id = id; + m_packet_id = packet_id; m_id = id; m_vnet = vnet; m_vc = vc; From 2513d39392395db99890d3d253aa734268a10bee Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 26 Jul 2023 12:50:49 -0700 Subject: [PATCH 37/55] misc: Update RELEASE-NOTES.md for v23.0.1.0 Change-Id: I5ae5081e0ac5524271e6c8300917d7d1e16d71ee --- RELEASE-NOTES.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 1a72428fc1..1e3a7fbadb 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,29 @@ +# Version 23.0.1.0 + +This minor release incorporates documentation updates, bug fixes, and some minor improvements. + +## Documentation updates + +* "TESTING.md" has been updated to more accurately reflect our current testing infrastructure. +* "README" has been replaced with "README.md" and includes more up-to-date information on using gem5. +* "CONTRIBUTING.md" has been updated to reflect our migration to GitHub and the changes in policy and proceedures. +* Where needed old references to Gerrit have been removed in favor of GitHub. + +## Bug Fixes + +* Fixes an assert failure when using ARM which was trigged when `shiftAmt` is 0 for a UQRSH instruction. +* Fixes `name 'fatal' is not defined` being thrown when tracing is off. +* Fixes a bug in ARM in which the TLBIOS instructions were decoded as normal MSR instructions with no effect on the TLBs. +* Fixes invalid `packet_id` value in flit. +* Fixes default CustomMesh for use with Garnet. + +## Minor Improvements + +* The gem5 resources downloader now outputs more helpful errors in the case of a failure. +* "util/github-runners-vagrant" has been added. This outlines how to setup a GitHub Action's set-hosted runner for gem5. +* The PyUnit tests have been refactored to no longer download large resources during testing. +* Using Perf is now optional when utilizing KVM CPUs. + # Version 23.0.0.1 **[HOTFIX]** Fixes compilation of `GCN3_X86` and `VEGA_X85`. From 0f358732cbb8c10a7d89abcf6499a6a42d2c799d Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 26 Jul 2023 10:38:17 -0700 Subject: [PATCH 38/55] misc: Add ".github" changes to minor release This is a simple copy of the current state of the .github on the develop branch, as of 2023-07-27. The stable branch .github dir should never be ahead of that on develop. Therefore this should be safe to do. Change-Id: I1e39de2d1f923d1834d0a77f79a1ff3220964bba --- .github/ISSUE_TEMPLATE/bug_report.md | 62 +++ .github/workflows/ci-tests.yaml | 44 +- .github/workflows/compiler-tests.yaml | 4 +- .github/workflows/daily-tests.yaml | 769 +++----------------------- .github/workflows/utils.yaml | 19 + .github/workflows/weekly-tests.yaml | 20 + 6 files changed, 217 insertions(+), 701 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/workflows/utils.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..179a37ac0b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,62 @@ +--- +name: Bug report +about: Create a report to help us find and fix the bug +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Affects version** +State which version of gem5 this bug was found in. If on the develop branch state the Commit revision ID you are working. + +**gem5 Modifications** +If you have modified gem5 in some way please state, to the best of your ability, how it has been modified. + +**To Reproduce** +Steps to reproduce the behavior. Please assume starting from a clean repository: +1. Compile gem5 with command ... +2. Execute the simulation with... + +If writing code, or a terminal command, use code blocks. Either an inline code block, \`scons build/ALL/gem5.opt\` (enclosed in two \`) or a multi-line codeblock: + + +\`\`\` + +int x=2; + +int y=3' + +print(x+y); + +\`\`\` + +If possible, please include the Python configuration script used and state clearly any parameters passed. + +**Terminal Output** +If applicable, add the terminal output here. If long, only include the relevant lines. +Please put the terminal output in code blocks. I.e.: + +\`\`\` + +#Terminal output here# + +\`\`\` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Host Operating System** +Ubuntu 22.04, Mac OS X, etc. + +**Host ISA** +ARM, X86, RISC-V, etc. + +**Compiler used** +State which compiler was used to compile gem5. Please include the compiler version. + +**Additional information** +Add any other information which does not fit in the previous sections but may be of use in fixing this bug. diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index c3188b87ee..9894716665 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -19,25 +19,27 @@ jobs: # ensures we have a change-id in every commit, needed for gerrit check-for-change-id: # runs on github hosted runner - runs-on: ubuntu-latest - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + runs-on: ubuntu-22.04 steps: - - uses: actions/github-script@v6 - env: - token: "Change-Id" - pattern: ".*" + - uses: actions/checkout@v3 with: - script: | - const commits = ${{ toJSON(github.event.commits) }} - for (const commit of commits) { - const id = "Change-Id: " - const message = commit.message; - if (!message.includes(id)) { - core.setFailed('One or more of the commits in this pull request is missing a Change-ID, which we require for any changes made to gem5. ' + - 'To automatically insert one, run the following:\n f=`git rev-parse --git-dir`/hooks/commit-msg ; mkdir -p $(dirname $f) ; ' + - 'curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request.') - } - } + fetch-depth: 0 + - name: Check for Change-Id + run: | + # loop through all the commits in the pull request + for commit in $(git rev-list ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}); do + git checkout $commit + if (git log -1 --pretty=format:"%B" | grep -q "Change-Id: ") + then + # passes as long as at least one change-id exists in the pull request + exit 0 + fi + done + # if we reach this part, none of the commits had a change-id + echo "None of the commits in this pull request contains a Change-ID, which we require for any changes made to gem5. "\ + "To automatically insert one, run the following:\n f=`git rev-parse --git-dir`/hooks/commit-msg ; mkdir -p $(dirname $f) ; "\ + "curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request." + exit 1 build-gem5: runs-on: [self-hosted, linux, x64, build] @@ -74,7 +76,7 @@ jobs: testlib-quick: runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, check-for-change-id, build-gem5] + needs: [pre-commit, build-gem5, check-for-change-id] timeout-minutes: 360 # 6 hours steps: - uses: actions/checkout@v3 @@ -100,4 +102,10 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + if: success() || failure() + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 52569e2e01..3b5ed1c901 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -16,7 +16,7 @@ jobs: all-compilers: strategy: matrix: - image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, gcc-version-7, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, clang-version-6.0, ubuntu-18.04_all-dependencies, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] + image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] timeout-minutes: 2880 # 48 hours @@ -35,7 +35,7 @@ jobs: latest-compilers-all-gem5-builds: strategy: matrix: - gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base] + gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base, VEGA_X86, GCN3_X86] image: [gcc-version-12, clang-version-14] opts: [.opt] runs-on: [self-hosted, linux, x64, run] diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 5fab058b79..679dba2978 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -8,746 +8,136 @@ on: - cron: '0 7 * * *' jobs: -# building all necessary versions of gem5 + name-artifacts: + runs-on: ubuntu-latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v2 + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S-")" >> $GITHUB_OUTPUT + build-gem5: + strategy: + matrix: + # NULL is in quotes since it is considered a keyword in yaml files + image: [ALL, ALL_CHI, ARM, ALL_MSI, ALL_MESI_Two_Level, "NULL", NULL_MI_example, RISCV, VEGA_X86] + # this allows us to pass additional command line parameters + # the default is to add -j $(nproc), but some images + # require more specifications when built + include: + - command-line: -j $(nproc) + - image: ALL_CHI + command-line: --default=ALL PROTOCOL=CHI -j $(nproc) + - image: ALL_MSI + command-line: --default=ALL PROTOCOL=MSI -j $(nproc) + - image: ALL_MESI_Two_Level + command-line: --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) + - image: NULL_MI_example + command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) runs-on: [self-hosted, linux, x64, build] + needs: name-artifacts container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} steps: - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL" >> $GITHUB_OUTPUT - name: Build gem5 - run: | - scons build/ALL/gem5.opt -j $(nproc) + run: scons build/${{ matrix.image }}/gem5.opt ${{ matrix.command-line }} - uses: actions/upload-artifact@v3 with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ALL/gem5.opt + name: ${{ needs.name-artifacts.outputs.build-name }}${{ matrix.image }} + path: build/${{ matrix.image }}/gem5.opt retention-days: 5 - run: echo "This job's status is ${{ job.status }}." - build-gem5-ALL_CHI: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_CHI" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/ALL_CHI/gem5.opt --default=ALL PROTOCOL=CHI -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ALL_CHI/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-ARM: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ARM" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/ARM/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ARM/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-ALL_MSI: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_MSI" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/ALL_MSI/gem5.opt --default=ALL PROTOCOL=MSI -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ALL_MSI/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-ALL_MESI_Two_Level: - runs-on: [self-hosted, linux, x64, build,] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL_MESI_Two_Level" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/ALL_MESI_Two_Level/gem5.opt --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ALL_MESI_Two_Level/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-NULL: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-NULL" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/NULL/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/NULL/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-NULL_MI_example: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-NULL_MI_example" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/NULL_MI_example/gem5.opt --default=NULL PROTOCOL=MI_example -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/NULL_MI_example/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-RISCV: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-RISCV" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/RISCV/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/RISCV/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - build-gem5-VEGA_X86: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-VEGA_X86" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/VEGA_X86/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/VEGA_X86/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." - - - # This runs the unit tests for the build/ALL/unittests.debug build. - unittests-all-debug: + # this builds both unittests.fast and unittests.debug + unittests-fast-debug: + strategy: + matrix: + type: [fast, debug] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest timeout-minutes: 60 - steps: - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. ref: develop - - name: ALL/unittests.debug UnitTests - run: | - ls - scons build/ALL/unittests.debug -j $(nproc) + - name: ALL/unittests.${{ matrix.type }} UnitTests + run: scons build/ALL/unittests.${{ matrix.type }} -j $(nproc) - # This runs the unit tests for the build/ALL/unittests.fast build. - unittests-all-fast: + # start running all of the long tests + testlib-long-tests: + strategy: + matrix: + test-type: [arm-boot-tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement-policies, riscv-boot-tests, stdlib, x86-boot-tests] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - timeout-minutes: 60 - - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - name: ALL/unittests.fast UnitTests - run: | - ls - scons build/ALL/unittests.fast -j $(nproc) - -# start running all of the long tests - testlib-long-arm-boot-tests: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5, build-gem5-ALL_CHI] - timeout-minutes: 1440 # 24 hours + needs: [name-artifacts, build-gem5] + timeout-minutes: 1440 # 24 hours for entire matrix to run steps: - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. ref: develop + # download all artifacts for each test + # since long tests can't start until the build matrix completes, + # we download all artifacts from the build for each test + # in this matrix - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}ALL path: build/ALL - run: chmod u+x build/ALL/gem5.opt - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5-ALL_CHI.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}ALL_CHI path: build/ALL_CHI - run: chmod u+x build/ALL_CHI/gem5.opt - - name: long arm-boot-tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/arm-boot-tests --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-fs: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5, build-gem5-ARM] - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-ARM.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}ARM path: build/ARM - run: chmod u+x build/ARM/gem5.opt - - name: long fs - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/fs --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-gem5_library_example_tests: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5, build-gem5-ALL_MESI_Two_Level] - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-ALL_MESI_Two_Level.outputs.build-name}} - path: build/ALL_MESI_Two_Level - - run: chmod u+x build/ALL_MESI_Two_Level/gem5.opt - - name: long gem5_library_example_tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/gem5_library_example_tests --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-gpu: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5, build-gem5-VEGA_X86] - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} - path: build/VEGA_X86 - - run: chmod u+x build/VEGA_X86/gem5.opt - - name: long gpu - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/gpu --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-insttest_se: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: long insttest_se - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/insttest_se --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - -# kvm tests don't work on github actions - # testlib-long-kvm-fork-tests: - # runs-on: [self-hosted, linux, x64, run] - # container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - # needs: build-gem5 - # timeout-minutes: 1440 # 24 hours - # steps: - # - uses: actions/checkout@v3 - # with: - # # Scheduled workflows run on the default branch by default. We - # # therefore need to explicitly checkout the develop branch. - # ref: develop - # - uses: actions/download-artifact@v3 - # with: - # name: ${{ env.artifact-name }} - # path: build/ALL/gem5.opt - # - run: chmod u+x build/ALL/gem5.opt - # - name: long kvm-fork-tests - # working-directory: ${{ github.workspace }}/tests - # run: ./main.py run gem5/kvm-fork-tests --length=long --skip-build -vv -t $(nproc) - # - uses: actions/upload-artifact@v3 - # env: - # MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - # with: - # name: ${{ env.MY_STEP_VAR }} - # path: tests/testing-results - # retention-days: 7 - # - run: echo "This job's status is ${{ job.status }}." - - # testlib-long-kvm-switch-tests: - # runs-on: [self-hosted, linux, x64, run] - # container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - # needs: build-gem5 - # timeout-minutes: 1440 # 24 hours - # steps: - # - uses: actions/checkout@v3 - # with: - # # Scheduled workflows run on the default branch by default. We - # # therefore need to explicitly checkout the develop branch. - # ref: develop - # - uses: actions/download-artifact@v3 - # with: - # name: ${{ env.artifact-name }} - # path: build/ALL/gem5.opt - # - run: chmod u+x build/ALL/gem5.opt - # - name: long kvm-switch-tests - # working-directory: ${{ github.workspace }}/tests - # run: ./main.py run gem5/kvm-switch-tests --length=long --skip-build -vv -t $(nproc) - # - uses: actions/upload-artifact@v3 - # env: - # MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - # with: - # name: ${{ env.MY_STEP_VAR }} - # path: tests/testing-results - # retention-days: 7 - # - run: echo "This job's status is ${{ job.status }}." - - testlib-long-learning_gem5: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5-ALL_MSI - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-ALL_MSI.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}ALL_MSI path: build/ALL_MSI - run: chmod u+x build/ALL_MSI/gem5.opt - - name: long learning_gem5 - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/learning_gem5 --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-m5_threads: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: long m5_threads - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/m5threads_test_atomic --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-memory: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5-NULL - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop + name: ${{needs.name-artifacts.outputs.build-name}}ALL_MESI_Two_Level + path: build/ALL_MESI_Two_Level + - run: chmod u+x build/ALL_MESI_Two_Level/gem5.opt - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5-NULL.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}NULL path: build/NULL - run: chmod u+x build/NULL/gem5.opt - - name: long memory - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/memory --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-multi_isa: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5-ARM, build-gem5-VEGA_X86, build-gem5-RISCV] - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5-ARM.outputs.build-name}} - path: build/ARM - - run: chmod u+x build/ARM/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} - path: build/VEGA_X86 - - run: chmod u+x build/VEGA_X86/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-RISCV.outputs.build-name}} - path: build/RISCV - - run: chmod u+x build/RISCV/gem5.opt - - name: long multi_isa - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/multi_isa --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-replacement-policies: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5-NULL_MI_example - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-NULL_MI_example.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}NULL_MI_example path: build/NULL_MI_example - run: chmod u+x build/NULL_MI_example/gem5.opt - - name: long replacement-policies - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/replacement-policies --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-riscv-boot-tests: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: long riscv-boot-tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/riscv-boot-tests --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." - - testlib-long-stdlib: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5-ARM, build-gem5-VEGA_X86, build-gem5-RISCV] - timeout-minutes: 1440 # 24 hours - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-ARM.outputs.build-name}} - path: build/ARM - - run: chmod u+x build/ARM/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-VEGA_X86.outputs.build-name}} - path: build/VEGA_X86 - - run: chmod u+x build/VEGA_X86/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5-RISCV.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}RISCV path: build/RISCV - run: chmod u+x build/RISCV/gem5.opt - - name: long stdlib + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}VEGA_X86 + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt + # run test + - name: long ${{ matrix.test-type }} tests working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/stdlib --length=long --skip-build -vv -t $(nproc) + run: ./main.py run gem5/${{ matrix.image }} --length=long --skip-build -vv -t $(nproc) - name: create zip of results if: success() || failure() run: | @@ -762,12 +152,23 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + if: success() || failure() + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." - testlib-long-x86-boot-tests: + # split library example tests into runs based on Suite UID + # so that they don't hog the runners for too long + testlib-long-gem5_library_example_tests: runs-on: [self-hosted, linux, x64, run] + strategy: + matrix: + test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 + needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours steps: - uses: actions/checkout@v3 @@ -777,12 +178,12 @@ jobs: ref: develop - uses: actions/download-artifact@v3 with: - name: ${{needs.build-gem5.outputs.build-name}} + name: ${{needs.name-artifacts.outputs.build-name}}ALL path: build/ALL - run: chmod u+x build/ALL/gem5.opt - - name: long x86-boot-tests + - name: long ${{ matrix.test-type }} gem5_library_example_tests working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/x86-boot-tests --length=long --skip-build -vv -t $(nproc) + run: ./main.py run --uid SuiteUID:tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py:test-${{ matrix.test-type }} --length=long --skip-build -vv - name: create zip of results if: success() || failure() run: | @@ -797,6 +198,12 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + if: success() || failure() + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." # This runs the SST-gem5 integration compilation and tests it with diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml new file mode 100644 index 0000000000..91d0bf1722 --- /dev/null +++ b/.github/workflows/utils.yaml @@ -0,0 +1,19 @@ +# This workflow file contains miscellaneous tasks to manage the repository. +name: Utils for Repository +on: + schedule: + - cron: '30 1 * * *' + workflow_dispatch: + +jobs: + # This job runs the stale action to close issues that have been inactive for 30 days. + # It is scheduled to run every day at 1:30 AM UTC. + close-stale-issues: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8.0.0 + with: + close-issue-message: 'This issue is being closed because it has been inactive waiting for response for 30 days. If this is still an issue, please open a new issue and reference this one.' + days-before-stale: 21 + days-before-close: 7 + any-of-labels: 'needs details' diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 4c3f6b55bc..dd81555f00 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -67,6 +67,11 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." @@ -103,6 +108,11 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." testlib-very-long-parsec-benchmarks: @@ -138,6 +148,11 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." testlib-very-long-x86-boot-tests: @@ -173,4 +188,9 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." From bfdb20fa104d576e3c703cd88f1d5801f137c02e Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 25 Jul 2023 19:43:24 -0700 Subject: [PATCH 39/55] misc: Update TESTING.md for subset selection This change: 1. Removes the 'Specifying a subset of tests to run' section. This section is no longer useful since tests are no longer divided up so neatly by tags as they once were. 2. Adds a section outlining the 'quick', 'long' and 'very-long' tests and how they may be selected and run. Change-Id: I61370dd80cc925a15d1a22755faa7d62e810862f --- TESTING.md | 63 +++++++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 41 deletions(-) diff --git a/TESTING.md b/TESTING.md index 50cc9bea3e..0f65c2b322 100644 --- a/TESTING.md +++ b/TESTING.md @@ -77,54 +77,35 @@ arguments: This will load every test in directory1 and directory2 (and their subdirectories). -## Specifying a subset of tests to run +## 'quick', 'long', and 'very-long' tests -You can use the tag query interface to specify the exact tests you want to run. -For instance, if you want to run only with `gem5.opt`, you can use +There are three categoties of tests which may be run from the "tests" directory: -```shell -./main.py run --variant opt +1. **'quick' tests**. This suite of tests are designed to finish execution in a few hours, inclusive of compilation of gem5. +We run these as part of our continuous integration tests on pull requests made to our repository. +These tests all utilize a binary build `scons build/ALL/gem5.opt`, and thus only rely on a single compilation for the tests to run. +2. **'long' tests**. This suite of tests are designed to finish execution in around 12 hours. +They incorporate longer running tests which are unsuitable to run as part of the 'quick' tests. +We run these daily via a scheduled job. +3. **'very-long' tests**. This suite of tests are designed to finish execution in days. +They incorporate tests which are too long to run frequntly +We run these daily via a scheduled job. + +When executing `./main.py run` the 'quick' tests are executed. +To run the 'long' tests execute: + +```sh +./main.py run --length=long ``` -Or, if you want to just run quick tests with the `gem5.opt` binary: +and to run the 'very-long' tests execute: -```shell -./main.py run --length quick --variant opt +```sh +./main.py run --length=very-long ``` - -To view all of the available tags, use - -```shell -./main.py list --all-tags -``` - -The output is split into tag *types* (e.g., isa, variant, length) and the -tags for each type are listed after the type name. - -Note that when using the isa tag type, tests were traditionally sorted based -on what compilation it required. However, as tests have switched to all be -compiled under the ALL compilation, which includes all ISAs so one doesn't -need to compile each one individually, using the isa tag for ISAs other than -ALL has become a less optimal way of searching for tests. It would instead -be better to run subsets of tests based on their directories, as described -above. - -You can specify "or" between tags within the same type by using the tag flag -multiple times. For instance, to run everything that is tagged "opt" or "fast" -use - -```shell -./main.py run --variant opt --variant fast -``` - -You can also specify "and" between different types of tags by specifying more -than one type on the command line. For instance, this will only run tests with -both the "ALL" and "opt" tags. - -```shell -./main.py run --isa All --variant opt -``` +In most cases we recommend running the 'quick' tests for most changes. +Only in some cases, such as contributions which significantly change the codebase, do we recommend running the 'long' or 'very-long' suite. ## Running tests in batch From 2dfdcac40fcd270754d39a11a51ac545e43829d7 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 25 Jul 2023 19:50:17 -0700 Subject: [PATCH 40/55] misc: Remove test binary sections from TESTING.md These sections are very out-of-date and confusing. Change-Id: I61aae0686f38671e46412e27ea516a5e06f4e6f2 --- TESTING.md | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/TESTING.md b/TESTING.md index 0f65c2b322..6ff79570bd 100644 --- a/TESTING.md +++ b/TESTING.md @@ -194,44 +194,6 @@ test library executes each python file it finds searching for tests. It's okay if the file causes an exception. This means there are no tests in that file (e.g., it's not a new-style test). - -## Binary test applications - -The code for some test binaries that are run in the gem5 guest during -testing can be found in `tests/test-progs`. -There's one directory per test application. -The source code is under the `source` directory. - -You may have a `bin` directory as well. -The `bin` directory is automatically created when running the test case that -uses the test binary. -This is not the case when a test is run via the --bin-path option. -In that scenario a bin directory will be created in the selected path -rather than in `tests/test-progs`. -The binary is downloaded from the gem5 servers the first -time it is referenced by a test. - -Some other tests (like Linux-boot) don't have sources inside gem5 and -are simply downloaded from gem5 servers. - -## Updating the test binaries - -The test infrastructure should check with the gem5 servers to ensure you have -the latest binaries. However, if you believe your binaries are out of date, -simply delete the `bin` directory and they will be re-downloaded to your local -machine. - -## Building (new-style) test binaries - -In each `src/` directory under `tests/test-progs`, there is a Makefile. -This Makefile downloads a docker image and builds the test binary for some ISA -(e.g., Makefile.x86 builds the binary for x86). Additionally, if you run `make -upload` it will upload the binaries to the gem5 server, if you have access to -modify the binaries. *If you need to modify the binaries for updating a test or -adding a new test and you don't have access to the gem5 server, contact a -maintainer (see MAINTAINERS).* - - ## Running Tests in Parallel Whimsy has support for parallel testing baked in. This system supports From 2b0a1df096bc10b18f3909347d104cdad54daa55 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 25 Jul 2023 20:04:43 -0700 Subject: [PATCH 41/55] misc: Add 'testing resources' sec to TESTING.md Change-Id: Ie8a9c9200461d4f9e272dea75de1755b1b18aceb --- TESTING.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/TESTING.md b/TESTING.md index 6ff79570bd..2c0ab0b3ab 100644 --- a/TESTING.md +++ b/TESTING.md @@ -7,7 +7,7 @@ gem5's testing infrastructure has the following goals: * Fast execution in the simple case * High coverage of gem5 code -# Running unit tests +## Running unit tests gem5 comes with unit tests, created using the Google Test framework. These can be built through SCons. @@ -41,7 +41,7 @@ To run a specific test function (e.g., BitUnionData.NormalBitfield): ./build/ALL/base/bitunion.test.opt --gtest_filter=BitUnionData.NormalBitfield ``` -# Running system-level tests +## Running system-level tests Within the `tests` directory we have system-level tests. These tests run the gem5 framework against various hardware configurations, with different @@ -77,7 +77,7 @@ arguments: This will load every test in directory1 and directory2 (and their subdirectories). -## 'quick', 'long', and 'very-long' tests +### 'quick', 'long', and 'very-long' tests There are three categoties of tests which may be run from the "tests" directory: @@ -107,7 +107,7 @@ and to run the 'very-long' tests execute: In most cases we recommend running the 'quick' tests for most changes. Only in some cases, such as contributions which significantly change the codebase, do we recommend running the 'long' or 'very-long' suite. -## Running tests in batch +### Running tests in batch The testing infrastructure provides the two needed methods to run tests in batch. First, you can list all of the tests based on the same tags as above in @@ -141,7 +141,7 @@ run more than one uid, you must call `./main.py` multiple times. Currently, you must specify `--skip-build` if you want to run a single suite or run in batch mode. Otherwise, you will build gem5 for all architectures. -## Rerunning failed tests +### Rerunning failed tests While developing software a common practice is to run tests, make a change, and assert that the tests still pass. If tests fail you'll likely want to @@ -159,7 +159,7 @@ using the `rerun` command. ./main.py rerun ``` -## If something goes wrong +### If something goes wrong The first step is to turn up the verbosity of the output using `-v`. This will allow you to see what tests are running and why a test is failing. @@ -167,7 +167,7 @@ allow you to see what tests are running and why a test is failing. If a test fails, the temporary directory where the gem5 output was saved is kept and the path to the directory is printed in the terminal. -## Debugging the testing infrastructure +### Debugging the testing infrastructure Every command takes an option for the verbosity. `-v`, `-vv`, `-vvv` will increase the verbosity level. If something isn't working correctly, you can @@ -178,7 +178,7 @@ contains the base code for tests, suites, fixtures, etc. The code in tests/gem5 is *gem5-specific* code. For the most part, the code in tests/gem5 extends the structures in ext/testlib. -## Common errors +### Common errors You may see a number of lines of output during test discovery that look like the following: @@ -194,7 +194,7 @@ test library executes each python file it finds searching for tests. It's okay if the file causes an exception. This means there are no tests in that file (e.g., it's not a new-style test). -## Running Tests in Parallel +### Running Tests in Parallel Whimsy has support for parallel testing baked in. This system supports running multiple suites at the same time on the same computer. To run @@ -204,6 +204,13 @@ For example, to run up to three test suites at the same time:: ./main.py run --skip-build -t 3 +### Testing resources + +By default binaries and testing resources are obtained via the [gem5 resources infrastructure](https://www.gem5.org/documentation/general_docs/gem5_resources/). +The downloaded resources are cached in "tests/gem5/resources". +The resources are cached to avoid re-downloading when tests are run multiple times, though some of these resources, such as disk images, are large. +It is therefore recommended you remove the "tests/gem5/resources" directory when you are done testing. + ## Running Tests within GitHub Actions To run these tests within GitHub Actions, we use the format of running From 22628b861c57f856feec232d1ddff016ca4de86f Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 25 Jul 2023 20:22:00 -0700 Subject: [PATCH 42/55] misc: Update GitHub Actions text in TESTING.md This change simplifies the explanation of how GitHub actions works. Change-Id: Ia1540008463b8584f172c40ca7b4826cbbf95eb7 --- TESTING.md | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/TESTING.md b/TESTING.md index 2c0ab0b3ab..f92d6abda6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -213,39 +213,15 @@ It is therefore recommended you remove the "tests/gem5/resources" directory when ## Running Tests within GitHub Actions -To run these tests within GitHub Actions, we use the format of running -tests by directory as shown above in the "Running Tests from Multiple -Directories" section. These tests are run within workflow files, -which can be found in the .github directory of this repository. -You can learn more about workflows -[here](https://docs.github.com/en/actions/using-workflows/about-workflows). +These tests outlined here are run as part of [GitHub Actions](https://github.com/features/actions). +These are outlined in [workflow files](https://docs.github.com/en/actions/using-workflows/about-workflows), which can be found in the repo's ".github" directory. +Each workflow is made up of individual jobs where ecch job consists of a series of steps which are executed within a [GitHub Runner](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners). -Each workflow is made up of individual jobs, where each job is a set -of tests that is executed on a runner within GitHub. In each -workflow, each version of gem5.opt needed is first built, and then -stored as an artifact for main.py to use. +### Adding Tests to GitHub Actions -There are two sets of runners within the gem5 repository: builders and -runners. The builders have more resources available to allow for a -quicker compilation of gem5, while the runners have less, and are -meant to only run tests. +To ensure tests added are run in GitHub Actions you may need to modify the worklfow files. +For tests run via `./main.py` we split up the tests via the subdirectories in "tests/gem5". +For example, all tests under "test/gem5/cpu_tests" are run as one job. +Therefore tests added to existing directories are likely to be included, but modifications to the workflow files may be needed if new directories are added. -After the gem5 artifact has been uploaded, a runner can then download -the versions needed for their tests. For example, in the daily-tests.yaml, -in order to run the multi_isa tests, you need artifacts of ARM, RISCV, -and VEGA_X86. - -## Adding Tests to GitHub Actions - -In order to add new tests to our GitHub Actions testing infastructure, -follow the format currently shown in the existing workflows. If the -new tests were added to an already existing directory (ex. A very-long -test in the gem5_library_example_tests), it will automatically be -included into the weekly testing, since weekly-tests.yaml already -contains a job for the gem5_library_example_tests. - -However, if a new directory is added to the tests, you need to manually -add a new step to the GitHub workflows. This would consist of both a -step to build whatever version of gem5 was required if it wasn't -already included in the file, as well as a step to run main.py -in the given directory after downloading the gem5 artifact. +We strongly recommend that when adding or ammending tests, that contributors check the ".github/workflows" files to ensure the tests they specify will be run as intended. From 340a5a9a57c9916a6696b0087c92b9fc537dbc8d Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 25 Jul 2023 20:34:11 -0700 Subject: [PATCH 43/55] misc: Add Pyunit Test info to TESTING.md Change-Id: Ibff77963653600ac7c9d706edca882d95e5c47df --- TESTING.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index f92d6abda6..19965157fb 100644 --- a/TESTING.md +++ b/TESTING.md @@ -7,9 +7,9 @@ gem5's testing infrastructure has the following goals: * Fast execution in the simple case * High coverage of gem5 code -## Running unit tests +## Running the CPP unit tests -gem5 comes with unit tests, created using the Google Test framework. These can +gem5 comes with unit tests for CPP, created using the Google Test framework. These can be built through SCons. To build and run all the unit tests: @@ -41,9 +41,30 @@ To run a specific test function (e.g., BitUnionData.NormalBitfield): ./build/ALL/base/bitunion.test.opt --gtest_filter=BitUnionData.NormalBitfield ``` +## Running the Python unit tests + +gem5 comes with Python unit tests. +These are built using the [Python unit testing framework](https://docs.python.org/3/library/unittest.html). +These tests can be found in "tests/gem5/pyunit". + +To run these tests a gem5 binary must first be compiled. +We recommend, `build/ALL/gem5.opt`: + +```sh +scons build/ALL/gem5.opt -j {number of compilation threads} +``` + +Then the Pyunit tests may be executed using: + +```sh +./build/ALL/gem5.opt tests/run_pyunit.py +``` + +**Note**: These tests are also run via the 'quick' system-level tests, explained below. + ## Running system-level tests -Within the `tests` directory we have system-level tests. These tests run +Within the "tests/gem5" directory we have system-level tests. These tests run the gem5 framework against various hardware configurations, with different ISAs, then verify the simulations execute correctly. These should be seen as high-level, coarse-grained tests to compliment the unit-tests. From d7b17d58700d401b8a9420bc1941047a6b6f18dc Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 26 Jul 2023 17:17:33 -0700 Subject: [PATCH 44/55] misc: Sync CONTRIBUTING.md with website This change syncs the repo's contributing documentation with that of the website's contributing documentation: https://www.gem5.org/contributing From now on we'll attempt to keep the repo's CONTRIBUTING.md documentation in sync with that on the website. Change-Id: I2c91e6dd5cd7a9b642377878b007d7da3f0ee2ad --- CONTRIBUTING.md | 725 +++++++++++++++++++++++------------------------- 1 file changed, 349 insertions(+), 376 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b41a73a4a6..208c9444e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,425 +1,390 @@ -If you've made changes to gem5 that might benefit others, we strongly encourage -you to contribute those changes to the public gem5 repository. There are -several reasons to do this: - * Share your work with others, so that they can benefit from new functionality. - * Support the scientific principle by enabling others to evaluate your - suggestions without having to guess what you did. - * Once your changes are part of the main repo, you no longer have to merge - them back in every time you update your local repo. This can be a huge time - saving! - * Once your code is in the main repo, other people have to make their changes - work with your code, and not the other way around. - * Others may build on your contributions to make them even better, or extend - them in ways you did not have time to do. - * You will have the satisfaction of contributing back to the community. +This document serves as a guide to contributing to gem5. +The following subsections outline, in order, the steps involved in contributing +to the gem5 project. -The main method for contributing code to gem5 is via our code review website: -https://github.com/gem5/gem5/pulls/. This documents describes the details of -how to create code changes, upload your changes, have your changes -reviewed, and finally push your changes to gem5. More information can be found -from the following sources: - * http://gem5.org/contributing - * https://docs.github.com/en/pull-requests - * https://git-scm.com/book +## Determining what you can contribute +The easiest way to see how you can contribute to gem5 is to check our Jira +issue tracker: or GitHub issue tracker: +. -High-level flow for submitting changes -====================================== +Browse these open issues and see if there are any which you are capable of +handling. When you find a task you are happy to carry out, verify no one else +is presently assigned, then leave a comment asking if you may assign yourself +this task. Though not mandatory, we +advise first-time contributors do this so developers more familiar with the +task may give advice on how best to implement the necessary changes. - +-------------+ - | Make change | - +------+------+ - | - | - v - +-------------+ - | Run tests |<--------------+ - +------+------+ | - | | - | | - v | - +------+------+ | - | Post review | | - +------+------+ | - | | - v | - +--------+---------+ | - | Wait for reviews | | - +--------+---------+ | - | | - | | - v | - +----+----+ No +------+------+ - |Reviewers+--------->+ Update code | - |happy? | +------+------+ - +----+----+ ^ - | | - | Yes | - v | - +----+-----+ No | - |Maintainer+----------------+ - |happy? | - +----+-----+ - | - | Yes - v - +------+------+ - | Submit code | - +-------------+ +Once a developers has replied to your comment (and given any advice they may +have), you may officially assign yourself the task. This helps the gem5 +development community understand which parts of the project are presently being +worked on. -After creating your change to gem5, you can post a review to git -via a pull request at: https://github.com/gem5/gem5/pulls/. Before being able to -submit your code to the mainline of gem5, the code is reviewed by others in the -community. Additionally, the maintainer for that part of the code must sign off -on it. +**If, for whatever reason, you stop working on a task, please unassign +yourself from the task.** -Cloning the gem5 repo to contribute -=================================== +## Obtaining the git repo -If you plan on contributing, it is strongly encouraged for you to clone the -repository directly, and checkout the `develop` branch from our git instance -at https://github.com/gem5/gem5/. +The gem5 git repository is hosted at . +**Please note: contributions made to other gem5 repos +will not be considered. Please contribute to +exclusively.** -To clone the gem5 repository: +To pull the gem5 git repo: -``` - git clone https://github.com/gem5/gem5/ +```sh +git clone https://github.com/gem5/gem5 ``` -By default, the stable branch is checked out. The stable branch contains the -latest released version of gem5. To obtain code still under-development (and -which contributions can be made): +If you wish to use gem5 and never contribute, this is fine. However, to +contribute, we use the [GitHub Pull-Request model](https://docs.github.com/en/pull-requests), and therefore recommend [Forking the gem5 repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo) prior to contributing. -``` -cd gem5 -git checkout --track origin/develop +### Forking + +Please consult the [GitHub documentation on Forking a GitHub repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo). +As we will be working atop the `develop` branch, please ensure you Fork all the repository's branches, not just the `stable` branch. + +This will create your own forked version of the gem5 repo on your own GitHub account. +You may then obtain it locally using: + +```sh +git clone https://github.com/{your github account}/gem5 ``` -Changes should be made to this develop branch. Changes to the stable branch -will be blocked. Once a change on the develop branch is properly incorporated -into the gem5 repo it will be merged into the stable branch upon the next -release of gem5. New releases of gem5 occur three times a year. Ergo, changes -made to the develop branch should appear on the stable branch within three to -four months as part of a stable release. +### stable / develop branch -Other gem5 repositories ------------------------ +When cloned the git repo will have the `stable` branch checked-out by default. The +`stable` branch is the gem5 stable release branch. I.e., the HEAD +of this branch contains the latest stable release of gem5. (execute `git tag` +on the `stable` branch to see the list of stable releases. A particular +release may be checked out by executing `git checkout `). As the +`stable` branch only contains officially released gem5 code **contributors +should not develop changes on top of the `stable` branch** they should instead +**develop changes on top of the `develop` branch**. -There are a few repositories other than the main gem5 development repository. +To switch to the `develop` branch: - * public/m5threads: The code for a pthreads implementation that works with - gem5's syscall emulation mode. - * public/gem5-resources: Resources to enable computer architecture research - with gem5. See the README.md file in the gem5-resources repository for more - information. - * public/gem5-website: The gem5.org website source. See the README.md file in - the gem5-website repository for more information. +```sh +git switch develop +``` -Making changes to gem5 -====================== +The develop `branch` is merged into the `stable` branch upon a gem5 release. +Therefore, any changes you make exist on the develop branch until the next release. -It is strongly encouraged to use git branches when making changes to gem5. -Additionally, keeping changes small and concise and only have a single logical -change per commit. +We strongly recommend creating your own local branches to do changes. +The flow of development works best if `develop` and `stable` are not modified directly. +This helps keep your changes organized across different branches in your forked repository. +The following example will create a new branch, from `develop`, called `new-feature`: -Unlike our previous flow with Mercurial and patch queues, when using git, you -will be committing changes to your local branch. By using separate branches in -git, you will be able to pull in and merge changes from mainline and simply -keep up with upstream changes. +```sh +git switch -c new-feature +``` -We use a rebase-always model for contributions to the develop branch of gem5. -In this model, the changes are rebased on top of the tip of develop instead of -merged. This means that to contribute, you will have to frequently rebase any -feature branches on top of develop. If you see a "merge conflict" in gerrit, it -can often be solved with a simple rebase. To find out more information about -rebasing and git, see the [git book]. +## Making modifications -[git book]: https://git-scm.com/book/en/v2/Git-Branching-Rebasing +### C/CPP +Different tasks will require the project to be modified in different ways. +Though, in all cases, our style-guide must be adhered to. The full C/C++ style +guide is outlined [here](/documentation/general_docs/development/coding_style). -Setting up pre-commit ---------------------- +As a high-level overview: -To help ensure the gem5 style guide is maintained, we use [pre-commit]( -https://pre-commit.com) to run checks on changes to be contributed. +* Lines must not exceed 79 characters in length. +* There should be no trailing white-space on any line. +* Indentations must be 4 spaces (no tab characters). +* Class names must use upper camel case (e.g., `ThisIsAClass`). +* Class member variables must use lower camel case (e.g., +`thisIsAMemberVariable`). +* Class member variables with their own public accessor must start with an +underscore (e.g., `_variableWithAccessor`). +* Local variables must use snake case (e.g., `this_is_a_local_variable`). +* Functions must use lower camel case (e.g., `thisIsAFunction`) +* Function parameters must use snake case. +* Macros must be in all caps with underscores (e.g., `THIS_IS_A_MACRO`). +* Function declaration return types must be on their own line. +* Function brackets must be on their own line. +* `for`/`if`/`while` branching operations must be followed by a white-space +before the conditional statement (e.g., `for (...)`). +* `for`/`if`/`while` branching operations' opening bracket must be on the +same line, with the closing bracket on its own line (e.g., +`for (...) {\n ... \n}\n`). There should be a space between the condition(s) +and the opening bracket. +* C++ access modifies must be indented by two spaces, with method/variables +defined within indented by four spaces. -To setup pre-commit, run the following in your gem5 directory to install the -pre-commit and commit message hooks. +Below is a simple toy example of how a class should be formatted: + +```C++ +#DEFINE EXAMPLE_MACRO 7 +class ExampleClass +{ + private: + int _fooBar; + int barFoo; + + public: + int + getFooBar() + { + return _fooBar; + } + + int + aFunction(int parameter_one, int parameter_two) + { + int local_variable = 0; + if (true) { + int local_variable = parameter_one + parameter_two + barFoo + + EXAMPLE_MACRO; + } + return local_variable; + } + +} +``` + +### Python + +We use [Python Black](https://github.com/psf/black) to format our Python code +to the correct style. To install: + +```sh +pip install black +``` + +Then run on modified/added python files using: + +```sh +black +``` + +For variable/method/etc. naming conventions, please follow the [PEP 8 naming +convention recommendations]( +https://peps.python.org/pep-0008/#naming-conventions). While we try our best to +enforce naming conventions across the gem5 project, we are aware there are +instances where they are not. In such cases please **follow the convention +of the code you are modifying**. + +### Using pre-commit + +To help enforce our style guide we use use [pre-commit]( +https://pre-commit.com). pre-commit is a git hook and, as such, must be +explicitly installed by a gem5 developer. + +To install the gem5 pre-commit checks, execute the following in the gem5 +directory: ```sh pip install pre-commit -pre-commit install -t pre-commit -t commit-msg +pre-commit install ``` -The hooks are also automatically installed when gem5 is compiled. +Once installed pre-commit will run checks on modified code prior to running the +`git commit` command (see [our section on committing](#committing) for more +details on committing your changes). If these tests fail you will not be able to +commit. -When you run a `git commit` command the pre-commit hook will run checks on your -committed code. The commit will be blocked if a check fails. +These same pre-commit checks are run as part our CI checks (those +which must pass in order for a change to be merged into the develop branch). It +is therefore strongly recommended that developers install pre-commit to catch +style errors early. -The same checks are run as part of github actions CI tests (those required to obtain -a Verified label, necessary for a change to be accepted to the develop branch). -Therefore setting up pre-commit in your local gem5 development environment is -recommended. +## Compiling and running tests -You can automatically format files to pass the pre-commit tests by running: +The minimum criteria for a change to be submitted is that the code is +compilable and the test cases pass. + +The following command both compiles the project and runs our "quick" +system-level checks: ```sh -pre-commit run --files +cd tests +./main.py run ``` -Requirements for change descriptions ------------------------------------- -To help reviewers and future contributors more easily understand and track -changes, we require all change descriptions be strictly formatted. +**Note: These tests can take several hours to build and execute. `main.py` may +be run on multiple threads with the `-j` flag. E.g.: `python main.py run +-j6`.** -A canonical commit message consists of three parts: - * A short summary line describing the change. This line starts with one or - more keywords (found in the MAINTAINERS file) separated by commas followed - by a colon and a description of the change. This short description is - written in the imperative mood, and should say what happens when the patch - is applied. Keep it short and simple. Write it in sentence case preferably - not ending in a period. This line should be no more than 65 characters long - since version control systems usually add a prefix that causes line-wrapping - for longer lines. - * (Optional, but highly recommended) A detailed description. This describes - what you have done and why. If the change isn't obvious, you might want to - motivate why it is needed. Lines need to be wrapped to 72 characters or - less. Leave a blank line between the first short summary line and this - detailed description. - * Tags describing patch metadata. You are highly recommended to use - tags to acknowledge reviewers for their work. +The unit tests should also pass. To run the unit tests: -Tags are an optional mechanism to store additional metadata about a patch and -acknowledge people who reported a bug or reviewed that patch. Tags are -generally appended to the end of the commit message in the order they happen. -We currently use the following tags: - * Signed-off-by: Added by the author and the submitter (if different). - This tag is a statement saying that you believe the patch to be correct and - have the right to submit the patch according to the license in the affected - files. Similarly, if you commit someone else's patch, this tells the rest - of the world that you have have the right to forward it to the main - repository. If you need to make any changes at all to submit the change, - these should be described within hard brackets just before your - Signed-off-by tag. By adding this line, the contributor certifies the - contribution is made under the terms of the Developer Certificate of Origin - (DCO) [https://developercertificate.org/]. - * Reviewed-by: Used to acknowledge patch reviewers. It's generally considered - good form to add these. Added automatically. - * Reported-by: Used to acknowledge someone for finding and reporting a bug. - * Reviewed-on: Link to the review request corresponding to this patch. Added - automatically. - * Change-Id: Used by Gerrit to track changes across rebases. Added - automatically with a commit hook by git. - * Tested-by: Used to acknowledge people who tested a patch. Sometimes added - automatically by review systems that integrate with CI systems. - * Issue-On: Used to link a commit to an issue in gem5's [issue tracker]. The - format should be https://gem5.atlassian.net/browse/GEM5- - -[issue tracker]: https://gem5.atlassian.net/ - -Other than the "Signed-off-by", "Issue-On", "Reported-by", and "Tested-by" -tags, you generally don't need to add these manually as they are added -automatically by Gerrit. - -It is encouraged for the author of the patch and the submitter to add a -Signed-off-by tag to the commit message. By adding this line, the contributor -certifies the contribution is made under the terms of the Developer Certificate -of Origin (DCO) [https://developercertificate.org/]. - -If your change relates to a [Jira Issue](https://gem5.atlassian.net), it is -advised that you provide a link to the issue in the commit message (or messages -if the Jira Issue relates to multiple commits). Though optional, doing this -can help reviewers understand the context of a change. - -It is imperative that you use your real name and your real email address in -both tags and in the author field of the changeset. - -For significant changes, authors are encouraged to add copyright information -and their names at the beginning of the file. The main purpose of the author -names on the file is to track who is most knowledgeable about the file (e.g., -who has contributed a significant amount of code to the file). The -`util/update-copyright.py` helper script can help to keep your copyright dates -up-to-date when you make further changes to files which already have your -copyright but with older dates. - -Note: If you do not follow these guidelines, the github actions will -automatically reject your patch. -If this happens, update your changeset descriptions to match the required style -and resubmit. The following is a useful git command to update the most recent -commit (HEAD). - -``` - git commit --amend +```sh +scons build/NULL/unittests.opt ``` -Running tests -============= +To compile an individual gem5 binary: -Before posting a change to the code review site, you should always run the -quick tests! -See TESTING.md for more information. - -Posting a review -================ - -If you have not signed up for an account on the github -(https://github.com/), you first have to create an account. - -Setting up an account ---------------------- - 1. Go to https://github.com/ - 2. Click "Sign up" in the upper right corner. - -Submitting a change -------------------- - -In github, to submit a review request, you can simply push your git commits to -a special named branch. For more information on git push see -https://git-scm.com/docs/git-push. - -Push changes to GitHub ----------------------------- -1. Fork the gem5 repository on GitHub from https://github.com/gem5/gem5/. -2. Create a new branch in your forked repository for your feature or bug fix. -3. Commit your changes to the new branch. -4. Push the branch to your forked repository. -5. Open a pull request from your branch in your forked repository to the main gem5 repository. - -We will continue to use the “develop” branch for development, so please ensure your pull requests are for the gem5 develop branch. Pull requests to the stable branch will be blocked. - -Branches -======== - -By default, contributions to gem5 should be made on the develop branch. The -stable branch is maintained as a stable release branch (i.e., it can be pulled -to obtain the latest official release of gem5). Creation of additional branches -is generally discouraged due to their tendency to bloat git repositories with -abandoned code. - -Reviewing patches -================= - -Reviewing patches is done on our github instance at -https://github.com/gem5/gem5/pulls/. - -After logging in with your GitHub account, you will be able to comment, review, -and push your own patches as well as review others' patches. All gem5 users are -encouraged to review patches. The only requirement to review patches is to be -polite and respectful of others. - -There are multiple labels in Gerrit that can be applied to each review detailed -below. - * Code-review: This is used by any gem5 user to review patches. When reviewing - a patch you can give it a score of -2 to +2 with the following semantics. - * -2: This blocks the patch. You believe that this patch should never be - committed. This label should be very rarely used. - * -1: You would prefer this is not merged as is - * 0: No score - * +1: This patch seems good, but you aren't 100% confident that it should be - pushed. - * +2: This is a good patch and should be pushed as is. - * Maintainer: Currently only PMC members are maintainers. At least one - maintainer must review your patch and give it a +1 before it can be merged. - * Verified: This is automatically generated from the continuous integrated - (CI) tests. Each patch must receive at least a +1 from the CI tests before - the patch can be merged. The patch will receive a +1 if gem5 builds and - runs, and it will receive a +2 if the stats match. - * Style-Check: This is automatically generated and tests the patch against the - gem5 code style - (http://www.gem5.org/documentation/general_docs/development/coding_style/). - The patch must receive a +1 from the style checker to be pushed. - -Note: Whenever the patch creator updates the patch all reviewers must re-review -the patch. There is no longer a "Fix it, then Ship It" option. - -Once you have received reviews for your patch, you will likely need to make -changes. To do this, you should update the original git changeset. Then, you -can simply push the changeset again to the same Gerrit branch to update the -review request. - -``` - git push origin HEAD:refs/for/develop +```sh +scons build/ALL/gem5.opt ``` -Committing changes -================== +This compiles a gem5 binary containing "ALL" ISA targets. For more information +on building gem5 please consult our [building documentation]( +/documentation/general_docs/building). -Each patch must meet the following criteria to be merged: - * At least one review with +2 - * At least one maintainer with +1 - * At least +1 from the CI tests (gem5 must build and run) - * At least +1 from the style checker +## Committing -Once a patch meets the above criteria, the submitter of the patch will be able -to merge the patch by pressing the "Submit" button on Gerrit. When the patch is -submitted, it is merged into the public gem5 branch. +When you feel your change is done, you may commit. Start by adding the changed +files: -Review moderation and guidelines --------------------------------- +```Shell +git add +``` -Once a change is submitted, reviewers shall review the change. This may require -several iterations before a merge. Comments from reviewers may include -questions, and requests for alterations to the change prior to merging. The -overarching philosophy in managing this process is that there should be -politeness and clear communication between all parties at all times, and, -whenever possible, permission should be asked before doing anything that may -inconvenience another party. Included below are some guidelines we expect -contributors and reviewers to follow. +Make sure these changes are being added to your forked repository. +Then commit using: - * In all forms of communication, contributors and reviewers must be polite. - Comments seen as being needlessly hostile or dismissive will not be - tolerated. - * Change contributors should respond to, or act upon, each item of feedback - given by reviewers. If there is disagreement with a piece of - feedback, a sufficiently detailed reason for this disagreement should - be given. Polite discussion, and sharing of information and expertise - is strongly encouraged. - * Contributors are advised to assign reviewers when submitting a change. - Anyone who contributes to gem5 can be assigned as a reviewer. However, - all changes must be accepted by at least one maintainer prior to a - merge, ergo assigning of at least one maintainer as a reviewer is - strongly recommended. Please see MAINTAINERS for a breakdown of - gem5 maintainers and which components they claim responsibility for. - Maintainers should be chosen based on which components the change is - targeting. Assigning of reviewers is not strictly enforced, though not - assigning reviewers may slow the time in which a change is reviewed. - * If a contributor posts a change and does not receive any reviews after two - working days (excluding regional holidays), it is acceptable to "prod" - reviewers. This can be done by adding a reply to the changeset review - (e.g., "Would it be possible for someone to review my change?"). If the - contributor has yet to assign reviewers, they are strongly advised to do so. - Reviewers will get notified when assigned to referee a change. - * By default, the original contributor is assumed to own a change. I.e., - they are assumed to be the sole party to submit patchsets. If someone - other than the original contributor wishes to submit patchsets to a - change on the original contributor's behalf, they should first ask - permission. If two working days pass without a response, a patchset may be - submitted without permission. Permission does not need to be asked to submit - a patchset consisting of minor, inoffensive, changes such a typo and format - fixes. - * Once a change is ready to merge, it enters a "Ready to Submit" state. The - original contributor should merge their change at this point, assuming they - are content with the commit in its present form. After two working days, a - reviewer may message a contributor to remind them of the change being in a - "Ready to Submit" state and ask if they can merge the change on the - contributors behalf. If a further two working days elapse without a - response, the reviewer may merge without permission. A contributor may keep - a change open for whatever reason though this should be communicated to the - reviewer when asked. - * After a month of inactivity from a contributor on an active change, a - reviewer may post a message on the change reminding the submitter, and - anyone else watching the change, of its active status and ask if they are - still interested in eventually merging the change. After two weeks of no - response the reviewer reserves the right to abandon the change under the - assumption there is no longer interest. - * The final arbiter in any dispute between reviewers and/or contributors - is the PMC (PMC members are highlighted in MAINTAINERS). Disputes requiring - intervention by the PMC are undesirable. Attempts should be made to resolve - disagreements via respectful and polite discourse before being escalated to - this level. +```Shell +git commit +``` -Releases -======== +The commit message must adhere to our style. The first line of the commit is +the "header". The header starts with a tag (or tags, separated by a comma), +then a colon. Which tags are used depend on which components of gem5 +you have modified. **Please refer to the [MAINTAINERS.yaml]( +https://github.com/gem5/gem5/blob/stable/MAINTAINERS.yaml) for +a comprehensive list of accepted tags**. After this colon a short description +of the commit must be provided. **This header line must not exceed 65 +characters**. + +After this, a more detailed description of the commit can be added. This is +inserted below the header, separated by an empty line. Including a description +is optional but it's strongly recommended. The description may span multiple +lines, and multiple paragraphs. **No line in the description may exceed 72 +characters.** + +To improve the navigability of the gem5 project we would appreciate if commit +messages include a link to the relevant Jira issue/issues. + +Below is an example of how a gem5 commit message should be formatted: + +``` +test,base: This commit tests some classes in the base component + +This is a more detailed description of the commit. This can be as long +as is necessary to adequately describe the change. + +A description may spawn multiple paragraphs if desired. + +Jira Issue: https://gem5.atlassian.net/browse/GEM5-186 +``` + +If you feel the need to change your commit, add the necessary files then +_amend_ the changes to the commit using: + +```sh +git commit --amend +``` + +This will give you opportunity to edit the commit message. + +You may continue to add more commits as a chain of commits to be included in the pull-request. +However, we recommend that pull-requests are kept small and focused. +For example, if you wish to add a different feature or fix a different bug, we recommend doing so in another pull requests. + +## Keeping your forked and local repositories up-to-date + +While working on your contribution, we recommend keeping your forked repository in-sync with the source gem5 repository. +To do so, regularly [Sync your fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork). +This can be done via the GitHub web interface and, if so, you should `git pull` on top of your local `stable` and `develop` branches to ensure your local repository is in-sync. +To do so from the command line: + +```sh +# Add the main gem5 repository as a remote on your local repository. This only +# needs done once. +git remote add upstream https://github.com/gem5/gem5.git + +git fetch upstream # Obtain the latest from the gem5 repo. +git switch develop # Switch to the develop branch. +git merge upstream/develop # Merge the latest changes into the develop branch. +git push # Push to develop to your forked repo. +git switch stable # Switch to the stable branch. +git merge upstream/stable # Merge the latest changes into the stable branch. +git push # Push the changes to stable to your forked repo. +``` + +As our local branch work atop the `develop` branch, once we've synced our forked repository, we can rebase our local branch on top of the `develop` branch. +Assuming our local branch is called `new-feature`: + +```sh +git switch develop # Switching back to the develop branch. +git pull # Ensuring we have the latest from the forked repository. +git switch new-feature # Switching back to our local branch. +git rebase develop # Rebasing our local branch on top of the develop branch. +``` + +Conflicts may need resolved between your branch and new changes. + +## Pushing and creating a pull request + +Once you have completed your changes locally, you can push to your forked gem5 repository. +Assuming the branch we are working on is `new-feature`: + +```sh +git switch new-feature # Ensure we are on the 'new-feature' branch. +git push --set-upstream origin new-feature +``` + +Now, via the GitHub web interface, you can [create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) of your changes from your forked repository's branch into the gem5 `develop` branch. + +## Passing the checks + +Once you have created a pull request, the gem5 Continuous Integration (CI) tests will run. +These run a series of checks to ensure your changes are valid. +These must pass before your changes can be merged into the gem5 `develop` branch. + +In addition to the CI tests, your changes will be reviewed by the gem5 community. +Your pull-request must have the approval of at least one community member prior to being merged. + +Once your pull-request has passed all the CI tests and has been approved by at least one community member, it will be merged a gem5 maintainer will do a [Merge](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges) on the pull-request. +The gem5 maintainers are individuals granted the ability to merge pull requests into the gem5 `develop` branch. + + +### Making iterative improvements based on feedback + +A reviewer will ask questions and post suggestions on GitHub. You should read +these comments and answer these questions. **All communications between +reviewers and contributors should be done in a polite manner. Rude and/or +dismissive remarks will not be tolerated.** + +When you understand what changes are required make amendments to the pull +request by adding patches to the same branch and then pushing to the forked repository. +A git "force push" (i.e., `git push --force`) is also acceptable if you wish to alter the commits locally in order to make the changes. +We encourage contributors to help keep our `git log` clean and readable. +We recommend that users rebase their changes frequently on top of the develop branch, squash their commits where appropriate (e.g., in cases where there are many small fix commits to a change in the same PR) then force push changes to keep their PR commits concise. + +Once pushed to the forked repository, the pull request will automatically update with your changes. +The reviewer will then re-review your changes and, if necessary, ask for further changes, or approve your pull-request. + +## Reviewing other contributions + +We encourage all gem5 developers to review other's contributions. +Anyone may review a gem5 change and, if they feel it is ready, approve it. +All pull-requests can be found at . + +When reviewing a pull request we enforce the followings guidelines. +These have been designed to ensure clear and polite communication between all parties: + +* In all forms of communication, contributors and reviewers must be polite. +Comments seen as being rude or dismissive will not be tolerated. +* If choosing to not approve a PR, please state clearly why. +When asking for changes, the commits should be specific and actionable. +General criticisms which cannot be addressed or understood by the contributor are unhelpful. +If the contribution needs improvement, reviewers should state what their requested changes are. +If more information is needed for the reviewers to make a decision the reviewer should ask clear questions. +If the PR is generally not seen as a worthwhile contribution, a good justification should be given so the contributor may fairly rebuttal. +* By default, the original contributor is assumed to own a change. +I.e., they are assumed to be the sole party to submit patches to the pull request. +If someone other than the original contributor wishes to submit patches on the original contributors behalf they should first ask permission. +Pull requests which appear abandoned may be adopted by a new contributor as long as there is good enough reason to assume the original contributor is no longer working on the pull request. +* Maintainers have the final say on whether a change is merged. +Your review will be taken into account by the maintainer. +It is expected, in all but the most extreme cases, that the reviewer's concerns must be addressed and for the reviewer to approve the the contribution prior to the maintainer merging the pull request. + +We also recommend consulting Google's ["How to write code review comments"](https://google.github.io/eng-practices/review/reviewer/comments.html) for advice on giving feedback to contributors. + +## Releases gem5 releases occur 3 times per year. The procedure for releasing gem5 is as follows: @@ -435,7 +400,7 @@ gem5-dev mailing list will be notified that the staging branch will be merged into the stable branch after two weeks, thus marking the new release. 3. The staging branch will have the full suite of gem5 tests run on it to ensure all tests pass and the to-be-released code is in a decent state. -4. If a user submits a changeset to the staging branch, it will be considered +4. If a user submits a pull request to the staging branch, it will be considered and undergo the standard github review process. However, only alterations that cannot wait until the following release will be accepted for submission into the branch (i.e., submissions to the staging branch for "last minute" @@ -444,8 +409,8 @@ fix). The project maintainers will use their discretion in deciding whether a change may be submitted directly to the staging branch. All other submissions to gem5 will continue to be made to the develop branch. Patches submitted into the staging branch do not need to be re-added to the develop branch. -5. Once signed off by members of the PMC the staging branch shall be merged -into the stable and develop branch. The staging branch will then be deleted. +5. Once the staging branch has been deemed ready for release, the [release procedures](https://www.gem5.org/documentation/general_docs/development/release_procedures/) will be carried out. +This will end with the staging branch being merged into the stable branch. 6. The stable branch shall be tagged with the correct version number for that release. gem5 conforms to a "v{YY}.{MAJOR}.{MINOR}.{HOTFIX}" versioning system. E.g., the first major release of 2022 will be "v22.0.0.0", followed by @@ -455,8 +420,16 @@ the minor release numbers in case this policy changes in the future. 7. The gem5-dev and gem5-user mailing lists shall be notified of the new gem5 release. -Hotfixes --------- +### Exemptions + +Due to limitations with GitHub we may update the ".github" directory in the gem5 repo's `stable` branch between gem5 releases. +This is due to certain processes carried out by the GitHub Actions infrastructure which rely on configurations being present on a repository's primary branch. +As the files in ".github" only influence the functionality of our GitHub actions and other GitHub activities, updating these files does not change the functionality of the gem5 in way. +It is therefore safe to do this. +Despite this exemption to our normal procedure we aim to ensure that **the ".github" directory on the `stable` is never "ahead" of that in the `develop` branch**. +Therefore contributors who wish to update files in ".github" should submit their changes to `develop` and then request their changes to be applied to the `stable` branch. + +### Hotfixes There may be circumstances in which a change to gem5 is deemed critical and cannot wait for an official release (e.g., a high-priority bug fix). In these From b7004492faa0d36ee77006aae05033a058aa5225 Mon Sep 17 00:00:00 2001 From: Hoa Nguyen Date: Wed, 26 Jul 2023 19:56:06 -0700 Subject: [PATCH 45/55] learning-gem5: Add a missing override Change-Id: I9acebe6f3096b499fa2c69b6d757373431f63c71 Signed-off-by: Hoa Nguyen --- src/learning_gem5/part2/hello_object.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/learning_gem5/part2/hello_object.hh b/src/learning_gem5/part2/hello_object.hh index c34dde304d..05c6dde4d6 100644 --- a/src/learning_gem5/part2/hello_object.hh +++ b/src/learning_gem5/part2/hello_object.hh @@ -69,7 +69,7 @@ class HelloObject : public SimObject * SimObjects have been constructed. It is called after the user calls * simulate() for the first time. */ - void startup(); + void startup() override; }; } // namespace gem5 From df60826fa35b04385e688fa562ed3dcbc0793a66 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 8 Aug 2023 13:01:30 -0700 Subject: [PATCH 46/55] misc: Merge develop .github into stable Change-Id: I07e5e5f3bc95b78459b77c0f1170923f6c9daf18 --- .github/workflows/ci-tests.yaml | 11 +-- .github/workflows/daily-tests.yaml | 30 +++--- .github/workflows/weekly-tests.yaml | 144 +++------------------------- 3 files changed, 32 insertions(+), 153 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 9894716665..2fb5910d44 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -79,6 +79,11 @@ jobs: needs: [pre-commit, build-gem5, check-for-change-id] timeout-minutes: 360 # 6 hours steps: + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -102,10 +107,4 @@ jobs: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 - - name: Clean runner - if: success() || failure() - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 679dba2978..595650e203 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -74,12 +74,17 @@ jobs: testlib-long-tests: strategy: matrix: - test-type: [arm-boot-tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement-policies, riscv-boot-tests, stdlib, x86-boot-tests] + test-type: [arm_boot_tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, stdlib, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours for entire matrix to run steps: + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We @@ -137,7 +142,7 @@ jobs: # run test - name: long ${{ matrix.test-type }} tests working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/${{ matrix.image }} --length=long --skip-build -vv -t $(nproc) + run: ./main.py run gem5/${{ matrix.test-type }} --length=long --skip-build -vv -t $(nproc) - name: create zip of results if: success() || failure() run: | @@ -147,17 +152,11 @@ jobs: if: success() || failure() uses: actions/upload-artifact@v3 env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} with: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 - - name: Clean runner - if: success() || failure() - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." # split library example tests into runs based on Suite UID @@ -171,6 +170,11 @@ jobs: needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours steps: + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We @@ -193,17 +197,11 @@ jobs: if: success() || failure() uses: actions/upload-artifact@v3 env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} with: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 - - name: Clean runner - if: success() || failure() - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." # This runs the SST-gem5 integration compilation and tests it with diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index dd81555f00..5ec65a9e5a 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -33,13 +33,21 @@ jobs: retention-days: 5 - run: echo "This job's status is ${{ job.status }}." -# start running the very-long tests - testlib-very-long-gem5_library_example_tests: + # start running the very-long tests + testlib-very-long-tests: + strategy: + matrix: + test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] timeout-minutes: 4320 # 3 days steps: + - name: Clean runner + run: + rm -rf ./* || true + rm -rf ./.??* || true + rm -rf ~/.cache || true - uses: actions/checkout@v3 with: # Scheduled workflows run on the default branch by default. We @@ -50,9 +58,9 @@ jobs: name: ${{needs.build-gem5.outputs.build-name}} path: build/ALL - run: chmod u+x build/ALL/gem5.opt - - name: very-long gem5_library_example_tests + - name: very-long ${{ matrix.test-type }} working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/gem5_library_example_tests --length very-long --skip-build -vv -t $(nproc) + run: ./main.py run gem5/${{ matrix.test-type }} --length very-long --skip-build -vv -t $(nproc) - name: create zip of results if: success() || failure() run: | @@ -62,135 +70,9 @@ jobs: if: success() || failure() uses: actions/upload-artifact@v3 env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} with: name: ${{ env.MY_STEP_VAR }} path: output.zip retention-days: 7 - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - run: echo "This job's status is ${{ job.status }}." - - - testlib-long-gem5-resources: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5] - timeout-minutes: 4320 # 3 days - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: very-long gem5-resources tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/gem5-resources --length very-long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - run: echo "This job's status is ${{ job.status }}." - - testlib-very-long-parsec-benchmarks: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 - timeout-minutes: 4320 # 3 days - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: very-long x86-boot-tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/x86-boot-tests --length very-long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - run: echo "This job's status is ${{ job.status }}." - - testlib-very-long-x86-boot-tests: - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: build-gem5 - timeout-minutes: 4320 # 3 days - steps: - - uses: actions/checkout@v3 - with: - # Scheduled workflows run on the default branch by default. We - # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: very-long x86-boot-tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/x86-boot-tests --length very-long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - run: echo "This job's status is ${{ job.status }}." From 90459612f591e3a396235375e6041c7b6f849f34 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 11 Aug 2023 16:19:21 -0700 Subject: [PATCH 47/55] misc: Sync GitHub Workflow files from develop to stable Change-Id: Ibbe8db6191cd248a5ec04c18ed72e04dbf8d791c --- .github/workflows/compiler-tests.yaml | 2 ++ .github/workflows/daily-tests.yaml | 3 +++ .github/workflows/weekly-tests.yaml | 1 + 3 files changed, 6 insertions(+) diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 3b5ed1c901..3ae771b017 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -19,6 +19,7 @@ jobs: image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] opts: [.opt, .fast] runs-on: [self-hosted, linux, x64, run] + continue-on-error: true timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: @@ -39,6 +40,7 @@ jobs: image: [gcc-version-12, clang-version-14] opts: [.opt] runs-on: [self-hosted, linux, x64, run] + continue-on-error: true timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 595650e203..4f9b0540e5 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -36,6 +36,7 @@ jobs: - image: NULL_MI_example command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) runs-on: [self-hosted, linux, x64, build] + continue-on-error: true needs: name-artifacts container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest steps: @@ -76,6 +77,7 @@ jobs: matrix: test-type: [arm_boot_tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, stdlib, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] + continue-on-error: true container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours for entire matrix to run @@ -167,6 +169,7 @@ jobs: matrix: test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + continue-on-error: true needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours steps: diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 5ec65a9e5a..ba7038006c 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -39,6 +39,7 @@ jobs: matrix: test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] + continue-on-error: true container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] timeout-minutes: 4320 # 3 days From 6966518dcca7c0da8cbff8cda23e60af2f7c3068 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 30 Aug 2023 17:16:12 -0700 Subject: [PATCH 48/55] misc: Copy .github directory from develop to stable Change-Id: I7755f90c1b5d81ff1cf66920f229be921d47e844 --- .github/workflows/ci-tests.yaml | 5 ++++ .github/workflows/compiler-tests.yaml | 12 ++++---- .github/workflows/daily-tests.yaml | 6 ++-- .github/workflows/docker-build.yaml | 42 +++++++++++++++++++++++++++ .github/workflows/weekly-tests.yaml | 33 ++++++++++++++++++++- 5 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docker-build.yaml diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 2fb5910d44..8def494668 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -11,6 +11,7 @@ jobs: pre-commit: # runs on github hosted runner runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 @@ -20,6 +21,7 @@ jobs: check-for-change-id: # runs on github hosted runner runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 with: @@ -43,6 +45,7 @@ jobs: build-gem5: runs-on: [self-hosted, linux, x64, build] + if: github.event.pull_request.draft == false container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes outputs: @@ -63,6 +66,7 @@ jobs: unittests-all-opt: runs-on: [self-hosted, linux, x64, run] + if: github.event.pull_request.draft == false container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes timeout-minutes: 60 @@ -75,6 +79,7 @@ jobs: testlib-quick: runs-on: [self-hosted, linux, x64, run] + if: github.event.pull_request.draft == false container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [pre-commit, build-gem5, check-for-change-id] timeout-minutes: 360 # 6 hours diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 3ae771b017..9a567653f5 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -15,11 +15,11 @@ jobs: # replication of compiler-tests.sh all-compilers: strategy: + fail-fast: false matrix: image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] opts: [.opt, .fast] - runs-on: [self-hosted, linux, x64, run] - continue-on-error: true + runs-on: [self-hosted, linux, x64, build] timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: @@ -29,18 +29,18 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} -j$(nproc) timeout-minutes: 600 # 10 hours # Tests the two latest gcc and clang supported compilers against all gem5 compilations. latest-compilers-all-gem5-builds: strategy: + fail-fast: false matrix: gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base, VEGA_X86, GCN3_X86] image: [gcc-version-12, clang-version-14] opts: [.opt] - runs-on: [self-hosted, linux, x64, run] - continue-on-error: true + runs-on: [self-hosted, linux, x64, build] timeout-minutes: 2880 # 48 hours container: gcr.io/gem5-test/${{ matrix.image }}:latest steps: @@ -50,5 +50,5 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} -j$(nproc) timeout-minutes: 600 # 10 hours diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 4f9b0540e5..cf57d804bc 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -19,6 +19,7 @@ jobs: build-gem5: strategy: + fail-fast: false matrix: # NULL is in quotes since it is considered a keyword in yaml files image: [ALL, ALL_CHI, ARM, ALL_MSI, ALL_MESI_Two_Level, "NULL", NULL_MI_example, RISCV, VEGA_X86] @@ -36,7 +37,6 @@ jobs: - image: NULL_MI_example command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) runs-on: [self-hosted, linux, x64, build] - continue-on-error: true needs: name-artifacts container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest steps: @@ -74,10 +74,10 @@ jobs: # start running all of the long tests testlib-long-tests: strategy: + fail-fast: false matrix: test-type: [arm_boot_tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, stdlib, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] - continue-on-error: true container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours for entire matrix to run @@ -166,10 +166,10 @@ jobs: testlib-long-gem5_library_example_tests: runs-on: [self-hosted, linux, x64, run] strategy: + fail-fast: false matrix: test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - continue-on-error: true needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours steps: diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml new file mode 100644 index 0000000000..cbacaad9aa --- /dev/null +++ b/.github/workflows/docker-build.yaml @@ -0,0 +1,42 @@ +# +name: Docker images build and push + +on: + push: + branches: + - develop + paths: + - 'util/docker/ubuntu-20.04_all-depenencies' + workflow_dispatch: + +env: + IMAGE_NAME: ubuntu-20.04_all-depenencies +# +jobs: + # This builds and pushes the docker image. + push: + runs-on: [self-hosted, linux, x64, run] + permissions: + packages: write + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Build image + run: | + cd util/docker/ubuntu-20.04_all-depenencies + docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + + - name: Log in to registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # This changes all uppercase characters to lowercase. + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag $IMAGE_NAME $IMAGE_ID:latest + docker push $IMAGE_ID::latest diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index ba7038006c..782647cb81 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -36,10 +36,10 @@ jobs: # start running the very-long tests testlib-very-long-tests: strategy: + fail-fast: false matrix: test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] runs-on: [self-hosted, linux, x64, run] - continue-on-error: true container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] timeout-minutes: 4320 # 3 days @@ -77,3 +77,34 @@ jobs: path: output.zip retention-days: 7 - run: echo "This job's status is ${{ job.status }}." + + dramsys-tests: + runs-on: [self-hosted, linux, x64, build] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 4320 # 3 days + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + + - name: Checkout DRAMSys + working-directory: ${{ github.workspace }}/ext/dramsys + run: | + git clone https://github.com/tukl-msd/DRAMSys DRAMSys + cd DRAMSys + git checkout -b gem5 09f6dcbb91351e6ee7cadfc7bc8b29d97625db8f + git submodule update --init --recursive + + # gem5 is built separately because it depends on the DRAMSys library + - name: Build gem5 + working-directory: ${{ github.workspace }} + run: scons build/ALL/gem5.opt -j $(nproc) + + - name: Run DRAMSys Checks + working-directory: ${{ github.workspace }} + run: | + ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/arm-hello-dramsys.py + ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/dramsys-traffic.py + ./build/ALL/gem5.opt configs/example/dramsys.py From 3157cde32449fea7b0a0ad5e8241481bc6ee76c3 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Sat, 14 Oct 2023 17:16:09 -0700 Subject: [PATCH 49/55] misc: Copy .github directory from develop to stable (#458) This is done periodically as the Github Action's infrastructure reads files from the repos main branch (`stable`) where as changes are made to `develop`'s ".github" which should be made live ASAP. This does not affect the gem5 build or any configuration scripts, only how testing is performed via GitHub and other info used by Github (e.g., ".github/ISSUE_TEMPLATE" which outlines the templates for creating issues via our GitHub issues page. --- .github/ISSUE_TEMPLATE/bug_report.md | 24 +- .github/workflows/ci-tests.yaml | 264 ++++++++------ .github/workflows/compiler-tests.yaml | 92 +++-- .github/workflows/daily-tests.yaml | 491 +++++++++++++------------- .github/workflows/docker-build.yaml | 76 ++-- .github/workflows/gpu-tests.yaml | 61 ++++ .github/workflows/utils.yaml | 26 +- .github/workflows/weekly-tests.yaml | 186 +++++----- 8 files changed, 699 insertions(+), 521 deletions(-) create mode 100644 .github/workflows/gpu-tests.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 179a37ac0b..6f89a7eb33 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,7 +4,6 @@ about: Create a report to help us find and fix the bug title: '' labels: bug assignees: '' - --- **Describe the bug** @@ -18,21 +17,18 @@ If you have modified gem5 in some way please state, to the best of your ability, **To Reproduce** Steps to reproduce the behavior. Please assume starting from a clean repository: + 1. Compile gem5 with command ... 2. Execute the simulation with... -If writing code, or a terminal command, use code blocks. Either an inline code block, \`scons build/ALL/gem5.opt\` (enclosed in two \`) or a multi-line codeblock: +If writing code, or a terminal command, use code blocks. Either an inline code block, `scons build/ALL/gem5.opt` (enclosed in two '`') or a multi-line codeblock: -\`\`\` - -int x=2; - -int y=3' - -print(x+y); - -\`\`\` +```python +int x=2 +int y=3 +print(x+y) +``` If possible, please include the Python configuration script used and state clearly any parameters passed. @@ -40,11 +36,9 @@ If possible, please include the Python configuration script used and state clear If applicable, add the terminal output here. If long, only include the relevant lines. Please put the terminal output in code blocks. I.e.: -\`\`\` - +```shell #Terminal output here# - -\`\`\` +``` **Expected behavior** A clear and concise description of what you expected to happen. diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 8def494668..08ec318660 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -1,115 +1,179 @@ +--- # This workflow runs after a pull-request has been approved by a reviewer. name: CI Tests on: - pull_request: - types: [opened, edited, synchronize, ready_for_review] + pull_request: + types: [opened, edited, synchronize, ready_for_review] +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true jobs: - pre-commit: + pre-commit: # runs on github hosted runner - runs-on: ubuntu-22.04 - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 # ensures we have a change-id in every commit, needed for gerrit - check-for-change-id: - # runs on github hosted runner - runs-on: ubuntu-22.04 - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check for Change-Id - run: | - # loop through all the commits in the pull request - for commit in $(git rev-list ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}); do - git checkout $commit - if (git log -1 --pretty=format:"%B" | grep -q "Change-Id: ") - then - # passes as long as at least one change-id exists in the pull request - exit 0 - fi - done - # if we reach this part, none of the commits had a change-id - echo "None of the commits in this pull request contains a Change-ID, which we require for any changes made to gem5. "\ - "To automatically insert one, run the following:\n f=`git rev-parse --git-dir`/hooks/commit-msg ; mkdir -p $(dirname $f) ; "\ - "curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request." - exit 1 + check-for-change-id: + # runs on github hosted runner + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Check for Change-Id + run: | + # loop through all the commits in the pull request + for commit in $(git rev-list ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}); do + git checkout $commit + if (git log -1 --pretty=format:"%B" | grep -q "Change-Id: ") + then + # passes as long as at least one change-id exists in the pull request + exit 0 + fi + done + # if we reach this part, none of the commits had a change-id + echo "None of the commits in this pull request contains a Change-ID, which we require for any changes made to gem5. "\ + "To automatically insert one, run the following:\n f=`git rev-parse --git-dir`/hooks/commit-msg ; mkdir -p $(dirname $f) ; "\ + "curl -Lo $f https://gerrit-review.googlesource.com/tools/hooks/commit-msg ; chmod +x $f\n Then amend the commit with git commit --amend --no-edit, and update your pull request." + exit 1 - build-gem5: - runs-on: [self-hosted, linux, x64, build] - if: github.event.pull_request.draft == false - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes - outputs: - artifactname: ${{ steps.name.outputs.test }} - steps: - - uses: actions/checkout@v3 - - id: name - run: echo "test=$(date +"%Y-%m-%d_%H.%M.%S")-artifact" >> $GITHUB_OUTPUT + unittests-all-opt: + runs-on: [self-hosted, linux, x64] + if: github.event.pull_request.draft == false + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + - name: CI Unittests + working-directory: ${{ github.workspace }} + run: scons build/ALL/unittests.opt -j $(nproc) + - run: echo "This job's status is ${{ job.status }}." - - name: Build gem5 - run: | - scons build/ALL/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.name.outputs.test }} - path: build/ALL/gem5.opt - - run: echo "This job's status is ${{ job.status }}." + testlib-quick-matrix: + runs-on: [self-hosted, linux, x64] + if: github.event.pull_request.draft == false + # In order to make sure the environment is exactly the same, we run in + # the same container we use to build gem5 and run the testlib tests. This + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id] + steps: + - uses: actions/checkout@v3 - unittests-all-opt: - runs-on: [self-hosted, linux, x64, run] - if: github.event.pull_request.draft == false - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, check-for-change-id] # only runs if pre-commit and change-id passes - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - name: CI Unittests - working-directory: ${{ github.workspace }} - run: scons build/ALL/unittests.opt -j $(nproc) - - run: echo "This job's status is ${{ job.status }}." + # Unfortunately the 'ubunutu-latest' image doesn't have jq installed. + # We therefore need to install it as a step here. + - name: Install jq + run: apt install -y jq - testlib-quick: - runs-on: [self-hosted, linux, x64, run] - if: github.event.pull_request.draft == false - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [pre-commit, build-gem5, check-for-change-id] - timeout-minutes: 360 # 6 hours - steps: - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.artifactname}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: The TestLib CI Tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run --skip-build -vv - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{github.job}}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." + - name: Get directories for testlib-quick + working-directory: ${{ github.workspace }}/tests + id: dir-matrix + run: echo "test-dirs-matrix=$(find gem5/* -type d -maxdepth 0 | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT + + - name: Get the build targets for testlib-quick-gem5-builds + working-directory: ${{ github.workspace }}/tests + id: build-matrix + run: echo "build-matrix=$(./main.py list --build-targets -q | jq -ncR '[inputs]')" >>$GITHUB_OUTPUT + + outputs: + build-matrix: ${{ steps.build-matrix.outputs.build-matrix }} + test-dirs-matrix: ${{ steps.dir-matrix.outputs.test-dirs-matrix }} + + testlib-quick-gem5-builds: + runs-on: [self-hosted, linux, x64] + if: github.event.pull_request.draft == false + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id, testlib-quick-matrix] + strategy: + matrix: + build-target: ${{ fromJson(needs.testlib-quick-matrix.outputs.build-matrix) }} + steps: + - uses: actions/checkout@v3 + - name: Build gem5 + run: scons ${{ matrix.build-target }} -j $(nproc) + + # Upload the gem5 binary as an artifact. + # Note: the "achor.txt" file is a hack to make sure the paths are + # preserverd in the artifact. The upload-artifact action finds the + # closest common directory and uploads everything relative to that. + # E.g., if we upload "build/ARM/gem5.opt" and "build/RISCV/gem5.opt" + # Then upload-artifact will upload "ARM/gem5.opt" and "RISCV/gem5.opt", + # stripping the "build" directory. By adding the "anchor.txt" file, we + # ensure the "build" directory is preserved. + - run: echo "anchor" > anchor.txt + - uses: actions/upload-artifact@v3 + with: + name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds + path: | + build/*/gem5.* + anchor.txt + retention-days: 7 + + testlib-quick-execution: + runs-on: [self-hosted, linux, x64] + if: github.event.pull_request.draft == false + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + needs: [pre-commit, check-for-change-id, testlib-quick-matrix, testlib-quick-gem5-builds] + timeout-minutes: 360 # 6 hours + strategy: + fail-fast: false + matrix: + test-dir: ${{ fromJson(needs.testlib-quick-matrix.outputs.test-dirs-matrix) }} + steps: + - name: Clean runner + run: rm -rf ./* || true rm -rf ./.??* || true rm -rf ~/.cache || true + + # Checkout the repository then download the gem5.opt artifact. + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: ci-tests-${{ github.run_number }}-testlib-quick-all-gem5-builds + + # Check that the gem5.opt artifact exists and is executable. + - name: Chmod gem5.{opt,debug,fast} to be executable + run: | + find . -name "gem5.opt" -exec chmod u+x {} \; + find . -name "gem5.debug" -exec chmod u+x {} \; + find . -name "gem5.fast" -exec chmod u+x {} \; + + # Run the testlib quick tests in the given directory. + - name: Run "tests/${{ matrix.test-dir }}" TestLib quick tests + id: run-tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run --skip-build -vv -j$(nproc) ${{ matrix.test-dir }} + + # Get the basename of the matrix.test-dir path (to name the artifact). + - name: Sanatize test-dir for artifact name + id: sanitize-test-dir + if: success() || failure() + run: echo "sanatized-test-dir=$(echo '${{ matrix.test-dir }}' | sed 's/\//-/g')" >> $GITHUB_OUTPUT + + # Upload the tests/testing-results directory as an artifact. + - name: Upload test results + if: success() || failure() + uses: actions/upload-artifact@v3 + with: + name: ci-tests-run-${{ github.run_number }}-attempt-${{ github.run_attempt }}-testlib-quick-${{ steps.sanitize-test-dir.outputs.sanatized-test-dir + }}-status-${{ steps.run-tests.outcome }}-output + path: tests/testing-results + retention-days: 30 + + testlib-quick: + # It is 'testlib-quick' which needs to pass for the pull request to be + # merged. The 'testlib-quick-execution' is a matrix job which runs all the + # the testlib quick tests. This job is therefore a stub which will pass if + # all the testlib-quick-execution jobs pass. + runs-on: ubuntu-22.04 + needs: testlib-quick-execution + steps: + - run: echo "This job's status is ${{ job.status }}." diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 9a567653f5..1a340e0bf5 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -1,54 +1,70 @@ +--- # This workflow runs all of the compiler tests name: Compiler Tests -run-name: ${{ github.actor }} is running compiler tests - on: # Runs every Friday from 7AM UTC - schedule: - - cron: '00 7 * * 5' + schedule: + - cron: 00 7 * * 5 # Allows us to manually start workflow for testing - workflow_dispatch: + workflow_dispatch: jobs: # replication of compiler-tests.sh - all-compilers: - strategy: - fail-fast: false - matrix: - image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-14, clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] - opts: [.opt, .fast] - runs-on: [self-hosted, linux, x64, build] - timeout-minutes: 2880 # 48 hours - container: gcr.io/gem5-test/${{ matrix.image }}:latest - steps: - - uses: actions/checkout@v3 - with: + all-compilers: + strategy: + fail-fast: false + matrix: + image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-16, clang-version-15, clang-version-14, + clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, + ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] + opts: [.opt, .fast] + runs-on: [self-hosted, linux, x64] + timeout-minutes: 2880 # 48 hours + container: gcr.io/gem5-test/${{ matrix.image }}:latest + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} -j$(nproc) - timeout-minutes: 600 # 10 hours + ref: develop + - name: Compile build/ALL/gem5${{ matrix.opts }} with ${{ matrix.image }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/ALL/gem5${{ matrix.opts }} -j$(nproc) + timeout-minutes: 600 # 10 hours # Tests the two latest gcc and clang supported compilers against all gem5 compilations. - latest-compilers-all-gem5-builds: - strategy: - fail-fast: false - matrix: - gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base, VEGA_X86, GCN3_X86] - image: [gcc-version-12, clang-version-14] - opts: [.opt] - runs-on: [self-hosted, linux, x64, build] - timeout-minutes: 2880 # 48 hours - container: gcr.io/gem5-test/${{ matrix.image }}:latest - steps: - - uses: actions/checkout@v3 - with: + latest-compilers-all-gem5-builds: + strategy: + fail-fast: false + matrix: + gem5-compilation: [ARM, ARM_MESI_Three_Level, ARM_MESI_Three_Level_HTM, ARM_MOESI_hammer, Garnet_standalone, GCN3_X86, MIPS, 'NULL', NULL_MESI_Two_Level, + NULL_MOESI_CMP_directory, NULL_MOESI_CMP_token, NULL_MOESI_hammer, POWER, RISCV, SPARC, X86, X86_MI_example, X86_MOESI_AMD_Base, VEGA_X86, + GCN3_X86] + image: [gcc-version-12, clang-version-16] + opts: [.opt] + runs-on: [self-hosted, linux, x64] + timeout-minutes: 2880 # 48 hours + container: gcr.io/gem5-test/${{ matrix.image }}:latest + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} - run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} -j$(nproc) - timeout-minutes: 600 # 10 hours + ref: develop + - name: Compile build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} with ${{ matrix.image }} + run: /usr/bin/env python3 /usr/bin/scons --ignore-style build/${{ matrix.gem5-compilation }}/gem5${{ matrix.opts }} -j$(nproc) + timeout-minutes: 600 # 10 hours + + compiler-tests: + # The dummy job is used to indicate whether the compiler tests have + # passed or not. This can be used as status check for pull requests. + # I.e., if we want to stop pull requests from being merged if the + # compiler tests are failing, we can add this job as a required status + # check. + runs-on: ubuntu-22.04 + needs: + - latest-compilers-all-gem5-builds + - all-compilers + steps: + - run: echo "This compiler tests have passed." diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index cf57d804bc..36006d03e4 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -1,286 +1,305 @@ +--- # This workflow runs all of the long tests within main.py, extra tests in nightly.sh, and unittests name: Daily Tests on: # Runs every day from 7AM UTC - schedule: - - cron: '0 7 * * *' + schedule: + - cron: 0 7 * * * jobs: - name-artifacts: - runs-on: ubuntu-latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v2 - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S-")" >> $GITHUB_OUTPUT + name-artifacts: + runs-on: ubuntu-latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v2 + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S-")" >> $GITHUB_OUTPUT - build-gem5: - strategy: - fail-fast: false - matrix: + build-gem5: + strategy: + fail-fast: false + matrix: # NULL is in quotes since it is considered a keyword in yaml files - image: [ALL, ALL_CHI, ARM, ALL_MSI, ALL_MESI_Two_Level, "NULL", NULL_MI_example, RISCV, VEGA_X86] + image: [ALL, ALL_CHI, ARM, ALL_MSI, ALL_MESI_Two_Level, 'NULL', NULL_MI_example, RISCV, VEGA_X86] # this allows us to pass additional command line parameters # the default is to add -j $(nproc), but some images # require more specifications when built - include: - - command-line: -j $(nproc) - - image: ALL_CHI - command-line: --default=ALL PROTOCOL=CHI -j $(nproc) - - image: ALL_MSI - command-line: --default=ALL PROTOCOL=MSI -j $(nproc) - - image: ALL_MESI_Two_Level - command-line: --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) - - image: NULL_MI_example - command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) - runs-on: [self-hosted, linux, x64, build] - needs: name-artifacts - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - steps: - - uses: actions/checkout@v3 - with: + include: + - command-line: -j $(nproc) + - image: ALL_CHI + command-line: --default=ALL PROTOCOL=CHI -j $(nproc) + - image: ALL_MSI + command-line: --default=ALL PROTOCOL=MSI -j $(nproc) + - image: ALL_MESI_Two_Level + command-line: --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) + - image: NULL_MI_example + command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) + runs-on: [self-hosted, linux, x64] + needs: name-artifacts + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Build gem5 - run: scons build/${{ matrix.image }}/gem5.opt ${{ matrix.command-line }} - - uses: actions/upload-artifact@v3 - with: - name: ${{ needs.name-artifacts.outputs.build-name }}${{ matrix.image }} - path: build/${{ matrix.image }}/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." + ref: develop + - name: Build gem5 + run: scons build/${{ matrix.image }}/gem5.opt ${{ matrix.command-line }} + - uses: actions/upload-artifact@v3 + with: + name: ${{ needs.name-artifacts.outputs.build-name }}${{ matrix.image }} + path: build/${{ matrix.image }}/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." # this builds both unittests.fast and unittests.debug - unittests-fast-debug: - strategy: - matrix: - type: [fast, debug] - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: + unittests-fast-debug: + strategy: + matrix: + type: [fast, debug] + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: ALL/unittests.${{ matrix.type }} UnitTests - run: scons build/ALL/unittests.${{ matrix.type }} -j $(nproc) + ref: develop + - name: ALL/unittests.${{ matrix.type }} UnitTests + run: scons build/ALL/unittests.${{ matrix.type }} -j $(nproc) # start running all of the long tests - testlib-long-tests: - strategy: - fail-fast: false - matrix: - test-type: [arm_boot_tests, fs, gem5_library_example_tests, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, stdlib, x86_boot_tests] - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [name-artifacts, build-gem5] - timeout-minutes: 1440 # 24 hours for entire matrix to run - steps: - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - uses: actions/checkout@v3 - with: + testlib-long-tests: + strategy: + fail-fast: false + matrix: + test-type: [arm_boot_tests, fs, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, + stdlib, x86_boot_tests] + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [name-artifacts, build-gem5] + timeout-minutes: 1440 # 24 hours for entire matrix to run + steps: + - name: Clean runner + run: rm -rf ./* || true rm -rf ./.??* || true rm -rf ~/.cache || true + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop + ref: develop # download all artifacts for each test # since long tests can't start until the build matrix completes, # we download all artifacts from the build for each test # in this matrix - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ALL - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ALL_CHI - path: build/ALL_CHI - - run: chmod u+x build/ALL_CHI/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ARM - path: build/ARM - - run: chmod u+x build/ARM/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ALL_MSI - path: build/ALL_MSI - - run: chmod u+x build/ALL_MSI/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ALL_MESI_Two_Level - path: build/ALL_MESI_Two_Level - - run: chmod u+x build/ALL_MESI_Two_Level/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}NULL - path: build/NULL - - run: chmod u+x build/NULL/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}NULL_MI_example - path: build/NULL_MI_example - - run: chmod u+x build/NULL_MI_example/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}RISCV - path: build/RISCV - - run: chmod u+x build/RISCV/gem5.opt - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}VEGA_X86 - path: build/VEGA_X86 - - run: chmod u+x build/VEGA_X86/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ALL + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ALL_CHI + path: build/ALL_CHI + - run: chmod u+x build/ALL_CHI/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ARM + path: build/ARM + - run: chmod u+x build/ARM/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ALL_MSI + path: build/ALL_MSI + - run: chmod u+x build/ALL_MSI/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ALL_MESI_Two_Level + path: build/ALL_MESI_Two_Level + - run: chmod u+x build/ALL_MESI_Two_Level/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}NULL + path: build/NULL + - run: chmod u+x build/NULL/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}NULL_MI_example + path: build/NULL_MI_example + - run: chmod u+x build/NULL_MI_example/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}RISCV + path: build/RISCV + - run: chmod u+x build/RISCV/gem5.opt + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}VEGA_X86 + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt # run test - - name: long ${{ matrix.test-type }} tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/${{ matrix.test-type }} --length=long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." + - name: long ${{ matrix.test-type }} tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/${{ matrix.test-type }} --length=long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." # split library example tests into runs based on Suite UID # so that they don't hog the runners for too long - testlib-long-gem5_library_example_tests: - runs-on: [self-hosted, linux, x64, run] - strategy: - fail-fast: false - matrix: - test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [name-artifacts, build-gem5] - timeout-minutes: 1440 # 24 hours - steps: - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - uses: actions/checkout@v3 - with: + testlib-long-gem5_library_example_tests: + runs-on: [self-hosted, linux, x64] + strategy: + fail-fast: false + matrix: + test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, + gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [name-artifacts, build-gem5] + timeout-minutes: 1440 # 24 hours + steps: + - name: Clean runner + run: rm -rf ./* || true rm -rf ./.??* || true rm -rf ~/.cache || true + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.name-artifacts.outputs.build-name}}ALL - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: long ${{ matrix.test-type }} gem5_library_example_tests - working-directory: ${{ github.workspace }}/tests - run: ./main.py run --uid SuiteUID:tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py:test-${{ matrix.test-type }} --length=long --skip-build -vv - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.name-artifacts.outputs.build-name}}ALL + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: long ${{ matrix.test-type }} gem5_library_example_tests + working-directory: ${{ github.workspace }}/tests + run: ./main.py run --uid SuiteUID:tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py:test-${{ matrix.test-type }} --length=long + --skip-build -vv + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." # This runs the SST-gem5 integration compilation and tests it with # ext/sst/sst/example.py. - sst-test: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/sst-env:latest - timeout-minutes: 180 + sst-test: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/sst-env:latest + timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - with: + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Build RISCV/libgem5_opt.so with SST - run: scons build/RISCV/libgem5_opt.so --without-tcmalloc --duplicate-sources --ignore-style -j $(nproc) - - name: Compile ext/sst - working-directory: ${{ github.workspace }}/ext/sst - run: make -j $(nproc) - - name: Run SST test - working-directory: ${{ github.workspace }}/ext/sst - run: sst --add-lib-path=./ sst/example.py + ref: develop + - name: Build RISCV/libgem5_opt.so with SST + run: scons build/RISCV/libgem5_opt.so --without-tcmalloc --duplicate-sources --ignore-style -j $(nproc) + - name: Makefile ext/sst + working-directory: ${{ github.workspace }}/ext/sst + run: mv Makefile.linux Makefile + - name: Compile ext/sst + working-directory: ${{ github.workspace }}/ext/sst + run: make -j $(nproc) + - name: Run SST test + working-directory: ${{ github.workspace }}/ext/sst + run: sst --add-lib-path=./ sst/example.py # This runs the gem5 within SystemC ingration and runs a simple hello-world # simulation with it. - systemc-test: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/systemc-env:latest - timeout-minutes: 180 + systemc-test: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/systemc-env:latest + timeout-minutes: 180 - steps: - - uses: actions/checkout@v3 - with: + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Build ARM/gem5.opt - run: scons build/ARM/gem5.opt --ignore-style --duplicate-sources -j$(nproc) - - name: Build ARM/libgem5_opt.so - run: scons build/ARM/libgem5_opt.so --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j$(nproc) --duplicate-sources - - name: Compile gem5 withing SystemC - working-directory: ${{ github.workspace }}/util/systemc/gem5_within_systemc - run: make - - name: Run gem5 within SystemC test - run: ./build/ARM/gem5.opt configs/deprecated/example/se.py -c tests/test-progs/hello/bin/arm/linux/hello - - name: Continue gem5 within SystemC test - run: LD_LIBRARY_PATH=build/ARM/:/opt/systemc/lib-linux64/ ./util/systemc/gem5_within_systemc/gem5.opt.sc m5out/config.ini + ref: develop + - name: Build ARM/gem5.opt + run: scons build/ARM/gem5.opt --ignore-style --duplicate-sources -j$(nproc) + - name: Build ARM/libgem5_opt.so + run: scons build/ARM/libgem5_opt.so --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j$(nproc) --duplicate-sources + - name: Compile gem5 withing SystemC + working-directory: ${{ github.workspace }}/util/systemc/gem5_within_systemc + run: make + - name: Run gem5 within SystemC test + run: ./build/ARM/gem5.opt configs/deprecated/example/se.py -c tests/test-progs/hello/bin/arm/linux/hello + - name: Continue gem5 within SystemC test + run: LD_LIBRARY_PATH=build/ARM/:/opt/systemc/lib-linux64/ ./util/systemc/gem5_within_systemc/gem5.opt.sc m5out/config.ini # Runs the gem5 Nighyly GPU tests. - gpu-tests: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/gcn-gpu:latest - timeout-minutes: 720 # 12 hours + gpu-tests: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/gcn-gpu:latest + timeout-minutes: 720 # 12 hours - steps: - - uses: actions/checkout@v3 - with: + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - name: Compile build/GCN3_X86/gem5.opt - run: scons build/GCN3_X86/gem5.opt -j $(nproc) - - name: Get Square test-prog from gem5-resources - uses: wei/wget@v1 - with: - args: -q http://dist.gem5.org/dist/develop/test-progs/square/square # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time: https://github.com/coder/sshcode/issues/102 - - name: Run Square test with GCN3_X86/gem5.opt (SE mode) - run: | - mkdir -p tests/testing-results - ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c square - - name: Get allSyncPrims-1kernel from gem5-resources - uses: wei/wget@v1 - with: - args: -q http://dist.gem5.org/dist/develop/test-progs/heterosync/gcn3/allSyncPrims-1kernel # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time - - name: Run allSyncPrims-1kernel sleepMutex test with GCN3_X86/gem5.opt (SE mode) - run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="sleepMutex 10 16 4" - - name: Run allSyncPrims-1kernel lfTreeBarrUsing test with GCN3_X86/gem5.opt (SE mode) - run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="lfTreeBarrUniq 10 16 4" + ref: develop + - name: Compile build/GCN3_X86/gem5.opt + run: scons build/GCN3_X86/gem5.opt -j $(nproc) + - name: Get Square test-prog from gem5-resources + uses: wei/wget@v1 + with: + args: -q http://dist.gem5.org/dist/develop/test-progs/square/square # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time: https://github.com/coder/sshcode/issues/102 + - name: Run Square test with GCN3_X86/gem5.opt (SE mode) + run: | + mkdir -p tests/testing-results + ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c square + - name: Get allSyncPrims-1kernel from gem5-resources + uses: wei/wget@v1 + with: + args: -q http://dist.gem5.org/dist/develop/test-progs/heterosync/gcn3/allSyncPrims-1kernel # Removed -N bc it wasn't available within actions, should be okay bc workspace is clean every time + - name: Run allSyncPrims-1kernel sleepMutex test with GCN3_X86/gem5.opt (SE mode) + run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="sleepMutex 10 16 + 4" + - name: Run allSyncPrims-1kernel lfTreeBarrUsing test with GCN3_X86/gem5.opt (SE mode) + run: ./build/GCN3_X86/gem5.opt configs/example/apu_se.py --reg-alloc-policy=dynamic -n3 -c allSyncPrims-1kernel --options="lfTreeBarrUniq + 10 16 4" + daily-tests: + # The dummy job is used to indicate whether the daily tests have + # passed or not. This can be used as status check for pull requests. + # I.e., if we want to stop pull requests from being merged if the + # daily tests are failing we can add this job as a required status + # check. + runs-on: ubuntu-22.04 + needs: + - unittests-fast-debug + - testlib-long-tests + - testlib-long-gem5_library_example_tests + - sst-test + - systemc-test + - gpu-tests + steps: + - run: echo "This daily tests have passed." diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index cbacaad9aa..68b0905e83 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -1,42 +1,54 @@ -# +--- name: Docker images build and push on: - push: - branches: - - develop - paths: - - 'util/docker/ubuntu-20.04_all-depenencies' - workflow_dispatch: - -env: - IMAGE_NAME: ubuntu-20.04_all-depenencies -# + workflow_dispatch: jobs: + obtain-dockerfiles: + runs-on: [self-hosted, linux, x64] + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest + + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/upload-artifact@v3 + with: + name: dockerfiles + path: util/dockerfiles + # This builds and pushes the docker image. - push: - runs-on: [self-hosted, linux, x64, run] - permissions: - packages: write - contents: read + build-and-push: + runs-on: [self-hosted, linux, x64] + needs: obtain-dockerfiles + permissions: + packages: write + contents: read - steps: - - uses: actions/checkout@v3 + steps: + - uses: actions/download-artifact@v3 + with: + name: dockerfiles + path: dockerfiles-docker-build - - name: Build image - run: | - cd util/docker/ubuntu-20.04_all-depenencies - docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + - uses: docker/setup-qemu-action@v2 + name: Setup QEMU - - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + - uses: docker/setup-buildx-action@v2 + name: Set up Docker Buildx - - name: Push image - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + - uses: docker/login-action@v2 + name: Login to the GitHub Container Registry + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} - # This changes all uppercase characters to lowercase. - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - - docker tag $IMAGE_NAME $IMAGE_ID:latest - docker push $IMAGE_ID::latest + - name: Build and push with bake + uses: docker/bake-action@v4 + with: + workdir: ./dockerfiles-docker-build + files: docker-bake.hcl + push: true diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml new file mode 100644 index 0000000000..43eff95529 --- /dev/null +++ b/.github/workflows/gpu-tests.yaml @@ -0,0 +1,61 @@ +--- +# This workflow runs all of the very-long tests within main.py + +name: Weekly Tests + +on: + # Runs every Sunday from 7AM UTC + schedule: + - cron: 00 7 * * 6 + # Allows us to manually start workflow for testing + workflow_dispatch: + +jobs: + build-gem5: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/gcn-gpu:latest + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - name: Build gem5 + run: scons build/GCN3_X86/gem5.opt -j $(nproc) --ignore-style + - uses: actions/upload-artifact@v3 + with: + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 + path: build/GCN3_X86/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." + + HACC-tests: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/gcn-gpu:latest + needs: build-gem5 + timeout-minutes: 120 # 2 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + - uses: actions/download-artifact@v3 + with: + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 + path: build/GCN3_X86 + - run: chmod u+x build/GCN3_X86/gem5.opt + - name: make hip directory + run: mkdir hip + - name: Compile m5ops and x86 + working-directory: ${{ github.workspace }}/util/m5 + run: | + export TERM=xterm-256color + scons build/x86/out/m5 + - name: Download tests + working-directory: ${{ github.workspace }}/hip + run: wget http://dist.gem5.org/dist/v22-1/test-progs/halo-finder/ForceTreeTest + - name: Run HACC tests + working-directory: ${{ github.workspace }} + run: | + build/GCN3_X86/gem5.opt configs/example/apu_se.py -n3 --reg-alloc-policy=dynamic --benchmark-root=hip -c ForceTreeTest --options="0.5 0.1 64 0.1 1 N 12 rcb" diff --git a/.github/workflows/utils.yaml b/.github/workflows/utils.yaml index 91d0bf1722..ccbd87c82f 100644 --- a/.github/workflows/utils.yaml +++ b/.github/workflows/utils.yaml @@ -1,19 +1,21 @@ +--- # This workflow file contains miscellaneous tasks to manage the repository. name: Utils for Repository on: - schedule: - - cron: '30 1 * * *' - workflow_dispatch: + schedule: + - cron: 30 1 * * * + workflow_dispatch: jobs: # This job runs the stale action to close issues that have been inactive for 30 days. # It is scheduled to run every day at 1:30 AM UTC. - close-stale-issues: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v8.0.0 - with: - close-issue-message: 'This issue is being closed because it has been inactive waiting for response for 30 days. If this is still an issue, please open a new issue and reference this one.' - days-before-stale: 21 - days-before-close: 7 - any-of-labels: 'needs details' + close-stale-issues: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8.0.0 + with: + close-issue-message: This issue is being closed because it has been inactive waiting for response for 30 days. If this is still an issue, + please open a new issue and reference this one. + days-before-stale: 21 + days-before-close: 7 + any-of-labels: needs details diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 782647cb81..b21613f7a1 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -1,110 +1,120 @@ +--- # This workflow runs all of the very-long tests within main.py name: Weekly Tests on: # Runs every Sunday from 7AM UTC - schedule: - - cron: '00 7 * * 6' + schedule: + - cron: 00 7 * * 6 # Allows us to manually start workflow for testing - workflow_dispatch: + workflow_dispatch: jobs: - build-gem5: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - outputs: - build-name: ${{ steps.artifact-name.outputs.name }} - steps: - - uses: actions/checkout@v3 - with: + build-gem5: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + outputs: + build-name: ${{ steps.artifact-name.outputs.name }} + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - id: artifact-name - run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL" >> $GITHUB_OUTPUT - - name: Build gem5 - run: | - scons build/ALL/gem5.opt -j $(nproc) - - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.artifact-name.outputs.name }} - path: build/ALL/gem5.opt - retention-days: 5 - - run: echo "This job's status is ${{ job.status }}." + ref: develop + - id: artifact-name + run: echo "name=$(date +"%Y-%m-%d_%H.%M.%S")-ALL" >> $GITHUB_OUTPUT + - name: Build gem5 + run: | + scons build/ALL/gem5.opt -j $(nproc) + - uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.name }} + path: build/ALL/gem5.opt + retention-days: 5 + - run: echo "This job's status is ${{ job.status }}." # start running the very-long tests - testlib-very-long-tests: - strategy: - fail-fast: false - matrix: - test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] - runs-on: [self-hosted, linux, x64, run] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - needs: [build-gem5] - timeout-minutes: 4320 # 3 days - steps: - - name: Clean runner - run: - rm -rf ./* || true - rm -rf ./.??* || true - rm -rf ~/.cache || true - - uses: actions/checkout@v3 - with: + testlib-very-long-tests: + strategy: + fail-fast: false + matrix: + test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + needs: [build-gem5] + timeout-minutes: 4320 # 3 days + steps: + - name: Clean runner + run: rm -rf ./* || true rm -rf ./.??* || true rm -rf ~/.cache || true + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop - - uses: actions/download-artifact@v3 - with: - name: ${{needs.build-gem5.outputs.build-name}} - path: build/ALL - - run: chmod u+x build/ALL/gem5.opt - - name: very-long ${{ matrix.test-type }} - working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/${{ matrix.test-type }} --length very-long --skip-build -vv -t $(nproc) - - name: create zip of results - if: success() || failure() - run: | - apt-get -y install zip - zip -r output.zip tests/testing-results - - name: upload zip - if: success() || failure() - uses: actions/upload-artifact@v3 - env: - MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} - with: - name: ${{ env.MY_STEP_VAR }} - path: output.zip - retention-days: 7 - - run: echo "This job's status is ${{ job.status }}." + ref: develop + - uses: actions/download-artifact@v3 + with: + name: ${{needs.build-gem5.outputs.build-name}} + path: build/ALL + - run: chmod u+x build/ALL/gem5.opt + - name: very-long ${{ matrix.test-type }} + working-directory: ${{ github.workspace }}/tests + run: ./main.py run gem5/${{ matrix.test-type }} --length very-long --skip-build -vv -t $(nproc) + - name: create zip of results + if: success() || failure() + run: | + apt-get -y install zip + zip -r output.zip tests/testing-results + - name: upload zip + if: success() || failure() + uses: actions/upload-artifact@v3 + env: + MY_STEP_VAR: ${{ matrix.test-type }}_COMMIT.${{github.sha}}_RUN.${{github.run_id}}_ATTEMPT.${{github.run_attempt}} + with: + name: ${{ env.MY_STEP_VAR }} + path: output.zip + retention-days: 7 + - run: echo "This job's status is ${{ job.status }}." - dramsys-tests: - runs-on: [self-hosted, linux, x64, build] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest - timeout-minutes: 4320 # 3 days - steps: - - uses: actions/checkout@v3 - with: + dramsys-tests: + runs-on: [self-hosted, linux, x64] + container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + timeout-minutes: 4320 # 3 days + steps: + - uses: actions/checkout@v3 + with: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. - ref: develop + ref: develop - - name: Checkout DRAMSys - working-directory: ${{ github.workspace }}/ext/dramsys - run: | - git clone https://github.com/tukl-msd/DRAMSys DRAMSys - cd DRAMSys - git checkout -b gem5 09f6dcbb91351e6ee7cadfc7bc8b29d97625db8f - git submodule update --init --recursive + - name: Checkout DRAMSys + working-directory: ${{ github.workspace }}/ext/dramsys + run: | + git clone https://github.com/tukl-msd/DRAMSys DRAMSys + cd DRAMSys + git checkout -b gem5 09f6dcbb91351e6ee7cadfc7bc8b29d97625db8f + git submodule update --init --recursive # gem5 is built separately because it depends on the DRAMSys library - - name: Build gem5 - working-directory: ${{ github.workspace }} - run: scons build/ALL/gem5.opt -j $(nproc) + - name: Build gem5 + working-directory: ${{ github.workspace }} + run: scons build/ALL/gem5.opt -j $(nproc) - - name: Run DRAMSys Checks - working-directory: ${{ github.workspace }} - run: | - ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/arm-hello-dramsys.py - ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/dramsys-traffic.py - ./build/ALL/gem5.opt configs/example/dramsys.py + - name: Run DRAMSys Checks + working-directory: ${{ github.workspace }} + run: | + ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/arm-hello-dramsys.py + ./build/ALL/gem5.opt configs/example/gem5_library/dramsys/dramsys-traffic.py + ./build/ALL/gem5.opt configs/example/dramsys.py + weekly-tests: + # The dummy job is used to indicate whether the weekly tests have + # passed or not. This can be used as status check for pull requests. + # I.e., if we want to stop pull requests from being merged if the + # weekly tests are failing we can add this job as a required status + # check. + runs-on: ubuntu-22.04 + needs: + - testlib-very-long-tests + - dramsys-tests + steps: + - run: echo "This weekly tests have passed." From 1bb9bb33086aec5f39fdc0148552d01e29326cb0 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 18 Oct 2023 22:19:24 -0700 Subject: [PATCH 50/55] misc: Copy .github directory from develop to stable (#485) --- .github/workflows/compiler-tests.yaml | 4 +-- .github/workflows/daily-tests.yaml | 14 ++++----- .github/workflows/gpu-tests.yaml | 43 ++++++++++++++++++++++++--- .github/workflows/weekly-tests.yaml | 6 ++-- 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index 1a340e0bf5..ac092cd3d3 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -22,7 +22,7 @@ jobs: opts: [.opt, .fast] runs-on: [self-hosted, linux, x64] timeout-minutes: 2880 # 48 hours - container: gcr.io/gem5-test/${{ matrix.image }}:latest + container: ghcr.io/gem5/${{ matrix.image }}:latest steps: - uses: actions/checkout@v3 with: @@ -45,7 +45,7 @@ jobs: opts: [.opt] runs-on: [self-hosted, linux, x64] timeout-minutes: 2880 # 48 hours - container: gcr.io/gem5-test/${{ matrix.image }}:latest + container: ghcr.io/gem5/${{ matrix.image }}:latest steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 36006d03e4..89a72fd852 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -39,7 +39,7 @@ jobs: command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) runs-on: [self-hosted, linux, x64] needs: name-artifacts - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest steps: - uses: actions/checkout@v3 with: @@ -61,7 +61,7 @@ jobs: matrix: type: [fast, debug] runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -80,7 +80,7 @@ jobs: test-type: [arm_boot_tests, fs, gpu, insttest_se, learning_gem5, m5threads_test_atomic, memory, multi_isa, replacement_policies, riscv_boot_tests, stdlib, x86_boot_tests] runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours for entire matrix to run steps: @@ -169,7 +169,7 @@ jobs: matrix: test-type: [gem5-library-example-x86-ubuntu-run-ALL-x86_64-opt, gem5-library-example-riscv-ubuntu-run-ALL-x86_64-opt, lupv-example-ALL-x86_64-opt, gem5-library-example-arm-ubuntu-run-test-ALL-x86_64-opt, gem5-library-example-riscvmatched-hello-ALL-x86_64-opt] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest needs: [name-artifacts, build-gem5] timeout-minutes: 1440 # 24 hours steps: @@ -209,7 +209,7 @@ jobs: # ext/sst/sst/example.py. sst-test: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/sst-env:latest + container: ghcr.io/gem5/sst-env:latest timeout-minutes: 180 steps: @@ -234,7 +234,7 @@ jobs: # simulation with it. systemc-test: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/systemc-env:latest + container: ghcr.io/gem5/systemc-env:latest timeout-minutes: 180 steps: @@ -258,7 +258,7 @@ jobs: # Runs the gem5 Nighyly GPU tests. gpu-tests: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/gcn-gpu:latest + container: ghcr.io/gem5/gcn-gpu:latest timeout-minutes: 720 # 12 hours steps: diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index 43eff95529..4b49b34945 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -1,7 +1,9 @@ --- -# This workflow runs all of the very-long tests within main.py +# This workflow runs all the Weekly GPU Tests. +# For now this file is kept separate as we are still developing and testing +# this workflow. It will eventually be merged with "weekly-tests.yaml" -name: Weekly Tests +name: Weekly Tests (GPU) on: # Runs every Sunday from 7AM UTC @@ -13,7 +15,7 @@ on: jobs: build-gem5: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/gcn-gpu:latest + container: ghcr.io/gem5/gcn-gpu:latest steps: - uses: actions/checkout@v3 with: @@ -29,9 +31,42 @@ jobs: retention-days: 5 - run: echo "This job's status is ${{ job.status }}." + LULESH-tests: + runs-on: [self-hosted, linux, x64] + container: ghcr.io/gem5-test/gcn-gpu:latest + needs: build-gem5 + timeout-minutes: 480 # 8 hours + steps: + - uses: actions/checkout@v3 + with: + # Scheduled workflows run on the default branch by default. We + # therefore need to explicitly checkout the develop branch. + ref: develop + + - name: Download build/GCN3_X86/gem5.opt + uses: actions/download-artifact@v3 + with: + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 + path: build/GCN3_X86 + # `download-artifact` does not preserve permissions so we need to set + # them again. + - run: chmod u+x build/GCN3_X86/gem5.opt + + - name: Obtain LULESH + working-directory: ${{ github.workspace }}/lulesh + # Obtains the latest LULESH compatible with this version of gem5 via + # gem5 Resources. + run: build/GCN3_X86/gem5.opt util/obtain-resource.py lulesh -p lulesh + + - name: Run LULUESH tests + working-directory: ${{ github.workspace }} + run: | + build/GCN3_X86/gem5.opt configs/example/apu_se.py -n3 --mem-size=8GB --reg-alloc-policy=dynamic --benchmark-root="lulesh" -c \ + lulesh 0.01 2 + HACC-tests: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/gcn-gpu:latest + container: ghcr.io/gem5/gcn-gpu:latest needs: build-gem5 timeout-minutes: 120 # 2 hours steps: diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index b21613f7a1..17f111fc42 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -13,7 +13,7 @@ on: jobs: build-gem5: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest outputs: build-name: ${{ steps.artifact-name.outputs.name }} steps: @@ -41,7 +41,7 @@ jobs: matrix: test-type: [gem5_library_example_tests, gem5_resources, parsec_benchmarks, x86_boot_tests] runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest needs: [build-gem5] timeout-minutes: 4320 # 3 days steps: @@ -78,7 +78,7 @@ jobs: dramsys-tests: runs-on: [self-hosted, linux, x64] - container: gcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest timeout-minutes: 4320 # 3 days steps: - uses: actions/checkout@v3 From e9da8d67bdbb23fbd0578a379f08f42bce50121d Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 20 Oct 2023 11:43:29 -0700 Subject: [PATCH 51/55] misc: Merge develop .github dir to stable (#495) --- .github/workflows/gpu-tests.yaml | 2 +- .github/workflows/weekly-tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index 4b49b34945..836bace7d7 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -33,7 +33,7 @@ jobs: LULESH-tests: runs-on: [self-hosted, linux, x64] - container: ghcr.io/gem5-test/gcn-gpu:latest + container: ghcr.io/gem5/gcn-gpu:latest needs: build-gem5 timeout-minutes: 480 # 8 hours steps: diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 17f111fc42..8e00df5e75 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -13,7 +13,7 @@ on: jobs: build-gem5: runs-on: [self-hosted, linux, x64] - container: ghcr.io/gem5-test/ubuntu-22.04_all-dependencies:latest + container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest outputs: build-name: ${{ steps.artifact-name.outputs.name }} steps: From d0d3c74ce00f6d852a39060f3b7ce083887b1cfd Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Tue, 14 Nov 2023 13:49:37 -0800 Subject: [PATCH 52/55] misc: Merge develop .github dir to stable (#566) --- .github/workflows/ci-tests.yaml | 16 ++++++++++++++++ .github/workflows/compiler-tests.yaml | 4 ++-- .github/workflows/gpu-tests.yaml | 1 - .github/workflows/weekly-tests.yaml | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 08ec318660..49928faf20 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -89,6 +89,22 @@ jobs: build-matrix: ${{ steps.build-matrix.outputs.build-matrix }} test-dirs-matrix: ${{ steps.dir-matrix.outputs.test-dirs-matrix }} + clang-fast-compilation: + # gem5 binaries built in `quick-gem5-builds` always use GCC. + # Clang is more strict than GCC. This job checks that gem5 compiles + # with Clang. It compiles build/ALL/gem5.fast to maximize the change + # for compilation error to be exposed. + runs-on: [self-hosted, linux, x64] + if: github.event.pull_request.draft == false + container: ghcr.io/gem5/clang-version-16:latest + needs: [pre-commit, check-for-change-id] + timeout-minutes: 90 + steps: + - uses: actions/checkout@v3 + - name: Clang Compilation + working-directory: ${{ github.workspace }} + run: scons build/ALL/gem5.fast -j $(nproc) + testlib-quick-gem5-builds: runs-on: [self-hosted, linux, x64] if: github.event.pull_request.draft == false diff --git a/.github/workflows/compiler-tests.yaml b/.github/workflows/compiler-tests.yaml index ac092cd3d3..4656563357 100644 --- a/.github/workflows/compiler-tests.yaml +++ b/.github/workflows/compiler-tests.yaml @@ -16,8 +16,8 @@ jobs: strategy: fail-fast: false matrix: - image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-9, gcc-version-8, clang-version-16, clang-version-15, clang-version-14, - clang-version-13, clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, + image: [gcc-version-12, gcc-version-11, gcc-version-10, gcc-version-8, clang-version-16, clang-version-15, clang-version-14, clang-version-13, + clang-version-12, clang-version-11, clang-version-10, clang-version-9, clang-version-8, clang-version-7, ubuntu-20.04_all-dependencies, ubuntu-22.04_all-dependencies, ubuntu-22.04_min-dependencies] opts: [.opt, .fast] runs-on: [self-hosted, linux, x64] diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index 836bace7d7..b390e0750f 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -53,7 +53,6 @@ jobs: - run: chmod u+x build/GCN3_X86/gem5.opt - name: Obtain LULESH - working-directory: ${{ github.workspace }}/lulesh # Obtains the latest LULESH compatible with this version of gem5 via # gem5 Resources. run: build/GCN3_X86/gem5.opt util/obtain-resource.py lulesh -p lulesh diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 8e00df5e75..72b1454a5e 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -59,7 +59,7 @@ jobs: - run: chmod u+x build/ALL/gem5.opt - name: very-long ${{ matrix.test-type }} working-directory: ${{ github.workspace }}/tests - run: ./main.py run gem5/${{ matrix.test-type }} --length very-long --skip-build -vv -t $(nproc) + run: ./main.py run gem5/${{ matrix.test-type }} --length very-long --skip-build -vv - name: create zip of results if: success() || failure() run: | From 3bf0b1d22a375f9a1da34150293e8f8ddf433391 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Mon, 27 Nov 2023 14:32:51 -0800 Subject: [PATCH 53/55] misc: Merge develop .github dir to stable (#608) --- .github/workflows/daily-tests.yaml | 27 ++++++++++++++++++++------- .github/workflows/gpu-tests.yaml | 2 +- .github/workflows/weekly-tests.yaml | 6 +----- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/daily-tests.yaml b/.github/workflows/daily-tests.yaml index 89a72fd852..987f5c1ad2 100644 --- a/.github/workflows/daily-tests.yaml +++ b/.github/workflows/daily-tests.yaml @@ -28,15 +28,20 @@ jobs: # the default is to add -j $(nproc), but some images # require more specifications when built include: - - command-line: -j $(nproc) + - setconfig-option: '' + - isa-option: '' - image: ALL_CHI - command-line: --default=ALL PROTOCOL=CHI -j $(nproc) + setconfig-option: RUBY_PROTOCOL_CHI=y + isa-option: ALL - image: ALL_MSI - command-line: --default=ALL PROTOCOL=MSI -j $(nproc) + setconfig-option: RUBY_PROTOCOL_MSI=y + isa-option: ALL - image: ALL_MESI_Two_Level - command-line: --default=ALL PROTOCOL=MESI_Two_Level -j $(nproc) + setconfig-option: RUBY_PROTOCOL_MESI_TWO_LEVEL=y + isa-option: ALL - image: NULL_MI_example - command-line: --default=NULL PROTOCOL=MI_example -j $(nproc) + setconfig-option: RUBY_PROTOCOL_MI_EXAMPLE=y + isa-option: 'NULL' runs-on: [self-hosted, linux, x64] needs: name-artifacts container: ghcr.io/gem5/ubuntu-22.04_all-dependencies:latest @@ -46,8 +51,14 @@ jobs: # Scheduled workflows run on the default branch by default. We # therefore need to explicitly checkout the develop branch. ref: develop + - name: defconfig gem5 + if: ${{ matrix.setconfig-option != '' }} + run: scons defconfig build/${{ matrix.image }} build_opts/${{ matrix.isa-option }} + - name: setconfig gem5 + if: ${{ matrix.setconfig-option != '' }} + run: scons setconfig build/${{ matrix.image }} ${{ matrix.setconfig-option }} - name: Build gem5 - run: scons build/${{ matrix.image }}/gem5.opt ${{ matrix.command-line }} + run: scons build/${{ matrix.image }}/gem5.opt -j $(nproc) - uses: actions/upload-artifact@v3 with: name: ${{ needs.name-artifacts.outputs.build-name }}${{ matrix.image }} @@ -245,8 +256,10 @@ jobs: ref: develop - name: Build ARM/gem5.opt run: scons build/ARM/gem5.opt --ignore-style --duplicate-sources -j$(nproc) + - name: disable systemc + run: scons setconfig build/ARM --ignore-style USE_SYSTEMC=n - name: Build ARM/libgem5_opt.so - run: scons build/ARM/libgem5_opt.so --with-cxx-config --without-python --without-tcmalloc USE_SYSTEMC=0 -j$(nproc) --duplicate-sources + run: scons build/ARM/libgem5_opt.so --with-cxx-config --without-python --without-tcmalloc -j$(nproc) --duplicate-sources - name: Compile gem5 withing SystemC working-directory: ${{ github.workspace }}/util/systemc/gem5_within_systemc run: make diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index b390e0750f..7edcab5ba5 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -61,7 +61,7 @@ jobs: working-directory: ${{ github.workspace }} run: | build/GCN3_X86/gem5.opt configs/example/apu_se.py -n3 --mem-size=8GB --reg-alloc-policy=dynamic --benchmark-root="lulesh" -c \ - lulesh 0.01 2 + lulesh --options="0.01 2" HACC-tests: runs-on: [self-hosted, linux, x64] diff --git a/.github/workflows/weekly-tests.yaml b/.github/workflows/weekly-tests.yaml index 72b1454a5e..6c211435c2 100644 --- a/.github/workflows/weekly-tests.yaml +++ b/.github/workflows/weekly-tests.yaml @@ -89,11 +89,7 @@ jobs: - name: Checkout DRAMSys working-directory: ${{ github.workspace }}/ext/dramsys - run: | - git clone https://github.com/tukl-msd/DRAMSys DRAMSys - cd DRAMSys - git checkout -b gem5 09f6dcbb91351e6ee7cadfc7bc8b29d97625db8f - git submodule update --init --recursive + run: git clone https://github.com/tukl-msd/DRAMSys --branch v5.0 --depth 1 DRAMSys # gem5 is built separately because it depends on the DRAMSys library - name: Build gem5 From b99af93183b2f77781b6f4759bc06f717d3bb9bc Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Wed, 29 Nov 2023 19:04:13 -0800 Subject: [PATCH 54/55] misc: Merge .github directory from develop to stable (#626) --- .github/workflows/gpu-tests.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index 7edcab5ba5..8e26873159 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -23,11 +23,11 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - name: Build gem5 - run: scons build/GCN3_X86/gem5.opt -j $(nproc) --ignore-style + run: scons build/VEGA_X86/gem5.opt -j $(nproc) --ignore-style - uses: actions/upload-artifact@v3 with: - name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 - path: build/GCN3_X86/gem5.opt + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-vega + path: build/VEGA_X86/gem5.opt retention-days: 5 - run: echo "This job's status is ${{ job.status }}." @@ -43,24 +43,24 @@ jobs: # therefore need to explicitly checkout the develop branch. ref: develop - - name: Download build/GCN3_X86/gem5.opt + - name: Download build/VEGA_X86/gem5.opt uses: actions/download-artifact@v3 with: - name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 - path: build/GCN3_X86 + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-vega + path: build/VEGA_X86 # `download-artifact` does not preserve permissions so we need to set # them again. - - run: chmod u+x build/GCN3_X86/gem5.opt + - run: chmod u+x build/VEGA_X86/gem5.opt - name: Obtain LULESH # Obtains the latest LULESH compatible with this version of gem5 via # gem5 Resources. - run: build/GCN3_X86/gem5.opt util/obtain-resource.py lulesh -p lulesh + run: build/VEGA_X86/gem5.opt util/obtain-resource.py lulesh -p lulesh - name: Run LULUESH tests working-directory: ${{ github.workspace }} run: | - build/GCN3_X86/gem5.opt configs/example/apu_se.py -n3 --mem-size=8GB --reg-alloc-policy=dynamic --benchmark-root="lulesh" -c \ + build/VEGA_X86/gem5.opt configs/example/apu_se.py -n3 --mem-size=8GB --reg-alloc-policy=dynamic --dgpu --gfx-version=gfx900 -c \ lulesh --options="0.01 2" HACC-tests: @@ -77,8 +77,8 @@ jobs: - uses: actions/download-artifact@v3 with: name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 - path: build/GCN3_X86 - - run: chmod u+x build/GCN3_X86/gem5.opt + path: build/VEGA_X86 + - run: chmod u+x build/VEGA_X86/gem5.opt - name: make hip directory run: mkdir hip - name: Compile m5ops and x86 @@ -92,4 +92,4 @@ jobs: - name: Run HACC tests working-directory: ${{ github.workspace }} run: | - build/GCN3_X86/gem5.opt configs/example/apu_se.py -n3 --reg-alloc-policy=dynamic --benchmark-root=hip -c ForceTreeTest --options="0.5 0.1 64 0.1 1 N 12 rcb" + build/VEGA_X86/gem5.opt configs/example/apu_se.py -n3 --reg-alloc-policy=dynamic --benchmark-root=hip -c ForceTreeTest --options="0.5 0.1 64 0.1 1 N 12 rcb" From 461af5157552d916baec732dfc7d8e97a2e781d8 Mon Sep 17 00:00:00 2001 From: "Bobby R. Bruce" Date: Fri, 1 Dec 2023 17:30:03 -0800 Subject: [PATCH 55/55] misc: Update .github dir in stable from develop (#643) Change-Id: I4609e260ab594f1ca8b5ed62ea4c5707f7669411 --- .github/workflows/gpu-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index 8e26873159..a60218e6e7 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -76,7 +76,7 @@ jobs: ref: develop - uses: actions/download-artifact@v3 with: - name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-gcn3 + name: weekly-test-${{ github.run_number }}-attempt-${{ github.run_attempt }}-gem5-build-vega path: build/VEGA_X86 - run: chmod u+x build/VEGA_X86/gem5.opt - name: make hip directory