derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

resources,stdlib: Add workload to resource specialization and deprecate workload.py (#212)

Browse Source
This commit is contained in:
Bobby R. Bruce
2023-09-07 12:45:45 -07:00
committed by GitHub
parent 9cdd6093bd bbe96d6485
commit eb5ae35341
15 changed files with 158 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
configs/example/gem5_library/arm-ubuntu-run.py
View File

@@ -43,7 +43,7 @@ scons build/ARM/gem5.opt -j<NUM_CPUS>
from gem5.isas import ISA
from m5.objects import ArmDefaultRelease
from gem5.utils.requires import requires
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
from gem5.simulate.simulator import Simulator
from m5.objects import VExpress_GEM5_Foundation
from gem5.coherence_protocol import CoherenceProtocol
@@ -100,7 +100,7 @@ board = ArmBoard(
# Here we set a full system workload. The "arm64-ubuntu-20.04-boot" boots
# Ubuntu 20.04.
board.set_workload(Workload("arm64-ubuntu-20.04-boot"))
board.set_workload(obtain_resource("arm64-ubuntu-20.04-boot"))
# We define the system with the aforementioned system defined.

4
configs/example/gem5_library/caches/octopi-cache-example.py
View File

@@ -51,7 +51,7 @@ from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.octopi import
from gem5.isas import ISA
from gem5.coherence_protocol import CoherenceProtocol
from gem5.simulate.simulator import Simulator
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
num_ccds = 1 # CCDs
num_cores_per_ccd = 8 # 8 cores/CCD
@@ -94,7 +94,7 @@ board = ArmBoard(
platform=platform,
)
board.set_workload(Workload("arm64-ubuntu-20.04-boot"))
board.set_workload(obtain_resource("arm64-ubuntu-20.04-boot"))
simulator = Simulator(board=board)
simulator.run()

6
configs/example/gem5_library/looppoints/create-looppoint-checkpoints.py
View File

@@ -56,7 +56,7 @@ from gem5.components.memory.single_channel import SingleChannelDDR3_1600
from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.components.processors.cpu_types import CPUTypes
from gem5.isas import ISA
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
from pathlib import Path
from gem5.simulate.exit_event_generators import (
looppoint_save_checkpoint_generator,
@@ -110,7 +110,9 @@ board = SimpleBoard(
cache_hierarchy=cache_hierarchy,
)
board.set_workload(Workload("x86-matrix-multiply-omp-100-8-looppoint-csv"))
board.set_workload(
obtain_resource("x86-matrix-multiply-omp-100-8-looppoint-csv")
)
dir = Path(args.checkpoint_path)
dir.mkdir(exist_ok=True)

3
configs/example/gem5_library/looppoints/restore-looppoint-checkpoint.py
View File

@@ -54,7 +54,6 @@ from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.components.processors.cpu_types import CPUTypes
from gem5.isas import ISA
from gem5.resources.resource import obtain_resource
from gem5.resources.workload import Workload
from m5.stats import reset, dump
requires(isa_required=ISA.X86)
@@ -113,7 +112,7 @@ board = SimpleBoard(
)
board.set_workload(
Workload(
obtain_resource(
f"x86-matrix-multiply-omp-100-8-looppoint-region-{args.checkpoint_region}"
)
)

4
configs/example/gem5_library/riscv-ubuntu-run.py
View File

@@ -50,7 +50,7 @@ from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.components.processors.cpu_types import CPUTypes
from gem5.isas import ISA
from gem5.simulate.simulator import Simulator
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
# This runs a check to ensure the gem5 binary is compiled for RISCV.
@@ -88,7 +88,7 @@ board = RiscvBoard(
# Ubuntu 20.04. Once the system successfully boots it encounters an `m5_exit`
# instruction which stops the simulation. When the simulation has ended you may
# inspect `m5out/system.pc.com_1.device` to see the stdout.
board.set_workload(Workload("riscv-ubuntu-20.04-boot"))
board.set_workload(obtain_resource("riscv-ubuntu-20.04-boot"))
simulator = Simulator(board=board)
simulator.run()

4
configs/example/gem5_library/riscvmatched-fs.py
View File

@@ -42,7 +42,7 @@ from gem5.prebuilt.riscvmatched.riscvmatched_board import RISCVMatchedBoard
from gem5.utils.requires import requires
from gem5.isas import ISA
from gem5.simulate.simulator import Simulator
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
import argparse
@@ -76,7 +76,7 @@ board = RISCVMatchedBoard(
# In the case where the `-i` flag is passed, we add the kernel argument
# `init=/root/exit.sh`. This means the simulation will exit after the Linux
# Kernel has booted.
workload = Workload("riscv-ubuntu-20.04-boot")
workload = obtain_resource("riscv-ubuntu-20.04-boot")
kernel_args = board.get_default_kernel_args()
if args.to_init:
kernel_args.append("init=/root/exit.sh")

4
configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py
View File

@@ -52,7 +52,7 @@ from gem5.isas import ISA
from gem5.coherence_protocol import CoherenceProtocol
from gem5.simulate.simulator import Simulator
from gem5.simulate.exit_event import ExitEvent
from gem5.resources.workload import Workload
from gem5.resources.resoruce import obtain_resource
# This simulation requires using KVM with gem5 compiled for X86 simulation
# and with MESI_Two_Level cache coherence protocol.
@@ -121,7 +121,7 @@ command = (
+ "m5 exit;"
)
workload = Workload("x86-ubuntu-18.04-boot")
workload = obtain_resource("x86-ubuntu-18.04-boot")
workload.set_parameter("readfile_contents", command)
board.set_workload(workload)

4
configs/example/gem5_library/x86-ubuntu-run-with-kvm.py
View File

@@ -51,7 +51,7 @@ from gem5.isas import ISA
from gem5.coherence_protocol import CoherenceProtocol
from gem5.simulate.simulator import Simulator
from gem5.simulate.exit_event import ExitEvent
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
# This runs a check to ensure the gem5 binary is compiled to X86 and to the
# MESI Two Level coherence protocol.
@@ -117,7 +117,7 @@ command = (
+ "m5 exit;"
)
workload = Workload("x86-ubuntu-18.04-boot")
workload = obtain_resource("x86-ubuntu-18.04-boot")
workload.set_parameter("readfile_contents", command)
board.set_workload(workload)

4
configs/example/gem5_library/x86-ubuntu-run.py
View File

@@ -45,7 +45,7 @@ scons build/X86/gem5.opt
"""
from gem5.prebuilt.demo.x86_demo_board import X86DemoBoard
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
from gem5.simulate.simulator import Simulator
@@ -56,7 +56,7 @@ board = X86DemoBoard()
# We then set the workload. Here we use the "x86-ubuntu-18.04-boot" workload.
# This boots Ubuntu 18.04 with Linux 5.4.49. If the required resources are not
# found locally, they will be downloaded.
board.set_workload(Workload("x86-ubuntu-18.04-boot"))
board.set_workload(obtain_resource("x86-ubuntu-18.04-boot"))
simulator = Simulator(board=board)
simulator.run()

4
src/python/gem5/components/boards/abstract_board.py
View File

@@ -28,7 +28,7 @@ from abc import ABCMeta, abstractmethod
import inspect
from .mem_mode import MemMode, mem_mode_to_string
from ...resources.workload import AbstractWorkload
from ...resources.resource import WorkloadResource
from m5.objects import (
AddrRange,
@@ -198,7 +198,7 @@ class AbstractBoard:
)
return self._is_fs
def set_workload(self, workload: AbstractWorkload) -> None:
def set_workload(self, workload: WorkloadResource) -> None:
"""
Set the workload for this board to run.

83
src/python/gem5/resources/resource.py
View File

@@ -35,7 +35,7 @@ from .downloader import get_resource
from .looppoint import LooppointCsvLoader, LooppointJsonLoader
from ..isas import ISA, get_isa_from_str
from typing import Optional, Dict, Union, Type, Tuple, List
from typing import Optional, Dict, Union, Type, Tuple, List, Any
from .client import get_resource_json_obj
@@ -554,6 +554,66 @@ class SimpointDirectoryResource(SimpointResource):
return simpoint_list, weight_list
class WorkloadResource(AbstractResource):
"""A workload resource. This resource is used to specify a workload to run
on a board. It contains the function to call and the parameters to pass to
that function.
"""
def __init__(
self,
function: str = None,
resource_version: Optional[str] = None,
description: Optional[str] = None,
source: Optional[str] = None,
local_path: Optional[str] = None,
parameters: Optional[Dict[str, Any]] = {},
**kwargs,
):
"""
:param function: The function to call on the board.
:param parameters: The parameters to pass to the function.
"""
super().__init__(
local_path=local_path,
description=description,
source=source,
resource_version=resource_version,
)
self._func = function
self._params = parameters
def get_function_str(self) -> str:
"""
Returns the name of the workload function to be run.
This function is called via the AbstractBoard's `set_workload`
function. The parameters from the `get_parameters` function are passed
to this function.
"""
return self._func
def get_parameters(self) -> Dict[str, Any]:
"""
Returns a dictionary mapping the workload parameters to their values.
These parameters are passed to the function specified by
`get_function_str` via the AbstractBoard's `set_workload` function.
"""
return self._params
def set_parameter(self, parameter: str, value: Any) -> None:
"""
Used to set or override a workload parameter
:param parameter: The parameter of the function to set.
:param value: The value to set to the parameter.
"""
self._params[parameter] = value
def obtain_resource(
resource_id: str,
resource_directory: Optional[str] = None,
@@ -658,6 +718,26 @@ def obtain_resource(
assert resources_category in _get_resource_json_type_map
resource_class = _get_resource_json_type_map[resources_category]
if resources_category == "workload":
# This parses the "resources" and "additional_params" fields of the
# workload resource into a dictionary of AbstractResource objects and
# strings respectively.
params = {}
if "resources" in resource_json:
for key in resource_json["resources"].keys():
assert isinstance(key, str)
value = resource_json["resources"][key]
assert isinstance(value, str)
params[key] = obtain_resource(
value,
)
if "additional_params" in resource_json:
for key in resource_json["additional_params"].keys():
assert isinstance(key, str)
value = resource_json["additional_params"][key]
assert isinstance(value, str)
params[key] = value
resource_json["parameters"] = params
# Once we know what AbstractResource subclass we are using, we create it.
# The fields in the JSON object are assumed to map like-for-like to the
# subclass contructor, so we can pass the resource_json map directly.
@@ -812,4 +892,5 @@ _get_resource_json_type_map = {
"resource": Resource,
"looppoint-pinpoint-csv": LooppointCsvResource,
"looppoint-json": LooppointJsonResource,
"workload": WorkloadResource,
}

248
src/python/gem5/resources/workload.py
View File

@@ -24,219 +24,67 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from .resource import obtain_resource
from .resource import obtain_resource, WorkloadResource
from .client import get_resource_json_obj
from _m5 import core
from m5.util import warn
from typing import Dict, Any, List, Optional
class AbstractWorkload:
def CustomWorkload(function: str, parameters: Dict[str, Any]):
"""
Workloads contain information needed to build a workload.
A custom workload gem5 resource. It can be used to specify a custom,
local workload.
A workload specifies a function and its parameters to run on a board to
set a workload. Workload's are passed to board via the `AbstractBoard`'s
`set_workload` function.
**Warning**: This `CustomWorkload` class is deprecated. It will be removed in a
future release of gem5. Please use the `gem5.resources.resource.WorkloadResource`
class instead.
The `AbstractBoard` has a `set_workload` function which accepts an
AbstractWorkload. The `set_workload` function uses the `get_function_str`
to determine which function should be called on the board and the
`get_parameters` function specifies the parameters to be passed.
Example
-------
```py
workload = CustomWorkload(
function = "set_se_binary_workload",
parameters = {
"binary" : obtain_resource("x86-print-this"),
"arguments" : ["hello", 6]
},
The class has been stealthily converted to a function which wraps the
`WorkloadResource` class.
"""
warn(
"The `CustomWorkload` class is deprecated. Please use "
"the `gem5.resources.resource.WorkloadResource` class instead."
)
return WorkloadResource(function=function, parameters=parameters)
board.set_workload(workload)
```
The above is the equivalent of:
def Workload(
workload_name: str,
resource_directory: Optional[str] = None,
resource_version: Optional[str] = None,
clients: Optional[List] = None,
gem5_version: Optional[str] = core.gem5Version,
):
"""
**Warning**: The `Workload` class is deprecated. It will be removed in a
future release of gem5. Please use the `gem5.resources.resource.WorkloadResource`
class instead.
```py
board.set_se_binary_workload(
binary = obtain_resource("x86-print-this"),
arguments = ["hello", 6],
The class has been stealthily converted to a function which wraps the
`WorkloadResource` class.
"""
warn(
"`Workload` has been deprecated. Please use the `obtain_resource` "
"function instead:\n\n"
"```\n"
"from gem5.resources.resource import obtain_resource\n"
"workload = obtain_resource(\n"
f' resource_id="{workload_name}",\n'
f' resource_directory="{resource_directory}",\n'
f' gem5_version="{gem5_version}",\n'
f" clients={clients},\n"
f" resource_version={resource_version},\n"
")\n"
"```"
)
```
Notes
-----
This class should not be used directly. Please use `Workload` or
`CustomWorkload`.
"""
def __init__(self, function: str, parameters: Dict[str, Any]) -> None:
self._func = function
self._params = parameters
def get_function_str(self) -> str:
"""
Returns the name of the workload function to be run.
This function is called via the AbstractBoard's `set_workload`
function. The parameters from the `get_parameters` function are passed
to this function.
"""
return self._func
def get_parameters(self) -> Dict[str, Any]:
"""
Returns a dictionary mapping the workload parameters to their values.
These parameters are passed to the function specified by
`get_function_str` via the AbstractBoard's `set_workload` function.
"""
return self._params
def set_parameter(self, parameter: str, value: Any) -> None:
"""
Used to set or override a workload parameter
:param parameter: The parameter of the function to set.
:param value: The value to set to the parameter.
"""
self._params[parameter] = value
class CustomWorkload(AbstractWorkload):
"""
A workload specified locally (i.e., not via gem5-resources as with the
`Workload` class). Here the user specifies the function and the parameters
to be passed.
Usage
-----
```py
workload = CustomWorkload(
function = "set_se_binary_workload",
parameters = {
"binary" : obtain_resource("x86-print-this"),
"arguments" : ["hello", 6]
},
return obtain_resource(
workload_name,
resource_directory=resource_directory,
gem5_version=gem5_version,
clients=clients,
resource_version=resource_version,
)
board.set_workload(workload)
```
"""
def __init__(self, function: str, parameters: Dict[str, Any]) -> None:
super().__init__(function=function, parameters=parameters)
class Workload(AbstractWorkload):
"""
The `Workload` class loads a workload's information from gem5-resources
based on a name/id passed via the constructor.
Usage
-----
```py
# Determine what workload we want to run.
workload = Workload("example-workload-id")
# Optionally we can override a parameter in the workload. In this example
# we are going to run this workload with a difference kernel.
workload.set_parameter("kernel",
obtain_resource("arm64-linux-kernel-4.14.134")
)
# We then set this workload to the board.
board.set_workload(workload)
```
"""
def __init__(
self,
workload_name: str,
resource_directory: Optional[str] = None,
resource_version: Optional[str] = None,
clients: Optional[List] = None,
gem5_version: Optional[str] = core.gem5Version,
) -> None:
"""
This constructor will load the workload details from the workload with
the given name/id.
This function assumes the dictionary returned by the downloader's
`get_workload_json_obj` is a dictionary. An example of the schema is
shown below:
```json
{
"category" : "workload",
"id" : "x86-ubuntu-18.04-echo-hello",
"description" : "Description of workload here",
"function" : "set_kernel_disk_workload",
"resources" : {
"kernel" : "x86-linux-kernel-5.4.49",
"disk-image" : "x86-ubuntu-18.04-img"
},
"additional_params" : {
"readfile_contents" : "m5_exit; echo 'hello'; m5_exit"
}
}
```
This resource will result in the equivalent of the following action
being taken:
```python
board.set_kernel_disk_workload(
kernel = obtain_resource("x86-linux-kernel-5.4.49"),
disk-image = obtain_resource("x86-ubuntu-18.04-img"),
readfile_contents = "m5_exit; echo 'hello'; m5_exit",
)
```
:param workload_name: The name of the workload in the resources.json
file to be loaded.
:param resource_directory: An optional parameter that specifies where
any resources should be download and accessed from. If None, a default
location will be used. None by default.
:param gem5_version: The gem5 version for the Workload to be loaded.
By default, the current gem5 version is used. This will filter
resources which are incompatible with the current gem5 version. If
None, no filtering will be done.
"""
workload_json = get_resource_json_obj(
workload_name,
resource_version=resource_version,
clients=clients,
gem5_version=gem5_version,
)
func = workload_json["function"]
assert isinstance(func, str)
params = {}
if "resources" in workload_json:
for key in workload_json["resources"].keys():
assert isinstance(key, str)
value = workload_json["resources"][key]
assert isinstance(value, str)
params[key] = obtain_resource(
value,
resource_directory=resource_directory,
gem5_version=gem5_version,
)
if "additional_params" in workload_json:
for key in workload_json["additional_params"]:
assert isinstance(key, str)
params[key] = workload_json["additional_params"][key]
super().__init__(function=func, parameters=params)

4
tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py
View File

@@ -40,7 +40,7 @@ from gem5.components.processors.cpu_types import CPUTypes
from gem5.components.boards.riscv_board import RiscvBoard
from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.simulate.simulator import Simulator
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
import argparse
import importlib
@@ -160,7 +160,7 @@ board = RiscvBoard(
)
# Set the workload.
workload = Workload(
workload = obtain_resource(
"riscv-ubuntu-20.04-boot", resource_directory=args.resource_directory
)
board.set_workload(workload)

4
tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py
View File

@@ -41,7 +41,7 @@ from gem5.components.processors.cpu_types import (
)
from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.simulate.simulator import Simulator
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource
import argparse
import importlib
@@ -184,7 +184,7 @@ if args.boot_type == "init":
kernal_args.append("init=/root/exit.sh")
# Set the workload.
workload = Workload(
workload = obtain_resource(
"x86-ubuntu-18.04-boot", resource_directory=args.resource_directory
)
workload.set_parameter("kernel_args", kernal_args)

5
tests/pyunit/stdlib/resources/pyunit_workload_checks.py
View File

@@ -32,6 +32,7 @@ from gem5.resources.resource import (
BinaryResource,
DiskImageResource,
obtain_resource,
WorkloadResource,
)
from typing import Dict
@@ -61,7 +62,7 @@ class CustomWorkloadTestSuite(unittest.TestCase):
new=ClientWrapper(mock_config_json),
)
def setUpClass(cls) -> None:
cls.custom_workload = CustomWorkload(
cls.custom_workload = WorkloadResource(
function="set_se_binary_workload",
parameters={
"binary": obtain_resource(
@@ -135,7 +136,7 @@ class WorkloadTestSuite(unittest.TestCase):
ClientWrapper(mock_config_json),
)
def setUpClass(cls):
cls.workload = Workload("simple-boot", gem5_version="develop")
cls.workload = obtain_resource("simple-boot", gem5_version="develop")
def test_get_function_str(self) -> None:
# Tests `Resource.get_function_str`