From ea5951a4675f115cab4f7edd643784527c2643de Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 18 Aug 2023 08:57:31 -0700 Subject: [PATCH 1/7] stdlib, resources: skeleton for workload resouce Change-Id: I5017ac479ad61c767ede36fae195105e0519304f --- .../gem5/components/boards/abstract_board.py | 4 +- src/python/gem5/resources/resource.py | 78 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/python/gem5/components/boards/abstract_board.py b/src/python/gem5/components/boards/abstract_board.py index 4ea8866009..aba080e239 100644 --- a/src/python/gem5/components/boards/abstract_board.py +++ b/src/python/gem5/components/boards/abstract_board.py @@ -28,7 +28,7 @@ from abc import ABCMeta, abstractmethod import inspect from .mem_mode import MemMode, mem_mode_to_string -from ...resources.workload import AbstractWorkload +from ...resources.resource import WorkloadResource from m5.objects import ( AddrRange, @@ -198,7 +198,7 @@ class AbstractBoard: ) return self._is_fs - def set_workload(self, workload: AbstractWorkload) -> None: + def set_workload(self, workload: WorkloadResource) -> None: """ Set the workload for this board to run. diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index bc9f4480ba..2f50f36631 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -35,7 +35,7 @@ from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str -from typing import Optional, Dict, Union, Type, Tuple, List +from typing import Optional, Dict, Union, Type, Tuple, List, Any from .client import get_resource_json_obj @@ -554,6 +554,79 @@ class SimpointDirectoryResource(SimpointResource): return simpoint_list, weight_list +class WorkloadResource(AbstractResource): + """A workload resource. This resource is used to specify a workload to run + on a board. It contains the function to call and the parameters to pass to + that function. + """ + + def __init__( + self, + resource_version: Optional[str] = None, + function: str = None, + resoucres: Dict[str, Dict[str, str]] = None, + additional_params: Dict[str, str] = None, + description: Optional[str] = None, + source: Optional[str] = None, + local_path: Optional[str] = None, + ): + """ + :param function: The function to call on the board. + :param parameters: The parameters to pass to the function. + """ + + super().__init__( + local_path=local_path, + description=description, + source=source, + resource_version=resource_version, + ) + + self._func = function + self._params = {} + for key in resoucres.keys(): + assert isinstance(key, str) + value = resoucres[key] + assert isinstance(value, dict) + self._params[key] = obtain_resource( + value["id"], + resource_version=value["resource_version"], + ) + for key in additional_params.keys(): + assert isinstance(key, str) + value = additional_params[key] + assert isinstance(value, str) + self._params[key] = value + + def get_function_str(self) -> str: + """ + Returns the name of the workload function to be run. + + This function is called via the AbstractBoard's `set_workload` + function. The parameters from the `get_parameters` function are passed + to this function. + """ + return self._func + + def get_parameters(self) -> Dict[str, Any]: + """ + Returns a dictionary mapping the workload parameters to their values. + + These parameters are passed to the function specified by + `get_function_str` via the AbstractBoard's `set_workload` function. + """ + return self._params + + def set_parameter(self, parameter: str, value: Any) -> None: + """ + Used to set or override a workload parameter + + :param parameter: The parameter of the function to set. + :param value: The value to set to the parameter. + """ + self._params[parameter] = value + + def obtain_resource( resource_id: str, resource_directory: Optional[str] = None, @@ -640,7 +713,7 @@ def obtain_resource( # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. resources_category = resource_json["category"] - + print(resource_json) if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their @@ -812,4 +885,5 @@ _get_resource_json_type_map = { "resource": Resource, "looppoint-pinpoint-csv": LooppointCsvResource, "looppoint-json": LooppointJsonResource, + "workload": WorkloadResource, } From a18b4b17ed43c315001c145e8d4fae44e7e1a324 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 18 Aug 2023 13:56:12 -0700 Subject: [PATCH 2/7] stdlib, resources: depricated workload - Added WrokloadResource in resource.py. - depricated Workload and CustomWorkload. - changed iscvmatched-fs.py with obtain resource for workload to test. Change-Id: I2267c44249b96ca37da3890bf630e0d15c7335ed Note: change example files back to original --- .../example/gem5_library/riscvmatched-fs.py | 3 +- src/python/gem5/resources/resource.py | 20 +- src/python/gem5/resources/workload.py | 226 +++++------------- 3 files changed, 76 insertions(+), 173 deletions(-) diff --git a/configs/example/gem5_library/riscvmatched-fs.py b/configs/example/gem5_library/riscvmatched-fs.py index 3e84b8c1ea..5d8bb113dc 100644 --- a/configs/example/gem5_library/riscvmatched-fs.py +++ b/configs/example/gem5_library/riscvmatched-fs.py @@ -43,6 +43,7 @@ from gem5.utils.requires import requires from gem5.isas import ISA from gem5.simulate.simulator import Simulator from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource import argparse @@ -76,7 +77,7 @@ board = RISCVMatchedBoard( # In the case where the `-i` flag is passed, we add the kernel argument # `init=/root/exit.sh`. This means the simulation will exit after the Linux # Kernel has booted. -workload = Workload("riscv-ubuntu-20.04-boot") +workload = obtain_resource("riscv-ubuntu-20.04-boot") kernel_args = board.get_default_kernel_args() if args.to_init: kernel_args.append("init=/root/exit.sh") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 2f50f36631..9c665e8284 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -562,13 +562,15 @@ class WorkloadResource(AbstractResource): def __init__( self, - resource_version: Optional[str] = None, function: str = None, - resoucres: Dict[str, Dict[str, str]] = None, - additional_params: Dict[str, str] = None, + resources: Optional[Dict[str, str]] = None, + additional_params: Optional[Dict[str, str]] = None, + resource_version: Optional[str] = None, description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, + parameters: Optional[Dict[str, Any]] = None, + **kwargs, ): """ :param function: The function to call on the board. @@ -583,14 +585,14 @@ class WorkloadResource(AbstractResource): ) self._func = function - self._params = {} - for key in resoucres.keys(): + self._params = parameters if parameters else {} + print(resources) + for key in resources.keys(): assert isinstance(key, str) - value = resoucres[key] - assert isinstance(value, dict) + value = resources[key] + assert isinstance(value, str) self._params[key] = obtain_resource( - value["id"], - resource_version=value["resource_version"], + value, ) for key in additional_params.keys(): assert isinstance(key, str) diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index 5b25bf879f..26197131dc 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -24,92 +24,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from .resource import obtain_resource +from .resource import obtain_resource, WorkloadResource from .client import get_resource_json_obj from _m5 import core from typing import Dict, Any, List, Optional - -class AbstractWorkload: - """ - Workloads contain information needed to build a workload. - - A workload specifies a function and its parameters to run on a board to - set a workload. Workload's are passed to board via the `AbstractBoard`'s - `set_workload` function. - - The `AbstractBoard` has a `set_workload` function which accepts an - AbstractWorkload. The `set_workload` function uses the `get_function_str` - to determine which function should be called on the board and the - `get_parameters` function specifies the parameters to be passed. - - Example - ------- - - ```py - workload = CustomWorkload( - function = "set_se_binary_workload", - parameters = { - "binary" : obtain_resource("x86-print-this"), - "arguments" : ["hello", 6] - }, - ) - - board.set_workload(workload) - ``` - - The above is the equivalent of: - - ```py - board.set_se_binary_workload( - binary = obtain_resource("x86-print-this"), - arguments = ["hello", 6], - ) - ``` - - Notes - ----- - This class should not be used directly. Please use `Workload` or - `CustomWorkload`. - """ - - def __init__(self, function: str, parameters: Dict[str, Any]) -> None: - self._func = function - self._params = parameters - - def get_function_str(self) -> str: - """ - Returns the name of the workload function to be run. - - This function is called via the AbstractBoard's `set_workload` - function. The parameters from the `get_parameters` function are passed - to this function. - """ - return self._func - - def get_parameters(self) -> Dict[str, Any]: - """ - Returns a dictionary mapping the workload parameters to their values. - - These parameters are passed to the function specified by - `get_function_str` via the AbstractBoard's `set_workload` function. - """ - return self._params - - def set_parameter(self, parameter: str, value: Any) -> None: - """ - Used to set or override a workload parameter - - :param parameter: The parameter of the function to set. - :param value: The value to set to the parameter. - """ - self._params[parameter] = value - - -class CustomWorkload(AbstractWorkload): - """ +""" A workload specified locally (i.e., not via gem5-resources as with the `Workload` class). Here the user specifies the function and the parameters to be passed. @@ -128,14 +50,14 @@ class CustomWorkload(AbstractWorkload): board.set_workload(workload) ``` - """ - - def __init__(self, function: str, parameters: Dict[str, Any]) -> None: - super().__init__(function=function, parameters=parameters) +""" -class Workload(AbstractWorkload): - """ +def CustomWorkload(function: str, parameters: Dict[str, Any]) -> None: + return WorkloadResource(function=function, parameters=parameters) + + +""" The `Workload` class loads a workload's information from gem5-resources based on a name/id passed via the constructor. @@ -156,87 +78,65 @@ class Workload(AbstractWorkload): board.set_workload(workload) ``` +""" + + +def Workload( + workload_name: str, + resource_directory: Optional[str] = None, + resource_version: Optional[str] = None, + clients: Optional[List] = None, + gem5_version: Optional[str] = core.gem5Version, +): """ + This constructor will load the workload details from the workload with + the given name/id. - def __init__( - self, - workload_name: str, - resource_directory: Optional[str] = None, - resource_version: Optional[str] = None, - clients: Optional[List] = None, - gem5_version: Optional[str] = core.gem5Version, - ) -> None: - """ - This constructor will load the workload details from the workload with - the given name/id. + This function assumes the dictionary returned by the downloader's + `get_workload_json_obj` is a dictionary. An example of the schema is + shown below: - This function assumes the dictionary returned by the downloader's - `get_workload_json_obj` is a dictionary. An example of the schema is - shown below: - - ```json - { - "category" : "workload", - "id" : "x86-ubuntu-18.04-echo-hello", - "description" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.4.49", - "disk-image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" - } + ```json + { + "category" : "workload", + "id" : "x86-ubuntu-18.04-echo-hello", + "description" : "Description of workload here", + "function" : "set_kernel_disk_workload", + "resources" : { + "kernel" : "x86-linux-kernel-5.4.49", + "disk-image" : "x86-ubuntu-18.04-img" + }, + "additional_params" : { + "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" } - ``` + } + ``` - This resource will result in the equivalent of the following action - being taken: + This resource will result in the equivalent of the following action + being taken: - ```python - board.set_kernel_disk_workload( - kernel = obtain_resource("x86-linux-kernel-5.4.49"), - disk-image = obtain_resource("x86-ubuntu-18.04-img"), - readfile_contents = "m5_exit; echo 'hello'; m5_exit", - ) - ``` + ```python + board.set_kernel_disk_workload( + kernel = obtain_resource("x86-linux-kernel-5.4.49"), + disk-image = obtain_resource("x86-ubuntu-18.04-img"), + readfile_contents = "m5_exit; echo 'hello'; m5_exit", + ) + ``` - :param workload_name: The name of the workload in the resources.json - file to be loaded. - :param resource_directory: An optional parameter that specifies where - any resources should be download and accessed from. If None, a default - location will be used. None by default. - :param gem5_version: The gem5 version for the Workload to be loaded. - By default, the current gem5 version is used. This will filter - resources which are incompatible with the current gem5 version. If - None, no filtering will be done. - """ - - workload_json = get_resource_json_obj( - workload_name, - resource_version=resource_version, - clients=clients, - gem5_version=gem5_version, - ) - - func = workload_json["function"] - assert isinstance(func, str) - - params = {} - if "resources" in workload_json: - for key in workload_json["resources"].keys(): - assert isinstance(key, str) - value = workload_json["resources"][key] - assert isinstance(value, str) - params[key] = obtain_resource( - value, - resource_directory=resource_directory, - gem5_version=gem5_version, - ) - - if "additional_params" in workload_json: - for key in workload_json["additional_params"]: - assert isinstance(key, str) - params[key] = workload_json["additional_params"][key] - - super().__init__(function=func, parameters=params) + :param workload_name: The name of the workload in the resources.json + file to be loaded. + :param resource_directory: An optional parameter that specifies where + any resources should be download and accessed from. If None, a default + location will be used. None by default. + :param gem5_version: The gem5 version for the Workload to be loaded. + By default, the current gem5 version is used. This will filter + resources which are incompatible with the current gem5 version. If + None, no filtering will be done. + """ + return obtain_resource( + workload_name, + resource_directory=resource_directory, + gem5_version=gem5_version, + clients=clients, + resource_version=resource_version, + ) From 328d140c703a4253d77e62ff10d80d4c3f10dfb2 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 23 Aug 2023 13:50:08 -0700 Subject: [PATCH 3/7] stdlib, resources: Added warn msgs and commets. - Added deprecated warnings to Workload and Abstract workload. - Added comments to the classes changed. Change-Id: I671daacf5ef455ea65103bd96aa442486142a486 --- .../example/gem5_library/riscvmatched-fs.py | 3 +- src/python/gem5/resources/resource.py | 2 - src/python/gem5/resources/workload.py | 108 ++++-------------- 3 files changed, 23 insertions(+), 90 deletions(-) diff --git a/configs/example/gem5_library/riscvmatched-fs.py b/configs/example/gem5_library/riscvmatched-fs.py index 5d8bb113dc..3e84b8c1ea 100644 --- a/configs/example/gem5_library/riscvmatched-fs.py +++ b/configs/example/gem5_library/riscvmatched-fs.py @@ -43,7 +43,6 @@ from gem5.utils.requires import requires from gem5.isas import ISA from gem5.simulate.simulator import Simulator from gem5.resources.workload import Workload -from gem5.resources.resource import obtain_resource import argparse @@ -77,7 +76,7 @@ board = RISCVMatchedBoard( # In the case where the `-i` flag is passed, we add the kernel argument # `init=/root/exit.sh`. This means the simulation will exit after the Linux # Kernel has booted. -workload = obtain_resource("riscv-ubuntu-20.04-boot") +workload = Workload("riscv-ubuntu-20.04-boot") kernel_args = board.get_default_kernel_args() if args.to_init: kernel_args.append("init=/root/exit.sh") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 9c665e8284..e71c7646a9 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -586,7 +586,6 @@ class WorkloadResource(AbstractResource): self._func = function self._params = parameters if parameters else {} - print(resources) for key in resources.keys(): assert isinstance(key, str) value = resources[key] @@ -715,7 +714,6 @@ def obtain_resource( # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. resources_category = resource_json["category"] - print(resource_json) if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index 26197131dc..424da2ebc6 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -28,59 +28,27 @@ from .resource import obtain_resource, WorkloadResource from .client import get_resource_json_obj from _m5 import core +from m5.util import warn from typing import Dict, Any, List, Optional -""" - A workload specified locally (i.e., not via gem5-resources as with the - `Workload` class). Here the user specifies the function and the parameters - to be passed. - Usage - ----- +def CustomWorkload(function: str, parameters: Dict[str, Any]): + """ + A custom workload gem5 resource. It can be used to specify a custom, + local workload. - ```py - workload = CustomWorkload( - function = "set_se_binary_workload", - parameters = { - "binary" : obtain_resource("x86-print-this"), - "arguments" : ["hello", 6] - }, + **Warning**: This class is deprecated and changed to a funtion and will be removed in future + releases of gem5. Please use the `WorkloadResource` class instead. This + class is merely a wrapper for it. + """ + warn( + "The `CustomWorkload` class is deprecated. Please use " + "`WorkloadResource` instead." ) - - board.set_workload(workload) - ``` -""" - - -def CustomWorkload(function: str, parameters: Dict[str, Any]) -> None: return WorkloadResource(function=function, parameters=parameters) -""" - The `Workload` class loads a workload's information from gem5-resources - based on a name/id passed via the constructor. - - Usage - ----- - - ```py - # Determine what workload we want to run. - workload = Workload("example-workload-id") - - # Optionally we can override a parameter in the workload. In this example - # we are going to run this workload with a difference kernel. - workload.set_parameter("kernel", - obtain_resource("arm64-linux-kernel-4.14.134") - ) - - # We then set this workload to the board. - board.set_workload(workload) - ``` - -""" - - def Workload( workload_name: str, resource_directory: Optional[str] = None, @@ -89,50 +57,18 @@ def Workload( gem5_version: Optional[str] = core.gem5Version, ): """ - This constructor will load the workload details from the workload with - the given name/id. + This function was created to maintain backwards compability for v23.0.0 + and prior releases of gem5 where `Workload` was a class. - This function assumes the dictionary returned by the downloader's - `get_workload_json_obj` is a dictionary. An example of the schema is - shown below: - - ```json - { - "category" : "workload", - "id" : "x86-ubuntu-18.04-echo-hello", - "description" : "Description of workload here", - "function" : "set_kernel_disk_workload", - "resources" : { - "kernel" : "x86-linux-kernel-5.4.49", - "disk-image" : "x86-ubuntu-18.04-img" - }, - "additional_params" : { - "readfile_contents" : "m5_exit; echo 'hello'; m5_exit" - } - } - ``` - - This resource will result in the equivalent of the following action - being taken: - - ```python - board.set_kernel_disk_workload( - kernel = obtain_resource("x86-linux-kernel-5.4.49"), - disk-image = obtain_resource("x86-ubuntu-18.04-img"), - readfile_contents = "m5_exit; echo 'hello'; m5_exit", - ) - ``` - - :param workload_name: The name of the workload in the resources.json - file to be loaded. - :param resource_directory: An optional parameter that specifies where - any resources should be download and accessed from. If None, a default - location will be used. None by default. - :param gem5_version: The gem5 version for the Workload to be loaded. - By default, the current gem5 version is used. This will filter - resources which are incompatible with the current gem5 version. If - None, no filtering will be done. + In the interests of gem5-resource specialization, the `Workload` class + has been dropped. Instead users are advized to use the `obtain_resource` + function which will return the correct `AbstractResource` implementation. + This function (disguised as a class) wraps this function. """ + warn( + "`Workload` has been deprecated. Please use the `obtain_resource` " + "function instead." + ) return obtain_resource( workload_name, resource_directory=resource_directory, From 7da0aeee7d6330c69da6f6142d1df00fe90d147c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 29 Aug 2023 09:08:44 -0700 Subject: [PATCH 4/7] stdlib, resources: Updated warn messages - Updated warn messages to be more clear. Change-Id: I2a8922e96b5b544f2f229b01b3c51fc5e79995b4 --- src/python/gem5/resources/resource.py | 1 + src/python/gem5/resources/workload.py | 34 ++++++++++++++++++--------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index e71c7646a9..8db1d4bdd4 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -714,6 +714,7 @@ def obtain_resource( # Obtain the type from the JSON. From this we will determine what subclass # of `AbstractResource` we are to create and return. resources_category = resource_json["category"] + if resources_category == "resource": # This is a stop-gap measure to ensure to work with older versions of # the "resource.json" file. These should be replaced with their diff --git a/src/python/gem5/resources/workload.py b/src/python/gem5/resources/workload.py index 424da2ebc6..bdb596921f 100644 --- a/src/python/gem5/resources/workload.py +++ b/src/python/gem5/resources/workload.py @@ -38,13 +38,16 @@ def CustomWorkload(function: str, parameters: Dict[str, Any]): A custom workload gem5 resource. It can be used to specify a custom, local workload. - **Warning**: This class is deprecated and changed to a funtion and will be removed in future - releases of gem5. Please use the `WorkloadResource` class instead. This - class is merely a wrapper for it. + **Warning**: This `CustomWorkload` class is deprecated. It will be removed in a + future release of gem5. Please use the `gem5.resources.resource.WorkloadResource` + class instead. + + The class has been stealthily converted to a function which wraps the + `WorkloadResource` class. """ warn( "The `CustomWorkload` class is deprecated. Please use " - "`WorkloadResource` instead." + "the `gem5.resources.resource.WorkloadResource` class instead." ) return WorkloadResource(function=function, parameters=parameters) @@ -57,17 +60,26 @@ def Workload( gem5_version: Optional[str] = core.gem5Version, ): """ - This function was created to maintain backwards compability for v23.0.0 - and prior releases of gem5 where `Workload` was a class. + **Warning**: The `Workload` class is deprecated. It will be removed in a + future release of gem5. Please use the `gem5.resources.resource.WorkloadResource` + class instead. - In the interests of gem5-resource specialization, the `Workload` class - has been dropped. Instead users are advized to use the `obtain_resource` - function which will return the correct `AbstractResource` implementation. - This function (disguised as a class) wraps this function. + The class has been stealthily converted to a function which wraps the + `WorkloadResource` class. """ warn( "`Workload` has been deprecated. Please use the `obtain_resource` " - "function instead." + "function instead:\n\n" + "```\n" + "from gem5.resources.resource import obtain_resource\n" + "workload = obtain_resource(\n" + f' resource_id="{workload_name}",\n' + f' resource_directory="{resource_directory}",\n' + f' gem5_version="{gem5_version}",\n' + f" clients={clients},\n" + f" resource_version={resource_version},\n" + ")\n" + "```" ) return obtain_resource( workload_name, From 7042d74ac2efa16b8ff6b2b7e1251f238cc4aff0 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 31 Aug 2023 09:19:24 -0700 Subject: [PATCH 5/7] stdlib, resources: Moved parsing params. - moved parsing from WorkloadResource __init__ to obtain_resource Change-Id: I9ed2aebb23af7f359bc1e5fff8ebe616a0da1374 --- src/python/gem5/resources/resource.py | 38 ++++++++++++++++----------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 8db1d4bdd4..9e6c79ab01 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -563,13 +563,11 @@ class WorkloadResource(AbstractResource): def __init__( self, function: str = None, - resources: Optional[Dict[str, str]] = None, - additional_params: Optional[Dict[str, str]] = None, resource_version: Optional[str] = None, description: Optional[str] = None, source: Optional[str] = None, local_path: Optional[str] = None, - parameters: Optional[Dict[str, Any]] = None, + parameters: Optional[Dict[str, Any]] = {}, **kwargs, ): """ @@ -585,19 +583,7 @@ class WorkloadResource(AbstractResource): ) self._func = function - self._params = parameters if parameters else {} - for key in resources.keys(): - assert isinstance(key, str) - value = resources[key] - assert isinstance(value, str) - self._params[key] = obtain_resource( - value, - ) - for key in additional_params.keys(): - assert isinstance(key, str) - value = additional_params[key] - assert isinstance(value, str) - self._params[key] = value + self._params = parameters def get_function_str(self) -> str: """ @@ -732,6 +718,26 @@ def obtain_resource( assert resources_category in _get_resource_json_type_map resource_class = _get_resource_json_type_map[resources_category] + if resources_category == "workload": + # This parses the "resources" and "additional_params" fields of the + # workload resource into a dictionary of AbstractResource objects and + # strings respectively. + params = {} + if "resources" in resource_json: + for key in resource_json["resources"].keys(): + assert isinstance(key, str) + value = resource_json["resources"][key] + assert isinstance(value, str) + params[key] = obtain_resource( + value, + ) + if "additional_params" in resource_json: + for key in resource_json["additional_params"].keys(): + assert isinstance(key, str) + value = resource_json["additional_params"][key] + assert isinstance(value, str) + params[key] = value + resource_json["parameters"] = params # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the # subclass contructor, so we can pass the resource_json map directly. From bf06d61c3572e7a6d41737a30e2f941a5f81b7a6 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 5 Sep 2023 12:39:38 -0700 Subject: [PATCH 6/7] stdlib, tests, resources: Updated tests - Updated workload tests to use WrokloadResource and obtain_resource Change-Id: I39194e7fe764566a528e5141c29f30efa14e0cde --- tests/pyunit/stdlib/resources/pyunit_workload_checks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py index ef7c3caba1..a9fa5e28e2 100644 --- a/tests/pyunit/stdlib/resources/pyunit_workload_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_workload_checks.py @@ -32,6 +32,7 @@ from gem5.resources.resource import ( BinaryResource, DiskImageResource, obtain_resource, + WorkloadResource, ) from typing import Dict @@ -61,7 +62,7 @@ class CustomWorkloadTestSuite(unittest.TestCase): new=ClientWrapper(mock_config_json), ) def setUpClass(cls) -> None: - cls.custom_workload = CustomWorkload( + cls.custom_workload = WorkloadResource( function="set_se_binary_workload", parameters={ "binary": obtain_resource( @@ -135,7 +136,7 @@ class WorkloadTestSuite(unittest.TestCase): ClientWrapper(mock_config_json), ) def setUpClass(cls): - cls.workload = Workload("simple-boot", gem5_version="develop") + cls.workload = obtain_resource("simple-boot", gem5_version="develop") def test_get_function_str(self) -> None: # Tests `Resource.get_function_str` From bbe96d6485e830c42311a53c7ee1c3f99b156851 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 6 Sep 2023 10:06:16 -0700 Subject: [PATCH 7/7] stdlib: Changed use of Workload to obtain_resource - Changed files calling Workload class to call obtain_resoucre instead. Change-Id: I41f5f0c3ccc7c08b39e7049eabef9609d6d68788 --- configs/example/gem5_library/arm-ubuntu-run.py | 4 ++-- configs/example/gem5_library/caches/octopi-cache-example.py | 4 ++-- .../gem5_library/looppoints/create-looppoint-checkpoints.py | 6 ++++-- .../gem5_library/looppoints/restore-looppoint-checkpoint.py | 3 +-- configs/example/gem5_library/riscv-ubuntu-run.py | 4 ++-- configs/example/gem5_library/riscvmatched-fs.py | 4 ++-- .../example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py | 4 ++-- configs/example/gem5_library/x86-ubuntu-run-with-kvm.py | 4 ++-- configs/example/gem5_library/x86-ubuntu-run.py | 4 ++-- tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py | 4 ++-- tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py | 4 ++-- 11 files changed, 23 insertions(+), 22 deletions(-) diff --git a/configs/example/gem5_library/arm-ubuntu-run.py b/configs/example/gem5_library/arm-ubuntu-run.py index 7f976f06db..78160c9976 100644 --- a/configs/example/gem5_library/arm-ubuntu-run.py +++ b/configs/example/gem5_library/arm-ubuntu-run.py @@ -43,7 +43,7 @@ scons build/ARM/gem5.opt -j from gem5.isas import ISA from m5.objects import ArmDefaultRelease from gem5.utils.requires import requires -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource from gem5.simulate.simulator import Simulator from m5.objects import VExpress_GEM5_Foundation from gem5.coherence_protocol import CoherenceProtocol @@ -100,7 +100,7 @@ board = ArmBoard( # Here we set a full system workload. The "arm64-ubuntu-20.04-boot" boots # Ubuntu 20.04. -board.set_workload(Workload("arm64-ubuntu-20.04-boot")) +board.set_workload(obtain_resource("arm64-ubuntu-20.04-boot")) # We define the system with the aforementioned system defined. diff --git a/configs/example/gem5_library/caches/octopi-cache-example.py b/configs/example/gem5_library/caches/octopi-cache-example.py index 1b39a8bee5..4a4926a174 100644 --- a/configs/example/gem5_library/caches/octopi-cache-example.py +++ b/configs/example/gem5_library/caches/octopi-cache-example.py @@ -51,7 +51,7 @@ from gem5.components.cachehierarchies.ruby.caches.mesi_three_level.octopi import from gem5.isas import ISA from gem5.coherence_protocol import CoherenceProtocol from gem5.simulate.simulator import Simulator -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource num_ccds = 1 # CCDs num_cores_per_ccd = 8 # 8 cores/CCD @@ -94,7 +94,7 @@ board = ArmBoard( platform=platform, ) -board.set_workload(Workload("arm64-ubuntu-20.04-boot")) +board.set_workload(obtain_resource("arm64-ubuntu-20.04-boot")) simulator = Simulator(board=board) simulator.run() diff --git a/configs/example/gem5_library/looppoints/create-looppoint-checkpoints.py b/configs/example/gem5_library/looppoints/create-looppoint-checkpoints.py index abb15fb7f8..4a15da55ff 100644 --- a/configs/example/gem5_library/looppoints/create-looppoint-checkpoints.py +++ b/configs/example/gem5_library/looppoints/create-looppoint-checkpoints.py @@ -56,7 +56,7 @@ from gem5.components.memory.single_channel import SingleChannelDDR3_1600 from gem5.components.processors.simple_processor import SimpleProcessor from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource from pathlib import Path from gem5.simulate.exit_event_generators import ( looppoint_save_checkpoint_generator, @@ -110,7 +110,9 @@ board = SimpleBoard( cache_hierarchy=cache_hierarchy, ) -board.set_workload(Workload("x86-matrix-multiply-omp-100-8-looppoint-csv")) +board.set_workload( + obtain_resource("x86-matrix-multiply-omp-100-8-looppoint-csv") +) dir = Path(args.checkpoint_path) dir.mkdir(exist_ok=True) diff --git a/configs/example/gem5_library/looppoints/restore-looppoint-checkpoint.py b/configs/example/gem5_library/looppoints/restore-looppoint-checkpoint.py index 21353a34a1..5224c6e6ea 100644 --- a/configs/example/gem5_library/looppoints/restore-looppoint-checkpoint.py +++ b/configs/example/gem5_library/looppoints/restore-looppoint-checkpoint.py @@ -54,7 +54,6 @@ from gem5.components.processors.simple_processor import SimpleProcessor from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA from gem5.resources.resource import obtain_resource -from gem5.resources.workload import Workload from m5.stats import reset, dump requires(isa_required=ISA.X86) @@ -113,7 +112,7 @@ board = SimpleBoard( ) board.set_workload( - Workload( + obtain_resource( f"x86-matrix-multiply-omp-100-8-looppoint-region-{args.checkpoint_region}" ) ) diff --git a/configs/example/gem5_library/riscv-ubuntu-run.py b/configs/example/gem5_library/riscv-ubuntu-run.py index 87b98cc5ba..9b172fd501 100644 --- a/configs/example/gem5_library/riscv-ubuntu-run.py +++ b/configs/example/gem5_library/riscv-ubuntu-run.py @@ -50,7 +50,7 @@ from gem5.components.processors.simple_processor import SimpleProcessor from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA from gem5.simulate.simulator import Simulator -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource # This runs a check to ensure the gem5 binary is compiled for RISCV. @@ -88,7 +88,7 @@ board = RiscvBoard( # Ubuntu 20.04. Once the system successfully boots it encounters an `m5_exit` # instruction which stops the simulation. When the simulation has ended you may # inspect `m5out/system.pc.com_1.device` to see the stdout. -board.set_workload(Workload("riscv-ubuntu-20.04-boot")) +board.set_workload(obtain_resource("riscv-ubuntu-20.04-boot")) simulator = Simulator(board=board) simulator.run() diff --git a/configs/example/gem5_library/riscvmatched-fs.py b/configs/example/gem5_library/riscvmatched-fs.py index 3e84b8c1ea..29ec76e16b 100644 --- a/configs/example/gem5_library/riscvmatched-fs.py +++ b/configs/example/gem5_library/riscvmatched-fs.py @@ -42,7 +42,7 @@ from gem5.prebuilt.riscvmatched.riscvmatched_board import RISCVMatchedBoard from gem5.utils.requires import requires from gem5.isas import ISA from gem5.simulate.simulator import Simulator -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource import argparse @@ -76,7 +76,7 @@ board = RISCVMatchedBoard( # In the case where the `-i` flag is passed, we add the kernel argument # `init=/root/exit.sh`. This means the simulation will exit after the Linux # Kernel has booted. -workload = Workload("riscv-ubuntu-20.04-boot") +workload = obtain_resource("riscv-ubuntu-20.04-boot") kernel_args = board.get_default_kernel_args() if args.to_init: kernel_args.append("init=/root/exit.sh") diff --git a/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py index 1c65357921..f25ffb62f8 100644 --- a/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py +++ b/configs/example/gem5_library/x86-ubuntu-run-with-kvm-no-perf.py @@ -52,7 +52,7 @@ from gem5.isas import ISA from gem5.coherence_protocol import CoherenceProtocol from gem5.simulate.simulator import Simulator from gem5.simulate.exit_event import ExitEvent -from gem5.resources.workload import Workload +from gem5.resources.resoruce import obtain_resource # This simulation requires using KVM with gem5 compiled for X86 simulation # and with MESI_Two_Level cache coherence protocol. @@ -121,7 +121,7 @@ command = ( + "m5 exit;" ) -workload = Workload("x86-ubuntu-18.04-boot") +workload = obtain_resource("x86-ubuntu-18.04-boot") workload.set_parameter("readfile_contents", command) board.set_workload(workload) diff --git a/configs/example/gem5_library/x86-ubuntu-run-with-kvm.py b/configs/example/gem5_library/x86-ubuntu-run-with-kvm.py index f55ec60f21..00c00d1459 100644 --- a/configs/example/gem5_library/x86-ubuntu-run-with-kvm.py +++ b/configs/example/gem5_library/x86-ubuntu-run-with-kvm.py @@ -51,7 +51,7 @@ from gem5.isas import ISA from gem5.coherence_protocol import CoherenceProtocol from gem5.simulate.simulator import Simulator from gem5.simulate.exit_event import ExitEvent -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource # This runs a check to ensure the gem5 binary is compiled to X86 and to the # MESI Two Level coherence protocol. @@ -117,7 +117,7 @@ command = ( + "m5 exit;" ) -workload = Workload("x86-ubuntu-18.04-boot") +workload = obtain_resource("x86-ubuntu-18.04-boot") workload.set_parameter("readfile_contents", command) board.set_workload(workload) diff --git a/configs/example/gem5_library/x86-ubuntu-run.py b/configs/example/gem5_library/x86-ubuntu-run.py index 50b52e6e3c..fe72d653f5 100644 --- a/configs/example/gem5_library/x86-ubuntu-run.py +++ b/configs/example/gem5_library/x86-ubuntu-run.py @@ -45,7 +45,7 @@ scons build/X86/gem5.opt """ from gem5.prebuilt.demo.x86_demo_board import X86DemoBoard -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource from gem5.simulate.simulator import Simulator @@ -56,7 +56,7 @@ board = X86DemoBoard() # We then set the workload. Here we use the "x86-ubuntu-18.04-boot" workload. # This boots Ubuntu 18.04 with Linux 5.4.49. If the required resources are not # found locally, they will be downloaded. -board.set_workload(Workload("x86-ubuntu-18.04-boot")) +board.set_workload(obtain_resource("x86-ubuntu-18.04-boot")) simulator = Simulator(board=board) simulator.run() diff --git a/tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py b/tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py index e9fc06b27b..3726d7de46 100644 --- a/tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py +++ b/tests/gem5/riscv_boot_tests/configs/riscv_boot_exit_run.py @@ -40,7 +40,7 @@ from gem5.components.processors.cpu_types import CPUTypes from gem5.components.boards.riscv_board import RiscvBoard from gem5.components.processors.simple_processor import SimpleProcessor from gem5.simulate.simulator import Simulator -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource import argparse import importlib @@ -160,7 +160,7 @@ board = RiscvBoard( ) # Set the workload. -workload = Workload( +workload = obtain_resource( "riscv-ubuntu-20.04-boot", resource_directory=args.resource_directory ) board.set_workload(workload) diff --git a/tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py b/tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py index e9eeacefd8..63b6625479 100644 --- a/tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py +++ b/tests/gem5/x86_boot_tests/configs/x86_boot_exit_run.py @@ -41,7 +41,7 @@ from gem5.components.processors.cpu_types import ( ) from gem5.components.processors.simple_processor import SimpleProcessor from gem5.simulate.simulator import Simulator -from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource import argparse import importlib @@ -184,7 +184,7 @@ if args.boot_type == "init": kernal_args.append("init=/root/exit.sh") # Set the workload. -workload = Workload( +workload = obtain_resource( "x86-ubuntu-18.04-boot", resource_directory=args.resource_directory ) workload.set_parameter("kernel_args", kernal_args)