From 8182f8084b0c0b33f7148930e7d37352bbef88ff Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 16 Aug 2023 15:14:19 -0700 Subject: [PATCH 1/2] stdlib, resources, tests: Introduce Suite of Workloads This patch introduces a new category called "suite". A suite is a collection of workloads. Each workload in a SuiteResource has a tag that can be narrowed down through the function with_input_group. Also, the set of input groups can be seen through list_input_groups. Added unit tests to test all functions of SuiteResource class. Change-Id: Iddda5c898b32b7cd874987dbe694ac09aa231f08 Co-authored-by: Kunal Pai --- .../riscvmatched-microbenchmark-suite.py | 86 +++++++++ src/python/gem5/resources/resource.py | 140 ++++++++++++++- .../stdlib/resources/pyunit_suite_checks.py | 170 ++++++++++++++++++ .../stdlib/resources/refs/suite-checks.json | 97 ++++++++++ 4 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 configs/example/gem5_library/riscvmatched-microbenchmark-suite.py create mode 100644 tests/pyunit/stdlib/resources/pyunit_suite_checks.py create mode 100644 tests/pyunit/stdlib/resources/refs/suite-checks.json diff --git a/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py b/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py new file mode 100644 index 0000000000..2d189717e2 --- /dev/null +++ b/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +This script shows how to use a suite. In this example, we will use the +RISCVMatchedBoard and the RISCV Vertical Microbenchmark Suite, +and show the different functionalities of the suite. + +The print statements in the script are for illustrative purposes only, +and are not required to run the script. +""" + +from gem5.resources.resource import obtain_resource +from gem5.simulate.simulator import Simulator +from gem5.prebuilt.riscvmatched.riscvmatched_board import RISCVMatchedBoard +from gem5.isas import ISA +from gem5.utils.requires import requires + +requires(isa_required=ISA.RISCV) + +# instantiate the riscv matched board with default parameters +board = RISCVMatchedBoard() + +# obtain the RISC-V Vertical Microbenchmarks +microbenchmarks = obtain_resource("riscv-vertical-microbenchmarks") + +# list all the microbenchmarks present in the suite +print("Microbenchmarks present in the suite:") +print("====================================") +for ( + id, + resource_version, + input_group, + workload, +) in microbenchmarks.get_all_workloads(): + print(f"Workload ID: {id}") + print(f"Workload Version: {resource_version}") + print(f"Workload Input Groups: {input_group}") + print(f"WorkloadResource Object: {workload}") + print("====================================") + +# list all the WorkloadResource objects present in the suite +for resource in microbenchmarks: + print(f"WorkloadResource Object: {resource}") + +# list all the available input groups in the suite +print("Input groups present in the suite:") +print(microbenchmarks.get_input_groups()) + +# for this example, we will filter the suite +# to run the Workload "riscv-cca-run" +# it has the input group 'cca', which is used as the filter +board.set_workload(list(microbenchmarks.with_input_group("cca"))[0]) + +# run the simulation with the RISCV Matched board +simulator = Simulator(board=board, full_system=False) +simulator.run() +print( + "Exiting @ tick {} because {}.".format( + simulator.get_current_tick(), + simulator.get_last_exit_event_cause(), + ) +) diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 4b4e3c0b79..ac7769b83f 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import ABCMeta +import json import os from pathlib import Path from m5.util import warn, fatal @@ -35,7 +36,17 @@ from .downloader import get_resource from .looppoint import LooppointCsvLoader, LooppointJsonLoader from ..isas import ISA, get_isa_from_str -from typing import Optional, Dict, Union, Type, Tuple, List, Any +from typing import ( + Optional, + Dict, + Union, + Type, + Tuple, + List, + Any, + Set, + Generator, +) from .client import get_resource_json_obj @@ -627,6 +638,111 @@ class SimpointDirectoryResource(SimpointResource): return "SimpointDirectoryResource" +class SuiteResource(AbstractResource): + """ + A suite resource. This resource is used to specify a suite of workloads to + run on a board. It contains a list of workloads to run, along with their + IDs and versions. + + Each workload in a suite is used to create a `WorkloadResource` object. + These objects are stored in a list and can be iterated over. + """ + + def __init__( + self, + workloads: Dict["WorkloadResource", Set[str]] = {}, + resource_version: Optional[str] = None, + description: Optional[str] = None, + source: Optional[str] = None, + id: Optional[str] = None, + **kwargs, + ) -> None: + """ + :param workloads: A list of `WorkloadResource` objects + created from the `_workloads` parameter. + :param local_path: The path on the host system where this resource is + located. + :param description: Description describing this resource. Not a + required parameter. By default is None. + :param source: The source (as in "source code") for this resource + on gem5-resources. Not a required parameter. By default is None. + :param resource_version: Version of the resource itself. + """ + self._workloads = workloads + self._description = description + self._source = source + self._resource_version = resource_version + + super().__init__( + id=id, + description=description, + source=source, + resource_version=resource_version, + ) + + def __iter__(self) -> Generator["WorkloadResource", None, None]: + """ + Returns a generator that iterates over the workloads in the suite. + + :yields: A generator that iterates over the workloads in the suite. + """ + for workload in self._workloads.keys(): + yield workload + + def __len__(self): + """ + Returns the number of workloads in the suite. + + :returns: The number of workloads in the suite. + """ + return len(self._workloads) + + def get_category_name(cls) -> str: + return "SuiteResource" + + def with_input_group(self, input_group: str) -> "SuiteResource": + """ + Returns a new SuiteResource object with only the workloads that use the + specified input group. + + :param input_group: The input group to filter the workloads by. + :returns: A new SuiteResource object with only the workloads that use + the specified input group. + """ + + if input_group not in self.get_input_groups(): + raise Exception( + f"Input group {input_group} not found in Suite.\n" + f"Available input groups are {self.get_input_groups()}" + ) + + filtered_workloads = {} + + for workload, input_groups in self._workloads.items(): + if input_group in input_groups: + filtered_workloads[workload] = input_groups + + return SuiteResource( + local_path=self._local_path, + resource_version=self._resource_version, + description=self._description, + source=self._source, + workloads=filtered_workloads, + ) + + def get_input_groups(self) -> Set[str]: + """ + Returns a set of all input groups used by the workloads in a suite. + + :returns: A set of all input groups used by the workloads in a suite. + """ + return { + input_group + for input_groups in self._workloads.values() + for input_group in input_groups + } + + class WorkloadResource(AbstractResource): """A workload resource. This resource is used to specify a workload to run on a board. It contains the function to call and the parameters to pass to @@ -657,9 +773,14 @@ class WorkloadResource(AbstractResource): resource_version=resource_version, ) + self._id = id self._func = function self._params = parameters + def get_id(self) -> str: + """Returns the ID of the workload.""" + return self._id + def get_function_str(self) -> str: """ Returns the name of the workload function to be run. @@ -821,6 +942,21 @@ def obtain_resource( assert resources_category in _get_resource_json_type_map resource_class = _get_resource_json_type_map[resources_category] + if resources_category == "suite": + workloads = resource_json["workloads"] + workloads_obj = {} + for workload in workloads: + workloads_obj[ + obtain_resource( + workload["id"], + resource_version=workload["resource_version"], + resource_directory=resource_directory, + clients=clients, + gem5_version=gem5_version, + ) + ] = set(workload["input_group"]) + resource_json["workloads"] = workloads_obj + if resources_category == "workload": # This parses the "resources" and "additional_params" fields of the # workload resource into a dictionary of AbstractResource objects and @@ -844,6 +980,7 @@ def obtain_resource( # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the # subclass contructor, so we can pass the resource_json map directly. + return resource_class(local_path=to_path, **resource_json) @@ -995,5 +1132,6 @@ _get_resource_json_type_map = { "resource": Resource, "looppoint-pinpoint-csv": LooppointCsvResource, "looppoint-json": LooppointJsonResource, + "suite": SuiteResource, "workload": WorkloadResource, } diff --git a/tests/pyunit/stdlib/resources/pyunit_suite_checks.py b/tests/pyunit/stdlib/resources/pyunit_suite_checks.py new file mode 100644 index 0000000000..5de68a6b1a --- /dev/null +++ b/tests/pyunit/stdlib/resources/pyunit_suite_checks.py @@ -0,0 +1,170 @@ +# Copyright (c) 2023 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import contextlib +import io +import unittest +import tempfile +import os +import shutil +from pathlib import Path +from gem5.resources.resource import ( + obtain_resource, + SuiteResource, + WorkloadResource, +) +from gem5.resources.client_api.client_wrapper import ClientWrapper +from unittest.mock import patch + +mock_config_json = { + "sources": { + "baba": { + "url": Path(__file__).parent / "refs/suite-checks.json", + "isMongo": False, + } + }, +} + + +class CustomSuiteResourceTestSuite(unittest.TestCase): + @classmethod + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def setUpClass(cls): + cls.workload1 = obtain_resource("simple-workload-1") + cls.workload2 = obtain_resource("simple-workload-2") + cls.SuiteResource = SuiteResource( + workload_obj_list={cls.workload1: set(), cls.workload2: set()} + ) + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_with_input_group(self) -> None: + """ + Tests the `with_input_group` function. + """ + # test if an input group can return a single workload in a suite resource + + with self.assertRaises(Exception) as context: + filtered_suite = self.SuiteResource.with_input_group("testtag2") + self.assertIsInstance(filtered_suite, SuiteResource) + self.assertEqual(len(filtered_suite), 0) + self.assertTrue( + f"Input group invalid not found in Suite.\n" + f"Available input groups are {filtered_suite.get_input_groups()}" + in str(context.exception) + ) + + def test_get_input_groups(self): + """ + Tests the `list_input_groups` function. + """ + self.assertEqual(self.SuiteResource.get_input_groups(), set()) + + +class SuiteResourceTestSuite(unittest.TestCase): + @classmethod + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def setUpClass(cls): + cls.suite = obtain_resource("suite-example", gem5_version="develop") + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_with_input_group(self) -> None: + """ + Tests the `with_input_group` function. + """ + # test if an input group can return a single workload in a suite resource + filtered_suite = self.suite.with_input_group("testtag2") + self.assertIsInstance(filtered_suite, SuiteResource) + self.assertEqual(len(filtered_suite), 1) + for workload in filtered_suite: + self.assertIsInstance(workload, WorkloadResource) + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_with_input_group_multiple(self) -> None: + # test if an input group can return multiple workloads in a suite resource + filtered_suite = self.suite.with_input_group("testtag1") + self.assertIsInstance(filtered_suite, SuiteResource) + self.assertEqual(len(filtered_suite), 2) + for workload in filtered_suite: + self.assertIsInstance(workload, WorkloadResource) + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_with_input_group_invalid(self) -> None: + """ + Tests the `with_input_group` function with an invalid input group. + """ + with self.assertRaises(Exception) as context: + filtered_suite = self.suite.with_input_group("invalid") + # check if exception is raised + self.assertTrue( + f"Input group invalid not found in Suite.\n" + f"Available input groups are {filtered_suite.get_input_groups()}" + in str(context.exception) + ) + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_get_input_groups(self) -> None: + """ + Tests the `list_input_groups` function. + """ + expected_input_groups = set(["testtag1", "testtag2", "testtag3"]) + self.assertEqual(self.suite.get_input_groups(), expected_input_groups) + + @patch( + "gem5.resources.client.clientwrapper", + new=ClientWrapper(mock_config_json), + ) + def test_get_input_groups_not_found(self) -> None: + """ + Tests the `list_input_groups` function with an invalid input group. + """ + with self.assertRaises(Exception) as context: + self.suite.get_input_groups("invalid") + self.assertTrue( + f"Input group invalid not found in Suite.\n" + f"Available input groups are {self.suite.get_input_groups()}" + in str(context.exception) + ) diff --git a/tests/pyunit/stdlib/resources/refs/suite-checks.json b/tests/pyunit/stdlib/resources/refs/suite-checks.json new file mode 100644 index 0000000000..7583020292 --- /dev/null +++ b/tests/pyunit/stdlib/resources/refs/suite-checks.json @@ -0,0 +1,97 @@ +[ + { + "id": "suite-example", + "category": "suite", + "resource_version": "1.0.0", + "gem5_versions": ["develop","23.1"], + "workloads": [ + { + "id": "simple-workload-1", + "resource_version": "1.0.0", + "input_group": ["testtag1", "testtag2"] + }, + { + "id": "simple-workload-2", + "resource_version": "1.0.0", + "input_group": ["testtag1", "testtag3"] + } + ] + }, + { + "category": "workload", + "id": "simple-workload-1", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3-example", + "disk-image": "x86-ubuntu-18.04-img-example" + }, + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" + }, + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "workload", + "id": "simple-workload-2", + "description": "Description of workload here", + "function": "set_kernel_disk_workload", + "resources": { + "kernel": "x86-linux-kernel-5.2.3-example", + "disk-image": "x86-ubuntu-18.04-img-example" + }, + "additional_params": { + "readfile_contents": "echo 'Boot successful'; m5 exit" + }, + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "kernel", + "id": "x86-linux-kernel-5.2.3-example", + "description": "The linux kernel (v5.2.3), compiled to X86.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "4838c99b77d33c8307b939c16624e4ac", + "url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.2.3", + "source": "src/linux-kernel", + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "disk-image", + "id": "x86-ubuntu-18.04-img-example", + "description": "A disk image containing Ubuntu 18.04 for x86..", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/x86-ubuntu", + "root_partition": "1", + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" + ] + }, + { + "category": "binary", + "id": "x86-hello64-static-example", + "description": "A 'Hello World!' binary.", + "architecture": "X86", + "is_zipped": false, + "md5sum": "dbf120338b37153e3334603970cebd8c", + "url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static", + "source": "src/simple", + "resource_version": "1.0.0", + "gem5_versions": [ + "develop" + ] + } +] From 3af3c1121b2bfead7d9a6a3893fe27cedb93238a Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 2 Oct 2023 23:27:32 -0700 Subject: [PATCH 2/2] stdlib, resources: Addressed requested changes Change-Id: I22abdc3bdcdde52301ed10cb3113e8925159c245 Co-authored-by: Kunal Pai --- .../riscvmatched-microbenchmark-suite.py | 12 +++--------- src/python/gem5/resources/resource.py | 1 - .../test_gem5_library_examples.py | 17 +++++++++++++++++ .../stdlib/resources/pyunit_suite_checks.py | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py b/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py index 2d189717e2..7e08355e31 100644 --- a/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py +++ b/configs/example/gem5_library/riscvmatched-microbenchmark-suite.py @@ -50,15 +50,9 @@ microbenchmarks = obtain_resource("riscv-vertical-microbenchmarks") # list all the microbenchmarks present in the suite print("Microbenchmarks present in the suite:") print("====================================") -for ( - id, - resource_version, - input_group, - workload, -) in microbenchmarks.get_all_workloads(): - print(f"Workload ID: {id}") - print(f"Workload Version: {resource_version}") - print(f"Workload Input Groups: {input_group}") +for workload in microbenchmarks: + print(f"Workload ID: {workload.get_id()}") + print(f"Workload Version: {workload.get_resource_version()}") print(f"WorkloadResource Object: {workload}") print("====================================") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index ac7769b83f..af8a8d9acd 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -980,7 +980,6 @@ def obtain_resource( # Once we know what AbstractResource subclass we are using, we create it. # The fields in the JSON object are assumed to map like-for-like to the # subclass contructor, so we can pass the resource_json map directly. - return resource_class(local_path=to_path, **resource_json) diff --git a/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py b/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py index 0882fb0dfd..1967560d99 100644 --- a/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py +++ b/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py @@ -334,6 +334,23 @@ gem5_verify_config( length=constants.very_long_tag, ) +gem5_verify_config( + name="test-gem5-library-example-riscvmatched-microbenchmark-suite", + fixtures=(), + verifiers=(), + config=joinpath( + config.base_dir, + "configs", + "example", + "gem5_library", + "riscvmatched-microbenchmark-suite.py", + ), + config_args=[], + valid_isas=(constants.all_compiled_tag,), + valid_hosts=constants.supported_hosts, + length=constants.long_tag, +) + # The LoopPoint-Checkpointing feature is still under development, therefore # these tests are temporarily disabled until this feature is complete.# diff --git a/tests/pyunit/stdlib/resources/pyunit_suite_checks.py b/tests/pyunit/stdlib/resources/pyunit_suite_checks.py index 5de68a6b1a..480ba9f024 100644 --- a/tests/pyunit/stdlib/resources/pyunit_suite_checks.py +++ b/tests/pyunit/stdlib/resources/pyunit_suite_checks.py @@ -59,7 +59,7 @@ class CustomSuiteResourceTestSuite(unittest.TestCase): cls.workload1 = obtain_resource("simple-workload-1") cls.workload2 = obtain_resource("simple-workload-2") cls.SuiteResource = SuiteResource( - workload_obj_list={cls.workload1: set(), cls.workload2: set()} + workloads={cls.workload1: set(), cls.workload2: set()} ) @patch(