derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stdlib, resources, tests: Introduce Suite of Workloads

Browse Source 
This patch introduces a new category called "suite".
A suite is a collection of workloads.
Each workload in a SuiteResource has a tag that can be narrowed down
through the function with_input_group.
Also, the set of input groups can be seen through list_input_groups.
Added unit tests to test all functions of SuiteResource class.

Change-Id: Iddda5c898b32b7cd874987dbe694ac09aa231f08

Co-authored-by: Kunal Pai <kunpai@ucdavis.edu>
This commit is contained in:
Harshil Patel
2023-08-16 15:14:19 -07:00
parent 62d34ef374
commit 8182f8084b
4 changed files with 492 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
configs/example/gem5_library/riscvmatched-microbenchmark-suite.py Normal file
View File

@@ -0,0 +1,86 @@
# Copyright (c) 2023 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
This script shows how to use a suite. In this example, we will use the
RISCVMatchedBoard and the RISCV Vertical Microbenchmark Suite,
and show the different functionalities of the suite.
The print statements in the script are for illustrative purposes only,
and are not required to run the script.
"""
from gem5.resources.resource import obtain_resource
from gem5.simulate.simulator import Simulator
from gem5.prebuilt.riscvmatched.riscvmatched_board import RISCVMatchedBoard
from gem5.isas import ISA
from gem5.utils.requires import requires
requires(isa_required=ISA.RISCV)
# instantiate the riscv matched board with default parameters
board = RISCVMatchedBoard()
# obtain the RISC-V Vertical Microbenchmarks
microbenchmarks = obtain_resource("riscv-vertical-microbenchmarks")
# list all the microbenchmarks present in the suite
print("Microbenchmarks present in the suite:")
print("====================================")
for (
id,
resource_version,
input_group,
workload,
) in microbenchmarks.get_all_workloads():
print(f"Workload ID: {id}")
print(f"Workload Version: {resource_version}")
print(f"Workload Input Groups: {input_group}")
print(f"WorkloadResource Object: {workload}")
print("====================================")
# list all the WorkloadResource objects present in the suite
for resource in microbenchmarks:
print(f"WorkloadResource Object: {resource}")
# list all the available input groups in the suite
print("Input groups present in the suite:")
print(microbenchmarks.get_input_groups())
# for this example, we will filter the suite
# to run the Workload "riscv-cca-run"
# it has the input group 'cca', which is used as the filter
board.set_workload(list(microbenchmarks.with_input_group("cca"))[0])
# run the simulation with the RISCV Matched board
simulator = Simulator(board=board, full_system=False)
simulator.run()
print(
"Exiting @ tick {} because {}.".format(
simulator.get_current_tick(),
simulator.get_last_exit_event_cause(),
)
)

140
src/python/gem5/resources/resource.py
View File

@@ -25,6 +25,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from abc import ABCMeta
import json
import os
from pathlib import Path
from m5.util import warn, fatal
@@ -35,7 +36,17 @@ from .downloader import get_resource
from .looppoint import LooppointCsvLoader, LooppointJsonLoader
from ..isas import ISA, get_isa_from_str
from typing import Optional, Dict, Union, Type, Tuple, List, Any
from typing import (
Optional,
Dict,
Union,
Type,
Tuple,
List,
Any,
Set,
Generator,
)
from .client import get_resource_json_obj
@@ -627,6 +638,111 @@ class SimpointDirectoryResource(SimpointResource):
return "SimpointDirectoryResource"
class SuiteResource(AbstractResource):
"""
A suite resource. This resource is used to specify a suite of workloads to
run on a board. It contains a list of workloads to run, along with their
IDs and versions.
Each workload in a suite is used to create a `WorkloadResource` object.
These objects are stored in a list and can be iterated over.
"""
def __init__(
self,
workloads: Dict["WorkloadResource", Set[str]] = {},
resource_version: Optional[str] = None,
description: Optional[str] = None,
source: Optional[str] = None,
id: Optional[str] = None,
**kwargs,
) -> None:
"""
:param workloads: A list of `WorkloadResource` objects
created from the `_workloads` parameter.
:param local_path: The path on the host system where this resource is
located.
:param description: Description describing this resource. Not a
required parameter. By default is None.
:param source: The source (as in "source code") for this resource
on gem5-resources. Not a required parameter. By default is None.
:param resource_version: Version of the resource itself.
"""
self._workloads = workloads
self._description = description
self._source = source
self._resource_version = resource_version
super().__init__(
id=id,
description=description,
source=source,
resource_version=resource_version,
)
def __iter__(self) -> Generator["WorkloadResource", None, None]:
"""
Returns a generator that iterates over the workloads in the suite.
:yields: A generator that iterates over the workloads in the suite.
"""
for workload in self._workloads.keys():
yield workload
def __len__(self):
"""
Returns the number of workloads in the suite.
:returns: The number of workloads in the suite.
"""
return len(self._workloads)
def get_category_name(cls) -> str:
return "SuiteResource"
def with_input_group(self, input_group: str) -> "SuiteResource":
"""
Returns a new SuiteResource object with only the workloads that use the
specified input group.
:param input_group: The input group to filter the workloads by.
:returns: A new SuiteResource object with only the workloads that use
the specified input group.
"""
if input_group not in self.get_input_groups():
raise Exception(
f"Input group {input_group} not found in Suite.\n"
f"Available input groups are {self.get_input_groups()}"
)
filtered_workloads = {}
for workload, input_groups in self._workloads.items():
if input_group in input_groups:
filtered_workloads[workload] = input_groups
return SuiteResource(
local_path=self._local_path,
resource_version=self._resource_version,
description=self._description,
source=self._source,
workloads=filtered_workloads,
)
def get_input_groups(self) -> Set[str]:
"""
Returns a set of all input groups used by the workloads in a suite.
:returns: A set of all input groups used by the workloads in a suite.
"""
return {
input_group
for input_groups in self._workloads.values()
for input_group in input_groups
}
class WorkloadResource(AbstractResource):
"""A workload resource. This resource is used to specify a workload to run
on a board. It contains the function to call and the parameters to pass to
@@ -657,9 +773,14 @@ class WorkloadResource(AbstractResource):
resource_version=resource_version,
)
self._id = id
self._func = function
self._params = parameters
def get_id(self) -> str:
"""Returns the ID of the workload."""
return self._id
def get_function_str(self) -> str:
"""
Returns the name of the workload function to be run.
@@ -821,6 +942,21 @@ def obtain_resource(
assert resources_category in _get_resource_json_type_map
resource_class = _get_resource_json_type_map[resources_category]
if resources_category == "suite":
workloads = resource_json["workloads"]
workloads_obj = {}
for workload in workloads:
workloads_obj[
obtain_resource(
workload["id"],
resource_version=workload["resource_version"],
resource_directory=resource_directory,
clients=clients,
gem5_version=gem5_version,
)
] = set(workload["input_group"])
resource_json["workloads"] = workloads_obj
if resources_category == "workload":
# This parses the "resources" and "additional_params" fields of the
# workload resource into a dictionary of AbstractResource objects and
@@ -844,6 +980,7 @@ def obtain_resource(
# Once we know what AbstractResource subclass we are using, we create it.
# The fields in the JSON object are assumed to map like-for-like to the
# subclass contructor, so we can pass the resource_json map directly.
return resource_class(local_path=to_path, **resource_json)
@@ -995,5 +1132,6 @@ _get_resource_json_type_map = {
"resource": Resource,
"looppoint-pinpoint-csv": LooppointCsvResource,
"looppoint-json": LooppointJsonResource,
"suite": SuiteResource,
"workload": WorkloadResource,
}

170
tests/pyunit/stdlib/resources/pyunit_suite_checks.py Normal file
View File

@@ -0,0 +1,170 @@
# Copyright (c) 2023 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import contextlib
import io
import unittest
import tempfile
import os
import shutil
from pathlib import Path
from gem5.resources.resource import (
obtain_resource,
SuiteResource,
WorkloadResource,
)
from gem5.resources.client_api.client_wrapper import ClientWrapper
from unittest.mock import patch
mock_config_json = {
"sources": {
"baba": {
"url": Path(__file__).parent / "refs/suite-checks.json",
"isMongo": False,
}
},
}
class CustomSuiteResourceTestSuite(unittest.TestCase):
@classmethod
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def setUpClass(cls):
cls.workload1 = obtain_resource("simple-workload-1")
cls.workload2 = obtain_resource("simple-workload-2")
cls.SuiteResource = SuiteResource(
workload_obj_list={cls.workload1: set(), cls.workload2: set()}
)
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_with_input_group(self) -> None:
"""
Tests the `with_input_group` function.
"""
# test if an input group can return a single workload in a suite resource
with self.assertRaises(Exception) as context:
filtered_suite = self.SuiteResource.with_input_group("testtag2")
self.assertIsInstance(filtered_suite, SuiteResource)
self.assertEqual(len(filtered_suite), 0)
self.assertTrue(
f"Input group invalid not found in Suite.\n"
f"Available input groups are {filtered_suite.get_input_groups()}"
in str(context.exception)
)
def test_get_input_groups(self):
"""
Tests the `list_input_groups` function.
"""
self.assertEqual(self.SuiteResource.get_input_groups(), set())
class SuiteResourceTestSuite(unittest.TestCase):
@classmethod
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def setUpClass(cls):
cls.suite = obtain_resource("suite-example", gem5_version="develop")
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_with_input_group(self) -> None:
"""
Tests the `with_input_group` function.
"""
# test if an input group can return a single workload in a suite resource
filtered_suite = self.suite.with_input_group("testtag2")
self.assertIsInstance(filtered_suite, SuiteResource)
self.assertEqual(len(filtered_suite), 1)
for workload in filtered_suite:
self.assertIsInstance(workload, WorkloadResource)
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_with_input_group_multiple(self) -> None:
# test if an input group can return multiple workloads in a suite resource
filtered_suite = self.suite.with_input_group("testtag1")
self.assertIsInstance(filtered_suite, SuiteResource)
self.assertEqual(len(filtered_suite), 2)
for workload in filtered_suite:
self.assertIsInstance(workload, WorkloadResource)
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_with_input_group_invalid(self) -> None:
"""
Tests the `with_input_group` function with an invalid input group.
"""
with self.assertRaises(Exception) as context:
filtered_suite = self.suite.with_input_group("invalid")
# check if exception is raised
self.assertTrue(
f"Input group invalid not found in Suite.\n"
f"Available input groups are {filtered_suite.get_input_groups()}"
in str(context.exception)
)
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_get_input_groups(self) -> None:
"""
Tests the `list_input_groups` function.
"""
expected_input_groups = set(["testtag1", "testtag2", "testtag3"])
self.assertEqual(self.suite.get_input_groups(), expected_input_groups)
@patch(
"gem5.resources.client.clientwrapper",
new=ClientWrapper(mock_config_json),
)
def test_get_input_groups_not_found(self) -> None:
"""
Tests the `list_input_groups` function with an invalid input group.
"""
with self.assertRaises(Exception) as context:
self.suite.get_input_groups("invalid")
self.assertTrue(
f"Input group invalid not found in Suite.\n"
f"Available input groups are {self.suite.get_input_groups()}"
in str(context.exception)
)

97
tests/pyunit/stdlib/resources/refs/suite-checks.json Normal file
View File

@@ -0,0 +1,97 @@
[
{
"id": "suite-example",
"category": "suite",
"resource_version": "1.0.0",
"gem5_versions": ["develop","23.1"],
"workloads": [
{
"id": "simple-workload-1",
"resource_version": "1.0.0",
"input_group": ["testtag1", "testtag2"]
},
{
"id": "simple-workload-2",
"resource_version": "1.0.0",
"input_group": ["testtag1", "testtag3"]
}
]
},
{
"category": "workload",
"id": "simple-workload-1",
"description": "Description of workload here",
"function": "set_kernel_disk_workload",
"resources": {
"kernel": "x86-linux-kernel-5.2.3-example",
"disk-image": "x86-ubuntu-18.04-img-example"
},
"additional_params": {
"readfile_contents": "echo 'Boot successful'; m5 exit"
},
"resource_version": "1.0.0",
"gem5_versions": [
"develop"
]
},
{
"category": "workload",
"id": "simple-workload-2",
"description": "Description of workload here",
"function": "set_kernel_disk_workload",
"resources": {
"kernel": "x86-linux-kernel-5.2.3-example",
"disk-image": "x86-ubuntu-18.04-img-example"
},
"additional_params": {
"readfile_contents": "echo 'Boot successful'; m5 exit"
},
"resource_version": "1.0.0",
"gem5_versions": [
"develop"
]
},
{
"category": "kernel",
"id": "x86-linux-kernel-5.2.3-example",
"description": "The linux kernel (v5.2.3), compiled to X86.",
"architecture": "X86",
"is_zipped": false,
"md5sum": "4838c99b77d33c8307b939c16624e4ac",
"url": "http://dist.gem5.org/dist/develop/kernels/x86/static/vmlinux-5.2.3",
"source": "src/linux-kernel",
"resource_version": "1.0.0",
"gem5_versions": [
"develop"
]
},
{
"category": "disk-image",
"id": "x86-ubuntu-18.04-img-example",
"description": "A disk image containing Ubuntu 18.04 for x86..",
"architecture": "X86",
"is_zipped": false,
"md5sum": "dbf120338b37153e3334603970cebd8c",
"url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static",
"source": "src/x86-ubuntu",
"root_partition": "1",
"resource_version": "1.0.0",
"gem5_versions": [
"develop"
]
},
{
"category": "binary",
"id": "x86-hello64-static-example",
"description": "A 'Hello World!' binary.",
"architecture": "X86",
"is_zipped": false,
"md5sum": "dbf120338b37153e3334603970cebd8c",
"url": "http://dist.gem5.org/dist/develop/test-progs/hello/bin/x86/linux/hello64-static",
"source": "src/simple",
"resource_version": "1.0.0",
"gem5_versions": [
"develop"
]
}
]