derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stdlib: Implement Simpoint Resources

Browse Source 
This patches does the following:
- Adds 'SimpointResource' which encapsulates Simpoint data and
  functionality. It replaces the old 'gem5.util.simpoint.SimPoint'
  class. Simpoints can be loaded from gem5-resources using the
  `obtain_resource` function.
- Adds 'SimpointDirectoryResource'. This inherits form
  'SimpointResource'. While 'SimpointResource' takes raw Simpoint data
  via parameters, 'SimpointDirectoryResource' assumes the data exists
  in files, in a directory.
- Updates the
  "configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py"
  and
  "configs/example/gem5_library/checkpoints/simpoints-se-restory.py"
  example files to utilize this new Simpoint resource classes.

**Note**: While the old "SimPoint" class
("src/python/gem5/util/simpoint.py") is marked as deprecated, it may be
difficult to utilize given updates to the APIs in the gem5 stdlib Cores
and Simulator modules.

Change-Id: I9bed5c643ffc735838c9f22a58c53547941010e7
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/67339
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu>
Maintainer: Bobby Bruce <bbruce@ucdavis.edu>
This commit is contained in:
Bobby R. Bruce
2023-01-20 13:40:22 +00:00
committed by Zhantong Qiu
parent cc838d72a6
commit e1601954f0
9 changed files with 354 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py
View File

@@ -58,6 +58,7 @@ from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.components.processors.cpu_types import CPUTypes
from gem5.isas import ISA
from gem5.resources.workload import Workload
from gem5.resources.resource import obtain_resource, SimpointResource
from pathlib import Path
from gem5.components.cachehierarchies.classic.no_cache import NoCache
from gem5.simulate.exit_event_generators import (
@@ -108,7 +109,23 @@ board = SimpleBoard(
cache_hierarchy=cache_hierarchy,
)
board.set_workload(Workload("x86-print-this-15000-with-simpoints"))
# board.set_workload(
# Workload("x86-print-this-15000-with-simpoints")
#
# **Note: This has been removed until we update the resources.json file to
# encapsulate the new Simpoint format.
# Below we set the simpount manually.
board.set_se_simpoint_workload(
binary=obtain_resource("x86-print-this"),
arguments=["print this", 15000],
simpoint=SimpointResource(
simpoint_interval=1000000,
simpoint_list=[2, 3, 4, 15],
weight_list=[0.1, 0.2, 0.4, 0.3],
warmup_interval=1000000,
),
)
dir = Path(args.checkpoint_path)
dir.mkdir(exist_ok=True)

27
configs/example/gem5_library/checkpoints/simpoints-se-restore.py
View File

@@ -63,8 +63,9 @@ from gem5.components.memory import DualChannelDDR4_2400
from gem5.components.processors.simple_processor import SimpleProcessor
from gem5.components.processors.cpu_types import CPUTypes
from gem5.isas import ISA
from gem5.resources.resource import Resource
from gem5.resources.resource import SimpointResource, obtain_resource
from gem5.resources.workload import Workload
from gem5.resources.resource import SimpointResource
from pathlib import Path
from m5.stats import reset, dump
@@ -96,11 +97,29 @@ board = SimpleBoard(
cache_hierarchy=cache_hierarchy,
)
# Here we obtain the workloadfrom gem5 resources, the checkpoint in this
# Here we obtain the workload from gem5 resources, the checkpoint in this
# workload was generated from
# `configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py`.
board.set_workload(
Workload("x86-print-this-15000-with-simpoints-and-checkpoint")
# board.set_workload(
# Workload("x86-print-this-15000-with-simpoints-and-checkpoint")
#
# **Note: This has been removed until we update the resources.json file to
# encapsulate the new Simpoint format.
# Below we set the simpount manually.
#
# This loads a single checkpoint as an example of using simpoints to simulate
# the function of a single simpoint region.
board.set_se_simpoint_workload(
binary=obtain_resource("x86-print-this"),
arguments=["print this", 15000],
simpoint=SimpointResource(
simpoint_interval=1000000,
simpoint_list=[2, 3, 4, 15],
weight_list=[0.1, 0.2, 0.4, 0.3],
warmup_interval=1000000,
),
checkpoint=obtain_resource("simpoints-se-checkpoints-v22-1-v2"),
)

27
src/python/gem5/components/boards/se_binary_workload.py
View File

@@ -31,8 +31,8 @@ from ...resources.resource import (
BinaryResource,
CheckpointResource,
SimpointResource,
SimpointDirectoryResource,
)
from gem5.utils.simpoint import SimPoint
from m5.objects import SEWorkload, Process
@@ -125,7 +125,7 @@ class SEBinaryWorkload:
self,
binary: BinaryResource,
arguments: List[str] = [],
simpoint: Union[SimpointResource, SimPoint] = None,
simpoint: SimpointResource = None,
checkpoint: Optional[Union[Path, CheckpointResource]] = None,
) -> None:
"""Set up the system to run a SimPoint workload.
@@ -135,28 +135,23 @@ class SEBinaryWorkload:
* Dynamically linked executables are partially supported when the host
ISA and the simulated ISA are the same.
**Warning:** SimPoints only works with one core
**Warning:** Simpoints only works with one core
:param binary: The resource encapsulating the binary to be run.
:param arguments: The input arguments for the binary
:param simpoint: The SimPoint object or Resource that contains the list of
:param simpoint: The SimpointResource that contains the list of
SimPoints starting instructions, the list of weights, and the SimPoints
interval
:param checkpoint: The checkpoint directory. Used to restore the
simulation to that checkpoint.
"""
# convert input to SimPoint if necessary
if isinstance(simpoint, SimpointResource):
self._simpoint_object = SimPoint(simpoint)
else:
assert isinstance(simpoint, SimPoint)
self._simpoint_object = simpoint
self._simpoint_resource = simpoint
if self.get_processor().get_num_cores() > 1:
warn("SimPoints only works with one core")
self.get_processor().get_cores()[0]._set_simpoint(
inst_starts=self._simpoint_object.get_simpoint_start_insts(),
inst_starts=self._simpoint_resource.get_simpoint_start_insts(),
board_initialized=False,
)
@@ -167,11 +162,11 @@ class SEBinaryWorkload:
checkpoint=checkpoint,
)
def get_simpoint(self) -> SimPoint:
def get_simpoint(self) -> SimpointResource:
"""
Returns the SimPoint object set. If no SimPoint object has been set an
exception is thrown.
Returns the SimpointResorce object set. If no SimpointResource object
has been set an exception is thrown.
"""
if getattr(self, "_simpoint_object", None):
return self._simpoint_object
if getattr(self, "_simpoint_resource", None):
return self._simpoint_resource
raise Exception("This board does not have a simpoint set.")

196
src/python/gem5/resources/resource.py
View File

@@ -27,13 +27,13 @@
from abc import ABCMeta
import os
from pathlib import Path
from m5.util import warn
from m5.util import warn, fatal
from .downloader import get_resource, get_resources_json_obj
from ..isas import ISA, get_isa_from_str
from typing import Optional, Dict, Union, Type
from typing import Optional, Dict, Union, Type, Tuple, List
"""
Resources are items needed to run a simulation, such as a disk image, kernel,
@@ -72,7 +72,7 @@ class AbstractResource:
):
"""
:param local_path: The path on the host system where this resource is
located
located.
:param documentation: Documentation describing this resource. Not a
required parameter. By default is None.
:param source: The source (as in "source code") for this resource. This
@@ -280,22 +280,205 @@ class CheckpointResource(DirectoryResource):
)
class SimpointResource(DirectoryResource):
"""A simpoint resource."""
class SimpointResource(AbstractResource):
"""A simpoint resource. This resource stores all information required to
perform a Simpoint creation and restore. It contains the Simpoint, the
Simpoint interval, the weight for each Simpoint, the full warmup length,
and the warmup length for each Simpoint.
"""
def __init__(
self,
local_path: str,
simpoint_interval: int = None,
simpoint_list: List[int] = None,
weight_list: List[float] = None,
warmup_interval: int = 0,
workload_name: Optional[str] = None,
documentation: Optional[str] = None,
source: Optional[str] = None,
local_path: Optional[str] = None,
**kwargs,
):
"""
:param simpoint_interval: The simpoint interval.
:param simpoint_list: The simpoint list.
:param weight_list: The weight list.
:param warmup_interval: The warmup interval. Default to zero (a value
of zero means effectively not set).
:param workload_name: Simpoints are typically associated with a
particular workload due to their dependency on chosen input parameters.
This field helps backtrack to that resource if required. This should
relate to a workload "name" field in the resource.json file.
"""
super().__init__(
local_path=local_path,
documentation=documentation,
source=source,
)
self._weight_list = weight_list
self._simpoint_list = simpoint_list
self._simpoint_interval = simpoint_interval
self._warmup_interval = warmup_interval
self._workload_name = workload_name
self._simpoint_start_insts = list(
inst * simpoint_interval for inst in self.get_simpoint_list()
)
if self._warmup_interval != 0:
self._warmup_list = self._set_warmup_list()
else:
self._warmup_list = [0] * len(self.get_simpoint_start_insts)
def get_simpoint_list(self) -> List[int]:
"""Returns the a list containing all the Simpoints for the workload."""
return self._simpoint_list
def get_simpoint_start_insts(self) -> List[int]:
"""Returns a lst containing all the Simpoint starting instrunction
points for the workload. This was calculated by multiplying the
Simpoint with the Simpoint interval when it was generated."""
return self._simpoint_start_insts
def get_warmup_interval(self) -> int:
"""Returns the instruction length of the warmup interval."""
return self._warmup_interval
def get_weight_list(self) -> List[float]:
"""Returns the list that contains the weight for each Simpoint. The
order of the weights matches that of the list returned by
`get_simpoint_list(). I.e. `get_weight_list()[3]` is the weight for
simpoint `get_simpoint_list()[3]`."""
return self._weight_list
def get_simpoint_interval(self) -> int:
"""Returns the Simpoint interval value."""
return self._simpoint_interval
def get_warmup_list(self) -> List[int]:
"""Returns the a list containing the warmup length for each Simpoint.
Each warmup length in this list corresponds to the Simpoint at the same
index in `get_simpoint_list()`. I.e., `get_warmup_list()[4]` is the
warmup length for Simpoint `get_simpoint_list()[4]`."""
return self._warmup_list
def get_workload_name(self) -> Optional[str]:
"""Return the workload name this Simpoint is associated with."""
return self._workload_name
def _set_warmup_list(self) -> List[int]:
"""
This function uses the warmup_interval, fits it into the
simpoint_start_insts, and outputs a list of warmup instruction lengths
for each SimPoint.
The warmup instruction length is calculated using the starting
instruction of a SimPoint to minus the warmup_interval and the ending
instruction of the last SimPoint. If it is less than 0, then the warmup
instruction length is the gap between the starting instruction of a
SimPoint and the ending instruction of the last SimPoint.
"""
warmup_list = []
for index, start_inst in enumerate(self.get_simpoint_start_insts()):
warmup_inst = start_inst - self.get_warmup_interval()
if warmup_inst < 0:
warmup_inst = start_inst
else:
warmup_inst = self.get_warmup_interval()
warmup_list.append(warmup_inst)
# change the starting instruction of a SimPoint to include the
# warmup instruction length
self._simpoint_start_insts[index] = start_inst - warmup_inst
return warmup_list
class SimpointDirectoryResource(SimpointResource):
"""A Simpoint diretory resource. This Simpoint Resource assumes the
existance of a directory containing a simpoint file and a weight file."""
def __init__(
self,
local_path: str,
simpoint_file: str,
weight_file: str,
simpoint_interval: int,
warmup_interval: int,
workload_name: Optional[str] = None,
documentation: Optional[str] = None,
source: Optional[str] = None,
**kwargs,
):
"""
:param simpoint_file: The Simpoint file. This file is a list of
Simpoints, each on its own line. It should map 1-to-1 to the weights
file.
:param weight_file: The Simpoint weights file. This file is a list of
weights, each on its own line.
"""
self._simpoint_file = simpoint_file
self._weight_file = weight_file
# This is a little hack. The functions `get_simpoint_file` and
# `get_weight_file` use the local path, so we set it here despite it
# also being set in the `AbstractResource` constructor. This isn't
# elegant but does not harm.
self._local_path = local_path
(
simpoint_list,
weight_list,
) = self._get_weights_and_simpoints_from_file()
super().__init__(
simpoint_interval=simpoint_interval,
simpoint_list=simpoint_list,
weight_list=weight_list,
warmup_interval=warmup_interval,
workload_name=workload_name,
local_path=local_path,
documentation=documentation,
source=source,
)
def get_simpoint_file(self) -> Path:
"""Return the Simpoint File path."""
return Path(Path(self._local_path) / self._simpoint_file)
def get_weight_file(self) -> Path:
"""Returns the Weight File path."""
return Path(Path(self._local_path) / self._weight_file)
def _get_weights_and_simpoints_from_file(
self,
) -> Tuple[List[int], List[int]]:
"""This is a helper function to extract the weights and simpoints from
the files.
"""
simpoint_weight_pair = []
with open(self.get_simpoint_file()) as simpoint_file, open(
self.get_weight_file()
) as weight_file:
while True:
line = simpoint_file.readline()
if not line:
break
interval = int(line.split(" ", 1)[0])
line = weight_file.readline()
if not line:
fatal("not engough weights")
weight = float(line.split(" ", 1)[0])
simpoint_weight_pair.append((interval, weight))
simpoint_weight_pair.sort(key=lambda obj: obj[0])
# use simpoint to sort
weight_list = []
simpoint_list = []
for simpoint, weight in simpoint_weight_pair:
simpoint_list.append(simpoint)
weight_list.append(weight)
return simpoint_list, weight_list
def obtain_resource(
resource_name: str,
@@ -529,5 +712,6 @@ _get_resource_json_type_map = {
"file": FileResource,
"directory": DirectoryResource,
"simpoint": SimpointResource,
"simpoint-directory": SimpointDirectoryResource,
"resource": Resource,
}

4
src/python/gem5/simulate/exit_event_generators.py
View File

@@ -28,7 +28,7 @@ from typing import Generator, Optional
import m5.stats
from ..components.processors.abstract_processor import AbstractProcessor
from ..components.processors.switchable_processor import SwitchableProcessor
from ..utils.simpoint import SimPoint
from ..resources.resource import SimpointResource
from m5.util import warn
from pathlib import Path
@@ -134,7 +134,7 @@ def skip_generator():
def simpoints_save_checkpoint_generator(
checkpoint_dir: Path, simpoint: SimPoint
checkpoint_dir: Path, simpoint: SimpointResource
):
"""
A generator for taking multiple checkpoints for SimPoints. It will save the

17
src/python/gem5/utils/simpoint.py
View File

@@ -24,10 +24,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from m5.util import fatal
from m5.util import fatal, warn
from pathlib import Path
from typing import List, Tuple
from gem5.resources.resource import Resource, CustomResource
from gem5.resources.resource import SimpointResource
class SimPoint:
@@ -39,7 +39,7 @@ class SimPoint:
def __init__(
self,
simpoint_resource: CustomResource = None,
simpoint_resource: SimpointResource = None,
simpoint_interval: int = None,
simpoint_file_path: Path = None,
weight_file_path: Path = None,
@@ -70,12 +70,19 @@ class SimPoint:
The warmup_list only works correctly with sorted simpoint_list.
"""
warn(
"This `SimPoint` class has been deprecated in favor of "
"`SimpointResource` and `SimpointDirectory` resource which may be "
"found in `gem5.resources.resource`. Please utilize these. This "
"`SimPoint` class will be removed in future releases of gem5."
)
# initalize input if you're passing in a CustomResource
if simpoint_resource is not None:
simpoint_directory = str(simpoint_resource.get_local_path())
simpoint_file_path = Path(simpoint_directory + "/simpoint.simpt")
weight_file_path = Path(simpoint_directory + "/simpoint.weight")
simpoint_file_path = simpoint_directory.get_simpoint_file()
weight_file_path = simpoint_resource.get_weight_file()
simpoint_interval = (
simpoint_resource.get_metadata()
.get("additional_metadata")

74
tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py
View File

@@ -94,44 +94,44 @@ gem5_verify_config(
length=constants.quick_tag,
)
# gem5_verify_config(
# name="test-simpoints-se-checkpoint",
# fixtures=(),
# verifiers=(),
# config=joinpath(
# config.base_dir,
# "configs",
# "example",
# "gem5_library",
# "checkpoints",
# "simpoints-se-checkpoint.py",
# ),
# config_args=[
# "--checkpoint-path",
# joinpath(resource_path, "se_checkpoint_folder-save"),
# ],
# valid_isas=(constants.all_compiled_tag,),
# valid_hosts=constants.supported_hosts,
# length=constants.quick_tag,
# )
gem5_verify_config(
name="test-simpoints-se-checkpoint",
fixtures=(),
verifiers=(),
config=joinpath(
config.base_dir,
"configs",
"example",
"gem5_library",
"checkpoints",
"simpoints-se-checkpoint.py",
),
config_args=[
"--checkpoint-path",
joinpath(resource_path, "se_checkpoint_folder-save"),
],
valid_isas=(constants.all_compiled_tag,),
valid_hosts=constants.supported_hosts,
length=constants.quick_tag,
)
# gem5_verify_config(
# name="test-simpoints-se-restore",
# fixtures=(),
# verifiers=(),
# config=joinpath(
# config.base_dir,
# "configs",
# "example",
# "gem5_library",
# "checkpoints",
# "simpoints-se-restore.py",
# ),
# config_args=[],
# valid_isas=(constants.all_compiled_tag,),
# valid_hosts=constants.supported_hosts,
# length=constants.quick_tag,
# )
gem5_verify_config(
name="test-simpoints-se-restore",
fixtures=(),
verifiers=(),
config=joinpath(
config.base_dir,
"configs",
"example",
"gem5_library",
"checkpoints",
"simpoints-se-restore.py",
),
config_args=[],
valid_isas=(constants.all_compiled_tag,),
valid_hosts=constants.supported_hosts,
length=constants.quick_tag,
)
if os.access("/dev/kvm", mode=os.R_OK | os.W_OK):
# The x86-ubuntu-run uses KVM cores, this test will therefore only be run

41
tests/pyunit/stdlib/resources/pyunit_resource_specialization.py
View File

@@ -26,6 +26,7 @@
import os
import unittest
from pathlib import Path
from gem5.resources.resource import *
from gem5.isas import ISA
@@ -155,6 +156,40 @@ class ResourceSpecializationSuite(unittest.TestCase):
self.assertIsNone(resource.get_documentation())
self.assertIsNone(resource.get_source())
def test_simpoint_directory_resource(self) -> None:
"""Tests the loading of a Simpoint directory resource."""
resource = obtain_resource(
resource_name="simpoint-directory-example",
resource_directory=self.get_resource_dir(),
)
self.assertIsInstance(resource, SimpointDirectoryResource)
self.assertEquals(
"simpoint directory documentation.", resource.get_documentation()
)
self.assertIsNone(resource.get_source())
self.assertEquals(1000000, resource.get_simpoint_interval())
self.assertEquals(1000000, resource.get_warmup_interval())
self.assertEquals(
Path(
Path(self.get_resource_dir())
/ "simpoint-directory-example"
/ "simpoint.simpt"
),
resource.get_simpoint_file(),
)
self.assertEquals(
Path(
Path(self.get_resource_dir())
/ "simpoint-directory-example"
/ "simpoint.weight"
),
resource.get_weight_file(),
)
self.assertEquals("Example Workload", resource.get_workload_name())
def test_simpoint_resource(self) -> None:
"""Tests the loading of a Simpoint resource."""
resource = obtain_resource(
@@ -168,6 +203,12 @@ class ResourceSpecializationSuite(unittest.TestCase):
"simpoint documentation.", resource.get_documentation()
)
self.assertIsNone(resource.get_source())
self.assertIsNone(resource.get_local_path())
self.assertEquals(1000000, resource.get_simpoint_interval())
self.assertEquals(23445, resource.get_warmup_interval())
self.assertEquals([2, 3, 4, 15], resource.get_simpoint_list())
self.assertEquals([0.1, 0.2, 0.4, 0.3], resource.get_weight_list())
def test_file_resource(self) -> None:
"""Tests the loading of a FileResource."""

26
tests/pyunit/stdlib/resources/refs/resource-specialization.json
View File

@@ -86,14 +86,28 @@
"url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar"
},
{
"type" : "simpoint",
"name" : "simpoint-example",
"documentation" : "simpoint documentation.",
"type": "simpoint-directory",
"name": "simpoint-directory-example",
"documentation": "simpoint directory documentation.",
"is_zipped" : false,
"md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace",
"md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb",
"source" : null,
"is_tar_archive" : true,
"url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar"
}
"url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar",
"simpoint_interval": 1000000,
"warmup_interval": 1000000,
"simpoint_file": "simpoint.simpt",
"weight_file": "simpoint.weight",
"workload_name": "Example Workload"
},
{
"type": "simpoint",
"name": "simpoint-example",
"documentation": "simpoint documentation.",
"simpoint_interval": 1000000,
"warmup_interval": 23445,
"simpoint_list" : [2,3,4,15],
"weight_list" : [0.1, 0.2, 0.4, 0.3]
}
]
}