diff --git a/configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py b/configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py index d2d1af730f..b5eb7e9912 100644 --- a/configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py +++ b/configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py @@ -58,6 +58,7 @@ from gem5.components.processors.simple_processor import SimpleProcessor from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA from gem5.resources.workload import Workload +from gem5.resources.resource import obtain_resource, SimpointResource from pathlib import Path from gem5.components.cachehierarchies.classic.no_cache import NoCache from gem5.simulate.exit_event_generators import ( @@ -108,7 +109,23 @@ board = SimpleBoard( cache_hierarchy=cache_hierarchy, ) -board.set_workload(Workload("x86-print-this-15000-with-simpoints")) +# board.set_workload( +# Workload("x86-print-this-15000-with-simpoints") +# +# **Note: This has been removed until we update the resources.json file to +# encapsulate the new Simpoint format. +# Below we set the simpount manually. + +board.set_se_simpoint_workload( + binary=obtain_resource("x86-print-this"), + arguments=["print this", 15000], + simpoint=SimpointResource( + simpoint_interval=1000000, + simpoint_list=[2, 3, 4, 15], + weight_list=[0.1, 0.2, 0.4, 0.3], + warmup_interval=1000000, + ), +) dir = Path(args.checkpoint_path) dir.mkdir(exist_ok=True) diff --git a/configs/example/gem5_library/checkpoints/simpoints-se-restore.py b/configs/example/gem5_library/checkpoints/simpoints-se-restore.py index f8f48d0ec1..5ff82dba04 100644 --- a/configs/example/gem5_library/checkpoints/simpoints-se-restore.py +++ b/configs/example/gem5_library/checkpoints/simpoints-se-restore.py @@ -63,8 +63,9 @@ from gem5.components.memory import DualChannelDDR4_2400 from gem5.components.processors.simple_processor import SimpleProcessor from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA -from gem5.resources.resource import Resource +from gem5.resources.resource import SimpointResource, obtain_resource from gem5.resources.workload import Workload +from gem5.resources.resource import SimpointResource from pathlib import Path from m5.stats import reset, dump @@ -96,11 +97,29 @@ board = SimpleBoard( cache_hierarchy=cache_hierarchy, ) -# Here we obtain the workloadfrom gem5 resources, the checkpoint in this +# Here we obtain the workload from gem5 resources, the checkpoint in this # workload was generated from # `configs/example/gem5_library/checkpoints/simpoints-se-checkpoint.py`. -board.set_workload( - Workload("x86-print-this-15000-with-simpoints-and-checkpoint") +# board.set_workload( +# Workload("x86-print-this-15000-with-simpoints-and-checkpoint") +# +# **Note: This has been removed until we update the resources.json file to +# encapsulate the new Simpoint format. +# Below we set the simpount manually. +# +# This loads a single checkpoint as an example of using simpoints to simulate +# the function of a single simpoint region. + +board.set_se_simpoint_workload( + binary=obtain_resource("x86-print-this"), + arguments=["print this", 15000], + simpoint=SimpointResource( + simpoint_interval=1000000, + simpoint_list=[2, 3, 4, 15], + weight_list=[0.1, 0.2, 0.4, 0.3], + warmup_interval=1000000, + ), + checkpoint=obtain_resource("simpoints-se-checkpoints-v22-1-v2"), ) diff --git a/src/python/gem5/components/boards/se_binary_workload.py b/src/python/gem5/components/boards/se_binary_workload.py index acedfaf9a9..31931106c9 100644 --- a/src/python/gem5/components/boards/se_binary_workload.py +++ b/src/python/gem5/components/boards/se_binary_workload.py @@ -31,8 +31,8 @@ from ...resources.resource import ( BinaryResource, CheckpointResource, SimpointResource, + SimpointDirectoryResource, ) -from gem5.utils.simpoint import SimPoint from m5.objects import SEWorkload, Process @@ -125,7 +125,7 @@ class SEBinaryWorkload: self, binary: BinaryResource, arguments: List[str] = [], - simpoint: Union[SimpointResource, SimPoint] = None, + simpoint: SimpointResource = None, checkpoint: Optional[Union[Path, CheckpointResource]] = None, ) -> None: """Set up the system to run a SimPoint workload. @@ -135,28 +135,23 @@ class SEBinaryWorkload: * Dynamically linked executables are partially supported when the host ISA and the simulated ISA are the same. - **Warning:** SimPoints only works with one core + **Warning:** Simpoints only works with one core :param binary: The resource encapsulating the binary to be run. :param arguments: The input arguments for the binary - :param simpoint: The SimPoint object or Resource that contains the list of + :param simpoint: The SimpointResource that contains the list of SimPoints starting instructions, the list of weights, and the SimPoints interval :param checkpoint: The checkpoint directory. Used to restore the simulation to that checkpoint. """ - # convert input to SimPoint if necessary - if isinstance(simpoint, SimpointResource): - self._simpoint_object = SimPoint(simpoint) - else: - assert isinstance(simpoint, SimPoint) - self._simpoint_object = simpoint + self._simpoint_resource = simpoint if self.get_processor().get_num_cores() > 1: warn("SimPoints only works with one core") self.get_processor().get_cores()[0]._set_simpoint( - inst_starts=self._simpoint_object.get_simpoint_start_insts(), + inst_starts=self._simpoint_resource.get_simpoint_start_insts(), board_initialized=False, ) @@ -167,11 +162,11 @@ class SEBinaryWorkload: checkpoint=checkpoint, ) - def get_simpoint(self) -> SimPoint: + def get_simpoint(self) -> SimpointResource: """ - Returns the SimPoint object set. If no SimPoint object has been set an - exception is thrown. + Returns the SimpointResorce object set. If no SimpointResource object + has been set an exception is thrown. """ - if getattr(self, "_simpoint_object", None): - return self._simpoint_object + if getattr(self, "_simpoint_resource", None): + return self._simpoint_resource raise Exception("This board does not have a simpoint set.") diff --git a/src/python/gem5/resources/resource.py b/src/python/gem5/resources/resource.py index 0987453c9a..678497eaa7 100644 --- a/src/python/gem5/resources/resource.py +++ b/src/python/gem5/resources/resource.py @@ -27,13 +27,13 @@ from abc import ABCMeta import os from pathlib import Path -from m5.util import warn +from m5.util import warn, fatal from .downloader import get_resource, get_resources_json_obj from ..isas import ISA, get_isa_from_str -from typing import Optional, Dict, Union, Type +from typing import Optional, Dict, Union, Type, Tuple, List """ Resources are items needed to run a simulation, such as a disk image, kernel, @@ -72,7 +72,7 @@ class AbstractResource: ): """ :param local_path: The path on the host system where this resource is - located + located. :param documentation: Documentation describing this resource. Not a required parameter. By default is None. :param source: The source (as in "source code") for this resource. This @@ -280,22 +280,205 @@ class CheckpointResource(DirectoryResource): ) -class SimpointResource(DirectoryResource): - """A simpoint resource.""" +class SimpointResource(AbstractResource): + """A simpoint resource. This resource stores all information required to + perform a Simpoint creation and restore. It contains the Simpoint, the + Simpoint interval, the weight for each Simpoint, the full warmup length, + and the warmup length for each Simpoint. + """ def __init__( self, - local_path: str, + simpoint_interval: int = None, + simpoint_list: List[int] = None, + weight_list: List[float] = None, + warmup_interval: int = 0, + workload_name: Optional[str] = None, documentation: Optional[str] = None, source: Optional[str] = None, + local_path: Optional[str] = None, **kwargs, ): + """ + :param simpoint_interval: The simpoint interval. + :param simpoint_list: The simpoint list. + :param weight_list: The weight list. + :param warmup_interval: The warmup interval. Default to zero (a value + of zero means effectively not set). + :param workload_name: Simpoints are typically associated with a + particular workload due to their dependency on chosen input parameters. + This field helps backtrack to that resource if required. This should + relate to a workload "name" field in the resource.json file. + """ + super().__init__( local_path=local_path, documentation=documentation, source=source, ) + self._weight_list = weight_list + self._simpoint_list = simpoint_list + self._simpoint_interval = simpoint_interval + self._warmup_interval = warmup_interval + self._workload_name = workload_name + + self._simpoint_start_insts = list( + inst * simpoint_interval for inst in self.get_simpoint_list() + ) + + if self._warmup_interval != 0: + self._warmup_list = self._set_warmup_list() + else: + self._warmup_list = [0] * len(self.get_simpoint_start_insts) + + def get_simpoint_list(self) -> List[int]: + """Returns the a list containing all the Simpoints for the workload.""" + return self._simpoint_list + + def get_simpoint_start_insts(self) -> List[int]: + """Returns a lst containing all the Simpoint starting instrunction + points for the workload. This was calculated by multiplying the + Simpoint with the Simpoint interval when it was generated.""" + return self._simpoint_start_insts + + def get_warmup_interval(self) -> int: + """Returns the instruction length of the warmup interval.""" + return self._warmup_interval + + def get_weight_list(self) -> List[float]: + """Returns the list that contains the weight for each Simpoint. The + order of the weights matches that of the list returned by + `get_simpoint_list(). I.e. `get_weight_list()[3]` is the weight for + simpoint `get_simpoint_list()[3]`.""" + return self._weight_list + + def get_simpoint_interval(self) -> int: + """Returns the Simpoint interval value.""" + return self._simpoint_interval + + def get_warmup_list(self) -> List[int]: + """Returns the a list containing the warmup length for each Simpoint. + Each warmup length in this list corresponds to the Simpoint at the same + index in `get_simpoint_list()`. I.e., `get_warmup_list()[4]` is the + warmup length for Simpoint `get_simpoint_list()[4]`.""" + return self._warmup_list + + def get_workload_name(self) -> Optional[str]: + """Return the workload name this Simpoint is associated with.""" + return self._workload_name + + def _set_warmup_list(self) -> List[int]: + """ + This function uses the warmup_interval, fits it into the + simpoint_start_insts, and outputs a list of warmup instruction lengths + for each SimPoint. + + The warmup instruction length is calculated using the starting + instruction of a SimPoint to minus the warmup_interval and the ending + instruction of the last SimPoint. If it is less than 0, then the warmup + instruction length is the gap between the starting instruction of a + SimPoint and the ending instruction of the last SimPoint. + """ + warmup_list = [] + for index, start_inst in enumerate(self.get_simpoint_start_insts()): + warmup_inst = start_inst - self.get_warmup_interval() + if warmup_inst < 0: + warmup_inst = start_inst + else: + warmup_inst = self.get_warmup_interval() + warmup_list.append(warmup_inst) + # change the starting instruction of a SimPoint to include the + # warmup instruction length + self._simpoint_start_insts[index] = start_inst - warmup_inst + return warmup_list + + +class SimpointDirectoryResource(SimpointResource): + """A Simpoint diretory resource. This Simpoint Resource assumes the + existance of a directory containing a simpoint file and a weight file.""" + + def __init__( + self, + local_path: str, + simpoint_file: str, + weight_file: str, + simpoint_interval: int, + warmup_interval: int, + workload_name: Optional[str] = None, + documentation: Optional[str] = None, + source: Optional[str] = None, + **kwargs, + ): + """ + :param simpoint_file: The Simpoint file. This file is a list of + Simpoints, each on its own line. It should map 1-to-1 to the weights + file. + :param weight_file: The Simpoint weights file. This file is a list of + weights, each on its own line. + """ + self._simpoint_file = simpoint_file + self._weight_file = weight_file + + # This is a little hack. The functions `get_simpoint_file` and + # `get_weight_file` use the local path, so we set it here despite it + # also being set in the `AbstractResource` constructor. This isn't + # elegant but does not harm. + self._local_path = local_path + ( + simpoint_list, + weight_list, + ) = self._get_weights_and_simpoints_from_file() + + super().__init__( + simpoint_interval=simpoint_interval, + simpoint_list=simpoint_list, + weight_list=weight_list, + warmup_interval=warmup_interval, + workload_name=workload_name, + local_path=local_path, + documentation=documentation, + source=source, + ) + + def get_simpoint_file(self) -> Path: + """Return the Simpoint File path.""" + return Path(Path(self._local_path) / self._simpoint_file) + + def get_weight_file(self) -> Path: + """Returns the Weight File path.""" + return Path(Path(self._local_path) / self._weight_file) + + def _get_weights_and_simpoints_from_file( + self, + ) -> Tuple[List[int], List[int]]: + """This is a helper function to extract the weights and simpoints from + the files. + """ + simpoint_weight_pair = [] + with open(self.get_simpoint_file()) as simpoint_file, open( + self.get_weight_file() + ) as weight_file: + while True: + line = simpoint_file.readline() + if not line: + break + interval = int(line.split(" ", 1)[0]) + line = weight_file.readline() + if not line: + fatal("not engough weights") + weight = float(line.split(" ", 1)[0]) + simpoint_weight_pair.append((interval, weight)) + simpoint_weight_pair.sort(key=lambda obj: obj[0]) + # use simpoint to sort + + weight_list = [] + simpoint_list = [] + for simpoint, weight in simpoint_weight_pair: + simpoint_list.append(simpoint) + weight_list.append(weight) + return simpoint_list, weight_list + def obtain_resource( resource_name: str, @@ -529,5 +712,6 @@ _get_resource_json_type_map = { "file": FileResource, "directory": DirectoryResource, "simpoint": SimpointResource, + "simpoint-directory": SimpointDirectoryResource, "resource": Resource, } diff --git a/src/python/gem5/simulate/exit_event_generators.py b/src/python/gem5/simulate/exit_event_generators.py index d6732bb49d..738e1281d9 100644 --- a/src/python/gem5/simulate/exit_event_generators.py +++ b/src/python/gem5/simulate/exit_event_generators.py @@ -28,7 +28,7 @@ from typing import Generator, Optional import m5.stats from ..components.processors.abstract_processor import AbstractProcessor from ..components.processors.switchable_processor import SwitchableProcessor -from ..utils.simpoint import SimPoint +from ..resources.resource import SimpointResource from m5.util import warn from pathlib import Path @@ -134,7 +134,7 @@ def skip_generator(): def simpoints_save_checkpoint_generator( - checkpoint_dir: Path, simpoint: SimPoint + checkpoint_dir: Path, simpoint: SimpointResource ): """ A generator for taking multiple checkpoints for SimPoints. It will save the diff --git a/src/python/gem5/utils/simpoint.py b/src/python/gem5/utils/simpoint.py index 9e861cc0a5..eab92e2291 100644 --- a/src/python/gem5/utils/simpoint.py +++ b/src/python/gem5/utils/simpoint.py @@ -24,10 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from m5.util import fatal +from m5.util import fatal, warn from pathlib import Path from typing import List, Tuple -from gem5.resources.resource import Resource, CustomResource +from gem5.resources.resource import SimpointResource class SimPoint: @@ -39,7 +39,7 @@ class SimPoint: def __init__( self, - simpoint_resource: CustomResource = None, + simpoint_resource: SimpointResource = None, simpoint_interval: int = None, simpoint_file_path: Path = None, weight_file_path: Path = None, @@ -70,12 +70,19 @@ class SimPoint: The warmup_list only works correctly with sorted simpoint_list. """ + warn( + "This `SimPoint` class has been deprecated in favor of " + "`SimpointResource` and `SimpointDirectory` resource which may be " + "found in `gem5.resources.resource`. Please utilize these. This " + "`SimPoint` class will be removed in future releases of gem5." + ) + # initalize input if you're passing in a CustomResource if simpoint_resource is not None: simpoint_directory = str(simpoint_resource.get_local_path()) - simpoint_file_path = Path(simpoint_directory + "/simpoint.simpt") - weight_file_path = Path(simpoint_directory + "/simpoint.weight") + simpoint_file_path = simpoint_directory.get_simpoint_file() + weight_file_path = simpoint_resource.get_weight_file() simpoint_interval = ( simpoint_resource.get_metadata() .get("additional_metadata") diff --git a/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py b/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py index 514894f8d2..9b5c2c67ff 100644 --- a/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py +++ b/tests/gem5/gem5_library_example_tests/test_gem5_library_examples.py @@ -94,44 +94,44 @@ gem5_verify_config( length=constants.quick_tag, ) -# gem5_verify_config( -# name="test-simpoints-se-checkpoint", -# fixtures=(), -# verifiers=(), -# config=joinpath( -# config.base_dir, -# "configs", -# "example", -# "gem5_library", -# "checkpoints", -# "simpoints-se-checkpoint.py", -# ), -# config_args=[ -# "--checkpoint-path", -# joinpath(resource_path, "se_checkpoint_folder-save"), -# ], -# valid_isas=(constants.all_compiled_tag,), -# valid_hosts=constants.supported_hosts, -# length=constants.quick_tag, -# ) +gem5_verify_config( + name="test-simpoints-se-checkpoint", + fixtures=(), + verifiers=(), + config=joinpath( + config.base_dir, + "configs", + "example", + "gem5_library", + "checkpoints", + "simpoints-se-checkpoint.py", + ), + config_args=[ + "--checkpoint-path", + joinpath(resource_path, "se_checkpoint_folder-save"), + ], + valid_isas=(constants.all_compiled_tag,), + valid_hosts=constants.supported_hosts, + length=constants.quick_tag, +) -# gem5_verify_config( -# name="test-simpoints-se-restore", -# fixtures=(), -# verifiers=(), -# config=joinpath( -# config.base_dir, -# "configs", -# "example", -# "gem5_library", -# "checkpoints", -# "simpoints-se-restore.py", -# ), -# config_args=[], -# valid_isas=(constants.all_compiled_tag,), -# valid_hosts=constants.supported_hosts, -# length=constants.quick_tag, -# ) +gem5_verify_config( + name="test-simpoints-se-restore", + fixtures=(), + verifiers=(), + config=joinpath( + config.base_dir, + "configs", + "example", + "gem5_library", + "checkpoints", + "simpoints-se-restore.py", + ), + config_args=[], + valid_isas=(constants.all_compiled_tag,), + valid_hosts=constants.supported_hosts, + length=constants.quick_tag, +) if os.access("/dev/kvm", mode=os.R_OK | os.W_OK): # The x86-ubuntu-run uses KVM cores, this test will therefore only be run diff --git a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py index e0a8dddd07..f31e35d719 100644 --- a/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py +++ b/tests/pyunit/stdlib/resources/pyunit_resource_specialization.py @@ -26,6 +26,7 @@ import os import unittest +from pathlib import Path from gem5.resources.resource import * from gem5.isas import ISA @@ -155,6 +156,40 @@ class ResourceSpecializationSuite(unittest.TestCase): self.assertIsNone(resource.get_documentation()) self.assertIsNone(resource.get_source()) + def test_simpoint_directory_resource(self) -> None: + """Tests the loading of a Simpoint directory resource.""" + resource = obtain_resource( + resource_name="simpoint-directory-example", + resource_directory=self.get_resource_dir(), + ) + + self.assertIsInstance(resource, SimpointDirectoryResource) + + self.assertEquals( + "simpoint directory documentation.", resource.get_documentation() + ) + self.assertIsNone(resource.get_source()) + + self.assertEquals(1000000, resource.get_simpoint_interval()) + self.assertEquals(1000000, resource.get_warmup_interval()) + self.assertEquals( + Path( + Path(self.get_resource_dir()) + / "simpoint-directory-example" + / "simpoint.simpt" + ), + resource.get_simpoint_file(), + ) + self.assertEquals( + Path( + Path(self.get_resource_dir()) + / "simpoint-directory-example" + / "simpoint.weight" + ), + resource.get_weight_file(), + ) + self.assertEquals("Example Workload", resource.get_workload_name()) + def test_simpoint_resource(self) -> None: """Tests the loading of a Simpoint resource.""" resource = obtain_resource( @@ -168,6 +203,12 @@ class ResourceSpecializationSuite(unittest.TestCase): "simpoint documentation.", resource.get_documentation() ) self.assertIsNone(resource.get_source()) + self.assertIsNone(resource.get_local_path()) + + self.assertEquals(1000000, resource.get_simpoint_interval()) + self.assertEquals(23445, resource.get_warmup_interval()) + self.assertEquals([2, 3, 4, 15], resource.get_simpoint_list()) + self.assertEquals([0.1, 0.2, 0.4, 0.3], resource.get_weight_list()) def test_file_resource(self) -> None: """Tests the loading of a FileResource.""" diff --git a/tests/pyunit/stdlib/resources/refs/resource-specialization.json b/tests/pyunit/stdlib/resources/refs/resource-specialization.json index 77ffc10705..01671b564b 100644 --- a/tests/pyunit/stdlib/resources/refs/resource-specialization.json +++ b/tests/pyunit/stdlib/resources/refs/resource-specialization.json @@ -86,14 +86,28 @@ "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" }, { - "type" : "simpoint", - "name" : "simpoint-example", - "documentation" : "simpoint documentation.", + "type": "simpoint-directory", + "name": "simpoint-directory-example", + "documentation": "simpoint directory documentation.", "is_zipped" : false, - "md5sum" : "3a57c1bb1077176c4587b8a3bf4f8ace", + "md5sum" : "3fcffe3956c8a95e3fb82e232e2b41fb", "source" : null, "is_tar_archive" : true, - "url": "{url_base}/checkpoints/riscv-hello-example-checkpoint.tar" - } + "url": "{url_base}/simpoints/x86-print-this-15000-simpoints-20221013.tar", + "simpoint_interval": 1000000, + "warmup_interval": 1000000, + "simpoint_file": "simpoint.simpt", + "weight_file": "simpoint.weight", + "workload_name": "Example Workload" + }, + { + "type": "simpoint", + "name": "simpoint-example", + "documentation": "simpoint documentation.", + "simpoint_interval": 1000000, + "warmup_interval": 23445, + "simpoint_list" : [2,3,4,15], + "weight_list" : [0.1, 0.2, 0.4, 0.3] + } ] }