From d661023de48488463f4c84ea88d81f7b097c62a0 Mon Sep 17 00:00:00 2001 From: Mahyar Samani Date: Tue, 28 May 2024 00:06:41 -0700 Subject: [PATCH] stdlib: Adding SpatterGenCore and SpatterGen This change adds code for SpatterGenCore and SpatterGen as well as SpatterKernel to the standard library. SpatterGenCore and SpatterGen follow the same structure as AbstractCore and AbstractProcessor. spatter_kernel.py adds utility functions to parse dictionaries as well as partition a list into multiple lists through interleaving to be used when setting up a multicore SpatterGen. Change-Id: I003553e97f901c0724f5feac0bb6e21a020bd6ad --- src/python/SConscript | 8 + .../processors/spatter_gen/__init__.py | 33 +++ .../spatter_gen/spatter_generator.py | 147 +++++++++++++ .../spatter_gen/spatter_generator_core.py | 73 +++++++ .../processors/spatter_gen/spatter_kernel.py | 200 ++++++++++++++++++ src/python/gem5/simulate/exit_event.py | 3 + .../gem5/simulate/exit_event_generators.py | 7 + src/python/gem5/simulate/simulator.py | 15 +- 8 files changed, 483 insertions(+), 3 deletions(-) create mode 100644 src/python/gem5/components/processors/spatter_gen/__init__.py create mode 100644 src/python/gem5/components/processors/spatter_gen/spatter_generator.py create mode 100644 src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py create mode 100644 src/python/gem5/components/processors/spatter_gen/spatter_kernel.py diff --git a/src/python/SConscript b/src/python/SConscript index fc2241fa09..af117e4a14 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -252,6 +252,14 @@ PySource('gem5.components.processors', 'gem5/components/processors/random_generator_core.py') PySource('gem5.components.processors', 'gem5/components/processors/random_generator.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/__init__.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_generator_core.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_generator.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_kernel.py') PySource('gem5.components.processors', 'gem5/components/processors/simple_core.py') PySource('gem5.components.processors', diff --git a/src/python/gem5/components/processors/spatter_gen/__init__.py b/src/python/gem5/components/processors/spatter_gen/__init__.py new file mode 100644 index 0000000000..3c1847b914 --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from .spatter_generator import SpatterGenerator +from .spatter_kernel import ( + SpatterKernel, + parse_kernel, + partition_trace, +) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_generator.py b/src/python/gem5/components/processors/spatter_gen/spatter_generator.py new file mode 100644 index 0000000000..72939f82dc --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_generator.py @@ -0,0 +1,147 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import ( + List, + Optional, + Union, +) + +from m5.objects import ( + SpatterProcessingMode, + SrcClockDomain, + VoltageDomain, +) +from m5.stats import dump as dump_stats +from m5.stats import reset as reset_stats +from m5.util import fatal + +from ....utils.override import overrides +from ..abstract_generator import AbstractGenerator +from .spatter_generator_core import SpatterGeneratorCore +from .spatter_kernel import SpatterKernel + + +class SpatterGenerator(AbstractGenerator): + def __init__( + self, + num_cores: int = 1, + processing_mode: Union[SpatterProcessingMode, str] = "synchronous", + int_regfile_size: int = 384, + fp_regfile_size: int = 224, + request_gen_latency: int = 2, + request_gen_rate: int = 4, + request_buffer_entries: int = 32, + send_rate: int = 2, + clk_freq: Optional[str] = None, + ) -> None: + super().__init__( + cores=self._create_cores( + num_cores, + processing_mode, + int_regfile_size, + fp_regfile_size, + request_gen_latency, + request_gen_rate, + request_buffer_entries, + send_rate, + ) + ) + # no need for else block since it will intialize generator.clk_domain + # the clock domain of its closest ancestor in the SimObject tree. + if not clk_freq is None: + clock_domain = SrcClockDomain( + clock=clk_freq, voltage_domain=VoltageDomain() + ) + for generator in self.cores: + generator.clk_domain = clock_domain + + self._num_kernels = 0 + self._sync = processing_mode == "synchronous" + + def _create_cores( + self, + num_cores: int, + processing_mode: Union[SpatterProcessingMode, str], + int_regfile_size: int, + fp_regfile_size: int, + request_gen_latency: int, + request_gen_rate: int, + request_buffer_entries: int, + send_rate: int, + ) -> List[SpatterGeneratorCore]: + return [ + SpatterGeneratorCore( + processing_mode, + int_regfile_size, + fp_regfile_size, + request_gen_latency, + request_gen_rate, + request_buffer_entries, + send_rate, + ) + for _ in range(num_cores) + ] + + def add_kernel(self, kernels: List[SpatterKernel]) -> None: + assert len(kernels) == len(self.cores) + for core, kernel in zip(self.cores, kernels): + if kernel.empty(): + fatal( + f"Cannot add {kernel} since it's empty. " + "At the moment SpatterGenerator (or gem5::SpatterGen) " + "does not support adding empty kernels to cores. As a " + "temporary fix you can try adding 1 dummy element to the " + "trace. You can also set fix_empty_trace to True in the " + "constructor of the SpatterKernel which automatically " + "inserts a dummy element (0) to the trace." + ) + core.add_kernel(kernel) + self._num_kernels += 1 + + @overrides(AbstractGenerator) + def start_traffic(self) -> None: + for core in self.cores: + core.start_traffic() + + def _proceed_past_sync_point(self) -> None: + if not self._sync: + return + for core in self.cores: + core.generator.proceedPastSyncPoint() + + def handle_spatter_exit(self): + spatter_exits_observed = 0 + sync_points_observed = 0 + sync_points_expected = self._num_kernels if self._sync else 1 + while True: + spatter_exits_observed += 1 + if spatter_exits_observed % len(self.cores) == 0: + sync_points_observed += 1 + dump_stats() + reset_stats() + self._proceed_past_sync_point() + yield not (sync_points_observed < sync_points_expected) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py b/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py new file mode 100644 index 0000000000..50799eae84 --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Union + +from m5.objects import ( + Port, + SpatterGen, + SpatterProcessingMode, +) + +from ....utils.override import overrides +from ..abstract_core import AbstractCore +from ..abstract_generator_core import AbstractGeneratorCore +from .spatter_kernel import SpatterKernel + + +class SpatterGeneratorCore(AbstractGeneratorCore): + def __init__( + self, + processing_mode: Union[SpatterProcessingMode, str], + int_regfile_size: int, + fp_regfile_size: int, + request_gen_latency: int, + request_gen_rate: int, + request_buffer_entries: int, + send_rate: int, + ): + super().__init__() + self.generator = SpatterGen( + processing_mode=processing_mode, + int_regfile_size=int_regfile_size, + fp_regfile_size=fp_regfile_size, + request_gen_latency=request_gen_latency, + request_gen_rate=request_gen_rate, + request_buffer_entries=request_buffer_entries, + send_rate=send_rate, + ) + self._kernels = [] + + @overrides(AbstractCore) + def connect_dcache(self, port: Port) -> None: + self.generator.port = port + + def add_kernel(self, kernel: SpatterKernel) -> None: + self._kernels.append(kernel) + + def start_traffic(self) -> None: + for kernel in self._kernels: + self.generator.addKernel(*kernel.cxx_call_args()) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py b/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py new file mode 100644 index 0000000000..4cf0ee814a --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py @@ -0,0 +1,200 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from math import ceil +from typing import ( + List, + Tuple, +) + +from m5.objects import SpatterKernelType +from m5.params import Addr +from m5.util import inform + + +def parse_kernel(kernel: dict, default_delta=8) -> Tuple[int, int, str, List]: + delta = kernel.get("delta", default_delta) + if delta < 0: + inform( + f"Negative delta found: {delta}. Setting it to {default_delta}." + ) + delta = default_delta + count = kernel.get("count", 1) + type = kernel.get("kernel", None) + if type is None: + raise ValueError(f"Keyword 'kernel' not found.") + type = SpatterKernelType(type.lower()) + trace = kernel.get("pattern", []) + if len(trace) == 0: + raise ValueError(f"Empty 'pattern' found.") + return (delta, count, type, trace) + + +def partition_trace(original_trace, num_partitions, interleave_size): + partitions = [[] for _ in range(num_partitions)] + num_leaves = ceil(len(original_trace) / interleave_size) + for i in range(num_leaves): + lower_bound = i * interleave_size + upper_bound = min(lower_bound + interleave_size, len(original_trace)) + partitions[i % num_partitions] += original_trace[ + lower_bound:upper_bound + ] + return partitions + + +class SpatterKernel: + """This class encapsulates one kernel in a spatter trace. + A spatter trace is represented with a json file. + An example of a spatter trace can be found here: + https://github.com/hpcgarage/spatter/blob/main/standard-suite/app-traces/amg.json + Each trace may have multiple kernels. + Each kernel represents a code execution like below + for (int iteration = 0; iteration < count; iteration++) + { + for (int i = 0; i < N; i++) { + value[index[i] + iteration * delta] = rand(); // kernel: scatter + // OR + sum += value[index[i] + iteration * delta]; // kernel: gather + } + } + Where `delta` and `count` are fields in each kernel. + `kernel` is another field that determines whether the accesses to value + are loads or stores. + The field `pattern` stores the index array. + + This file provides two utility function to parse spatter traces: + parse_kernel: takes a dictionary and returns a tuple of + delta, count, type, and trace. + partition_trace: takes the original trace, number of partitions, + and interleave_size. + It returns a list of `num_partitions` partitions where each partition + is an list including interleaved elements from `original_trace`. + The elements in the `original_trace` are interleaved with a + granularity of `interleave_size`. + The code snippet below shows how to use these functions to create kernels. + generator = SpatterGenerator(num_cores) + + with open(trace_path, "r") as trace_file: + kernels = json.load(trace_file) + + for i, kernel in enumerate(kernels): + delta, count, type, og_trace = parse_kernel(kernel) + traces = partition_trace(og_trace, num_cores, 128) + kernels = [SpatterKernel( + kernel_id=i, + kernel_delta=delta, + kernel_count=count, + kernel_type=type, + kernel_trace=trace, + index_size=4, + base_index_addr=0, + value_size=8, + base_value_addr=0x400000000 + ) + for trace in traces + ] + generator.add_kernel(kernels) + + Args: + kernel_id (int): The ID of the kernel. + User defined, i.e. spatter traces don't have this field. + It's used to identify the kernel in the simulation. + kernel_delta (int): The delta value of the kernel. + `delta` from spatter trace. + kernel_count (int): The count value of the kernel. + `count` from spatter trace. + kernel_type (SpatterKernelType): The type of the kernel. + `kernel` from spatter trace. + kernel_trace (List[int]): The elements of the `index` array. + `pattern` from spatter trace. + index_size (int): The size of elements in `index`. + User defined, i.e. spatter traces don't have this field. + It represents the size of elements in the `index` array in code above. + base_index_addr (Addr): The base address of the index. + User defined, i.e. spatter traces don't have this field. + It represents the pointer to the `index` array in the code above. + value_size (int): The size of elements in `value`. + User defined, i.e. spatter traces don't have this field. + It represents the size of elements in the `value` array in code above. + base_value_addr (Addr): The base address of the value. + User defined, i.e. spatter traces don't have this field. + It represents the pointer to the `value` array in the code above. + """ + + def __init__( + self, + kernel_id: int, + kernel_delta: int, + kernel_count: int, + kernel_type: SpatterKernelType, + kernel_trace: List[int], + index_size: int, + base_index_addr: Addr, + value_size: int, + base_value_addr: Addr, + fix_empty_trace: bool = False, + ): + self._id = kernel_id + self._delta = kernel_delta + self._count = kernel_count + self._trace = kernel_trace + self._type = kernel_type + self._index_size = index_size + self._base_index_addr = base_index_addr + self._value_size = value_size + self._base_value_addr = base_value_addr + + if fix_empty_trace and len(kernel_trace) == 0: + inform( + "Empty trace found. Fixing it by adding a dummy element. " + "Also setting delta to 0 and count to 1.", + ) + self._trace = [0] + self._delta = 0 + self._count = 1 + + def empty(self): + return len(self._trace) == 0 + + def cxx_call_args(self): + return [ + self._id, + self._delta, + self._count, + self._type.getValue(), + self._index_size, + self._base_index_addr, + self._value_size, + self._base_value_addr, + self._trace, + ] + + def __str__(self): + return ( + f"SpatterKernel(id={self._id}, delta={self._delta}, " + f"count={self._count}, type={self._type}, " + f"trace[:8]={self._trace[:8]}" + ) diff --git a/src/python/gem5/simulate/exit_event.py b/src/python/gem5/simulate/exit_event.py index b902643a3f..5a0bb3d1d7 100644 --- a/src/python/gem5/simulate/exit_event.py +++ b/src/python/gem5/simulate/exit_event.py @@ -39,6 +39,7 @@ class ExitEvent(Enum): EXIT = "exit" # A standard vanilla exit. WORKBEGIN = "workbegin" # An exit because a ROI has been reached. WORKEND = "workend" # An exit because a ROI has ended. + SPATTER_EXIT = "spatter exit" # An exit because a spatter core has ended. SWITCHCPU = "switchcpu" # An exit needed to switch CPU cores. FAIL = "fail" # An exit because the simulation has failed. CHECKPOINT = "checkpoint" # An exit to load a checkpoint. @@ -115,6 +116,8 @@ class ExitEvent(Enum): elif exit_string.endswith("is finished updating the memory.\n"): # This is for the gups generator exit event return ExitEvent.EXIT + elif exit_string.endswith("received all expected responses."): + return ExitEvent.SPATTER_EXIT raise NotImplementedError( f"Exit event '{exit_string}' not implemented" ) diff --git a/src/python/gem5/simulate/exit_event_generators.py b/src/python/gem5/simulate/exit_event_generators.py index 4d18b4cee0..b237b064e2 100644 --- a/src/python/gem5/simulate/exit_event_generators.py +++ b/src/python/gem5/simulate/exit_event_generators.py @@ -36,6 +36,7 @@ from m5.util import warn from gem5.resources.looppoint import Looppoint from ..components.processors.abstract_processor import AbstractProcessor +from ..components.processors.spatter_gen import SpatterGenerator from ..components.processors.switchable_processor import SwitchableProcessor from ..resources.resource import SimpointResource @@ -221,3 +222,9 @@ def looppoint_save_checkpoint_generator( yield False yield True + + +def spatter_exit_generator(spatter_gen: SpatterGenerator): + while True: + assert isinstance(spatter_gen, SpatterGenerator) + yield from spatter_gen.handle_spatter_exit() diff --git a/src/python/gem5/simulate/simulator.py b/src/python/gem5/simulate/simulator.py index 5a5cf9af89..66f67d6ffb 100644 --- a/src/python/gem5/simulate/simulator.py +++ b/src/python/gem5/simulate/simulator.py @@ -53,6 +53,7 @@ from .exit_event_generators import ( reset_stats_generator, save_checkpoint_generator, skip_generator, + spatter_exit_generator, switch_generator, warn_default_decorator, ) @@ -281,6 +282,12 @@ class Simulator: "creating a checkpoint and continuing", )(), ExitEvent.FAIL: exit_generator(), + ExitEvent.SPATTER_EXIT: warn_default_decorator( + spatter_exit_generator, + "spatter exit", + "dumping and resetting stats after each sync point. " + "Note that there will be num_cores*sync_points spatter_exits.", + )(spatter_gen=board.get_processor()), ExitEvent.SWITCHCPU: warn_default_decorator( switch_generator, "switch CPU", @@ -518,9 +525,11 @@ class Simulator: self._board._pre_instantiate() root = Root( - full_system=self._full_system - if self._full_system is not None - else self._board.is_fullsystem(), + full_system=( + self._full_system + if self._full_system is not None + else self._board.is_fullsystem() + ), board=self._board, )