diff --git a/src/python/SConscript b/src/python/SConscript index fc2241fa09..af117e4a14 100644 --- a/src/python/SConscript +++ b/src/python/SConscript @@ -252,6 +252,14 @@ PySource('gem5.components.processors', 'gem5/components/processors/random_generator_core.py') PySource('gem5.components.processors', 'gem5/components/processors/random_generator.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/__init__.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_generator_core.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_generator.py') +PySource('gem5.components.processors.spatter_gen', + 'gem5/components/processors/spatter_gen/spatter_kernel.py') PySource('gem5.components.processors', 'gem5/components/processors/simple_core.py') PySource('gem5.components.processors', diff --git a/src/python/gem5/components/processors/spatter_gen/__init__.py b/src/python/gem5/components/processors/spatter_gen/__init__.py new file mode 100644 index 0000000000..3c1847b914 --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/__init__.py @@ -0,0 +1,33 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from .spatter_generator import SpatterGenerator +from .spatter_kernel import ( + SpatterKernel, + parse_kernel, + partition_trace, +) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_generator.py b/src/python/gem5/components/processors/spatter_gen/spatter_generator.py new file mode 100644 index 0000000000..72939f82dc --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_generator.py @@ -0,0 +1,147 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import ( + List, + Optional, + Union, +) + +from m5.objects import ( + SpatterProcessingMode, + SrcClockDomain, + VoltageDomain, +) +from m5.stats import dump as dump_stats +from m5.stats import reset as reset_stats +from m5.util import fatal + +from ....utils.override import overrides +from ..abstract_generator import AbstractGenerator +from .spatter_generator_core import SpatterGeneratorCore +from .spatter_kernel import SpatterKernel + + +class SpatterGenerator(AbstractGenerator): + def __init__( + self, + num_cores: int = 1, + processing_mode: Union[SpatterProcessingMode, str] = "synchronous", + int_regfile_size: int = 384, + fp_regfile_size: int = 224, + request_gen_latency: int = 2, + request_gen_rate: int = 4, + request_buffer_entries: int = 32, + send_rate: int = 2, + clk_freq: Optional[str] = None, + ) -> None: + super().__init__( + cores=self._create_cores( + num_cores, + processing_mode, + int_regfile_size, + fp_regfile_size, + request_gen_latency, + request_gen_rate, + request_buffer_entries, + send_rate, + ) + ) + # no need for else block since it will intialize generator.clk_domain + # the clock domain of its closest ancestor in the SimObject tree. + if not clk_freq is None: + clock_domain = SrcClockDomain( + clock=clk_freq, voltage_domain=VoltageDomain() + ) + for generator in self.cores: + generator.clk_domain = clock_domain + + self._num_kernels = 0 + self._sync = processing_mode == "synchronous" + + def _create_cores( + self, + num_cores: int, + processing_mode: Union[SpatterProcessingMode, str], + int_regfile_size: int, + fp_regfile_size: int, + request_gen_latency: int, + request_gen_rate: int, + request_buffer_entries: int, + send_rate: int, + ) -> List[SpatterGeneratorCore]: + return [ + SpatterGeneratorCore( + processing_mode, + int_regfile_size, + fp_regfile_size, + request_gen_latency, + request_gen_rate, + request_buffer_entries, + send_rate, + ) + for _ in range(num_cores) + ] + + def add_kernel(self, kernels: List[SpatterKernel]) -> None: + assert len(kernels) == len(self.cores) + for core, kernel in zip(self.cores, kernels): + if kernel.empty(): + fatal( + f"Cannot add {kernel} since it's empty. " + "At the moment SpatterGenerator (or gem5::SpatterGen) " + "does not support adding empty kernels to cores. As a " + "temporary fix you can try adding 1 dummy element to the " + "trace. You can also set fix_empty_trace to True in the " + "constructor of the SpatterKernel which automatically " + "inserts a dummy element (0) to the trace." + ) + core.add_kernel(kernel) + self._num_kernels += 1 + + @overrides(AbstractGenerator) + def start_traffic(self) -> None: + for core in self.cores: + core.start_traffic() + + def _proceed_past_sync_point(self) -> None: + if not self._sync: + return + for core in self.cores: + core.generator.proceedPastSyncPoint() + + def handle_spatter_exit(self): + spatter_exits_observed = 0 + sync_points_observed = 0 + sync_points_expected = self._num_kernels if self._sync else 1 + while True: + spatter_exits_observed += 1 + if spatter_exits_observed % len(self.cores) == 0: + sync_points_observed += 1 + dump_stats() + reset_stats() + self._proceed_past_sync_point() + yield not (sync_points_observed < sync_points_expected) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py b/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py new file mode 100644 index 0000000000..50799eae84 --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_generator_core.py @@ -0,0 +1,73 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import Union + +from m5.objects import ( + Port, + SpatterGen, + SpatterProcessingMode, +) + +from ....utils.override import overrides +from ..abstract_core import AbstractCore +from ..abstract_generator_core import AbstractGeneratorCore +from .spatter_kernel import SpatterKernel + + +class SpatterGeneratorCore(AbstractGeneratorCore): + def __init__( + self, + processing_mode: Union[SpatterProcessingMode, str], + int_regfile_size: int, + fp_regfile_size: int, + request_gen_latency: int, + request_gen_rate: int, + request_buffer_entries: int, + send_rate: int, + ): + super().__init__() + self.generator = SpatterGen( + processing_mode=processing_mode, + int_regfile_size=int_regfile_size, + fp_regfile_size=fp_regfile_size, + request_gen_latency=request_gen_latency, + request_gen_rate=request_gen_rate, + request_buffer_entries=request_buffer_entries, + send_rate=send_rate, + ) + self._kernels = [] + + @overrides(AbstractCore) + def connect_dcache(self, port: Port) -> None: + self.generator.port = port + + def add_kernel(self, kernel: SpatterKernel) -> None: + self._kernels.append(kernel) + + def start_traffic(self) -> None: + for kernel in self._kernels: + self.generator.addKernel(*kernel.cxx_call_args()) diff --git a/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py b/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py new file mode 100644 index 0000000000..4cf0ee814a --- /dev/null +++ b/src/python/gem5/components/processors/spatter_gen/spatter_kernel.py @@ -0,0 +1,200 @@ +# Copyright (c) 2024 The Regents of the University of California +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from math import ceil +from typing import ( + List, + Tuple, +) + +from m5.objects import SpatterKernelType +from m5.params import Addr +from m5.util import inform + + +def parse_kernel(kernel: dict, default_delta=8) -> Tuple[int, int, str, List]: + delta = kernel.get("delta", default_delta) + if delta < 0: + inform( + f"Negative delta found: {delta}. Setting it to {default_delta}." + ) + delta = default_delta + count = kernel.get("count", 1) + type = kernel.get("kernel", None) + if type is None: + raise ValueError(f"Keyword 'kernel' not found.") + type = SpatterKernelType(type.lower()) + trace = kernel.get("pattern", []) + if len(trace) == 0: + raise ValueError(f"Empty 'pattern' found.") + return (delta, count, type, trace) + + +def partition_trace(original_trace, num_partitions, interleave_size): + partitions = [[] for _ in range(num_partitions)] + num_leaves = ceil(len(original_trace) / interleave_size) + for i in range(num_leaves): + lower_bound = i * interleave_size + upper_bound = min(lower_bound + interleave_size, len(original_trace)) + partitions[i % num_partitions] += original_trace[ + lower_bound:upper_bound + ] + return partitions + + +class SpatterKernel: + """This class encapsulates one kernel in a spatter trace. + A spatter trace is represented with a json file. + An example of a spatter trace can be found here: + https://github.com/hpcgarage/spatter/blob/main/standard-suite/app-traces/amg.json + Each trace may have multiple kernels. + Each kernel represents a code execution like below + for (int iteration = 0; iteration < count; iteration++) + { + for (int i = 0; i < N; i++) { + value[index[i] + iteration * delta] = rand(); // kernel: scatter + // OR + sum += value[index[i] + iteration * delta]; // kernel: gather + } + } + Where `delta` and `count` are fields in each kernel. + `kernel` is another field that determines whether the accesses to value + are loads or stores. + The field `pattern` stores the index array. + + This file provides two utility function to parse spatter traces: + parse_kernel: takes a dictionary and returns a tuple of + delta, count, type, and trace. + partition_trace: takes the original trace, number of partitions, + and interleave_size. + It returns a list of `num_partitions` partitions where each partition + is an list including interleaved elements from `original_trace`. + The elements in the `original_trace` are interleaved with a + granularity of `interleave_size`. + The code snippet below shows how to use these functions to create kernels. + generator = SpatterGenerator(num_cores) + + with open(trace_path, "r") as trace_file: + kernels = json.load(trace_file) + + for i, kernel in enumerate(kernels): + delta, count, type, og_trace = parse_kernel(kernel) + traces = partition_trace(og_trace, num_cores, 128) + kernels = [SpatterKernel( + kernel_id=i, + kernel_delta=delta, + kernel_count=count, + kernel_type=type, + kernel_trace=trace, + index_size=4, + base_index_addr=0, + value_size=8, + base_value_addr=0x400000000 + ) + for trace in traces + ] + generator.add_kernel(kernels) + + Args: + kernel_id (int): The ID of the kernel. + User defined, i.e. spatter traces don't have this field. + It's used to identify the kernel in the simulation. + kernel_delta (int): The delta value of the kernel. + `delta` from spatter trace. + kernel_count (int): The count value of the kernel. + `count` from spatter trace. + kernel_type (SpatterKernelType): The type of the kernel. + `kernel` from spatter trace. + kernel_trace (List[int]): The elements of the `index` array. + `pattern` from spatter trace. + index_size (int): The size of elements in `index`. + User defined, i.e. spatter traces don't have this field. + It represents the size of elements in the `index` array in code above. + base_index_addr (Addr): The base address of the index. + User defined, i.e. spatter traces don't have this field. + It represents the pointer to the `index` array in the code above. + value_size (int): The size of elements in `value`. + User defined, i.e. spatter traces don't have this field. + It represents the size of elements in the `value` array in code above. + base_value_addr (Addr): The base address of the value. + User defined, i.e. spatter traces don't have this field. + It represents the pointer to the `value` array in the code above. + """ + + def __init__( + self, + kernel_id: int, + kernel_delta: int, + kernel_count: int, + kernel_type: SpatterKernelType, + kernel_trace: List[int], + index_size: int, + base_index_addr: Addr, + value_size: int, + base_value_addr: Addr, + fix_empty_trace: bool = False, + ): + self._id = kernel_id + self._delta = kernel_delta + self._count = kernel_count + self._trace = kernel_trace + self._type = kernel_type + self._index_size = index_size + self._base_index_addr = base_index_addr + self._value_size = value_size + self._base_value_addr = base_value_addr + + if fix_empty_trace and len(kernel_trace) == 0: + inform( + "Empty trace found. Fixing it by adding a dummy element. " + "Also setting delta to 0 and count to 1.", + ) + self._trace = [0] + self._delta = 0 + self._count = 1 + + def empty(self): + return len(self._trace) == 0 + + def cxx_call_args(self): + return [ + self._id, + self._delta, + self._count, + self._type.getValue(), + self._index_size, + self._base_index_addr, + self._value_size, + self._base_value_addr, + self._trace, + ] + + def __str__(self): + return ( + f"SpatterKernel(id={self._id}, delta={self._delta}, " + f"count={self._count}, type={self._type}, " + f"trace[:8]={self._trace[:8]}" + ) diff --git a/src/python/gem5/simulate/exit_event.py b/src/python/gem5/simulate/exit_event.py index b902643a3f..5a0bb3d1d7 100644 --- a/src/python/gem5/simulate/exit_event.py +++ b/src/python/gem5/simulate/exit_event.py @@ -39,6 +39,7 @@ class ExitEvent(Enum): EXIT = "exit" # A standard vanilla exit. WORKBEGIN = "workbegin" # An exit because a ROI has been reached. WORKEND = "workend" # An exit because a ROI has ended. + SPATTER_EXIT = "spatter exit" # An exit because a spatter core has ended. SWITCHCPU = "switchcpu" # An exit needed to switch CPU cores. FAIL = "fail" # An exit because the simulation has failed. CHECKPOINT = "checkpoint" # An exit to load a checkpoint. @@ -115,6 +116,8 @@ class ExitEvent(Enum): elif exit_string.endswith("is finished updating the memory.\n"): # This is for the gups generator exit event return ExitEvent.EXIT + elif exit_string.endswith("received all expected responses."): + return ExitEvent.SPATTER_EXIT raise NotImplementedError( f"Exit event '{exit_string}' not implemented" ) diff --git a/src/python/gem5/simulate/exit_event_generators.py b/src/python/gem5/simulate/exit_event_generators.py index 4d18b4cee0..b237b064e2 100644 --- a/src/python/gem5/simulate/exit_event_generators.py +++ b/src/python/gem5/simulate/exit_event_generators.py @@ -36,6 +36,7 @@ from m5.util import warn from gem5.resources.looppoint import Looppoint from ..components.processors.abstract_processor import AbstractProcessor +from ..components.processors.spatter_gen import SpatterGenerator from ..components.processors.switchable_processor import SwitchableProcessor from ..resources.resource import SimpointResource @@ -221,3 +222,9 @@ def looppoint_save_checkpoint_generator( yield False yield True + + +def spatter_exit_generator(spatter_gen: SpatterGenerator): + while True: + assert isinstance(spatter_gen, SpatterGenerator) + yield from spatter_gen.handle_spatter_exit() diff --git a/src/python/gem5/simulate/simulator.py b/src/python/gem5/simulate/simulator.py index 5a5cf9af89..66f67d6ffb 100644 --- a/src/python/gem5/simulate/simulator.py +++ b/src/python/gem5/simulate/simulator.py @@ -53,6 +53,7 @@ from .exit_event_generators import ( reset_stats_generator, save_checkpoint_generator, skip_generator, + spatter_exit_generator, switch_generator, warn_default_decorator, ) @@ -281,6 +282,12 @@ class Simulator: "creating a checkpoint and continuing", )(), ExitEvent.FAIL: exit_generator(), + ExitEvent.SPATTER_EXIT: warn_default_decorator( + spatter_exit_generator, + "spatter exit", + "dumping and resetting stats after each sync point. " + "Note that there will be num_cores*sync_points spatter_exits.", + )(spatter_gen=board.get_processor()), ExitEvent.SWITCHCPU: warn_default_decorator( switch_generator, "switch CPU", @@ -518,9 +525,11 @@ class Simulator: self._board._pre_instantiate() root = Root( - full_system=self._full_system - if self._full_system is not None - else self._board.is_fullsystem(), + full_system=( + self._full_system + if self._full_system is not None + else self._board.is_fullsystem() + ), board=self._board, )