derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

stdlib: Improving synthetic traffic generation.

Browse Source 
This change adds a new traffic generator module to the standard
library that can read a .cfg file describing the traffic pattern
as a state machine. It wraps around the TrafficGen SimObject.
In addition it adds a method to ComplexGenerator to set the
traffic from outside using python generators like the example
found in configs/dram/sweep.py.

Change-Id: I5989bb900d26091e6e0e85ea63c741441b72069c
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/62473
Maintainer: Bobby Bruce <bbruce@ucdavis.edu>
Reviewed-by: Bobby Bruce <bbruce@ucdavis.edu>
Tested-by: kokoro <noreply+kokoro@google.com>
This commit is contained in:
Mahyar Samani
2022-08-17 16:54:52 -07:00
parent 26ea5a1c72
commit 8ba46bafb0
15 changed files with 300 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/python/SConscript
View File

@@ -178,6 +178,8 @@ PySource('gem5.components.processors',
'gem5/components/processors/abstract_core.py')
PySource('gem5.components.processors',
'gem5/components/processors/abstract_generator_core.py')
PySource('gem5.components.processors',
'gem5/components/processors/abstract_generator.py')
PySource('gem5.components.processors',
'gem5/components/processors/abstract_processor.py')
PySource('gem5.components.processors',
@@ -215,6 +217,10 @@ PySource('gem5.components.processors',
PySource('gem5.components.processors',
'gem5/components/processors/switchable_processor.py')
PySource('gem5.utils', 'gem5/utils/simpoint.py')
PySource('gem5.components.processors',
'gem5/components/processors/traffic_generator_core.py')
PySource('gem5.components.processors',
'gem5/components/processors/traffic_generator.py')
PySource('gem5.prebuilt', 'gem5/prebuilt/__init__.py')
PySource('gem5.prebuilt.demo', 'gem5/prebuilt/demo/__init__.py')
PySource('gem5.prebuilt.demo', 'gem5/prebuilt/demo/x86_demo_board.py')

67
src/python/gem5/components/processors/abstract_generator.py Normal file
View File

@@ -0,0 +1,67 @@
# Copyright (c) 2022 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from abc import abstractmethod
from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .abstract_generator_core import AbstractGeneratorCore
from .abstract_processor import AbstractProcessor
from ..boards.abstract_board import AbstractBoard
from typing import List
class AbstractGenerator(AbstractProcessor):
"""The abstract generator
It defines the external interface of every generator component.
"""
def __init__(self, cores: List[AbstractGeneratorCore]) -> None:
"""
Create a list of AbstractGeneratorCore (which is an AbstractCore),
to pass to the constructor of the AbstractProcessor. Due to the
different prototypes for the constructor of different generator types
inputs are noted as *args. This way the abstract method _create_cores
could be called without AbstractGenerator having to know what the
prototype for the constructor of the inheriting class is. It also
limits the _create_cores function to only using positional arguments.
keyword (optional arguments) are still allowable in the constructor of
the inheriting classes.
"""
super().__init__(cores=cores)
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@abstractmethod
def start_traffic(self) -> None:
"""
Depending on what the internal generator core for inheriting classes is
this method needs to be implemented in detail or implmeneted as pass.
"""
raise NotImplementedError

12
src/python/gem5/components/processors/abstract_generator_core.py
View File

@@ -25,13 +25,12 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from abc import abstractmethod
from m5.objects import Port, PortTerminator
from ...utils.override import overrides
from .cpu_types import CPUTypes
from .abstract_core import AbstractCore
from ...isas import ISA
from ...utils.requires import requires
from typing import Optional
@@ -102,3 +101,12 @@ class AbstractGeneratorCore(AbstractCore):
connect them to walker ports. Just pass here.
"""
pass
@abstractmethod
def start_traffic(self):
"""
External interface to start generating the trace of addresses.
Depending on what SimObject is wrapped by this component this method
might need be implemented.
"""
raise NotImplementedError

32
src/python/gem5/components/processors/complex_generator.py
View File

@@ -25,18 +25,15 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .complex_generator_core import ComplexGeneratorCore
from .abstract_generator import AbstractGenerator
from .abstract_processor import AbstractProcessor
from ..boards.abstract_board import AbstractBoard
from typing import Iterator, List, Any
class ComplexGenerator(AbstractProcessor):
class ComplexGenerator(AbstractGenerator):
def __init__(self, num_cores: int = 1) -> None:
super().__init__(
cores=[ComplexGeneratorCore() for i in range(num_cores)]
)
super().__init__(cores=self._create_cores(num_cores=num_cores))
"""The complex generator
This class defines an external interface to create a list of complex
@@ -45,9 +42,11 @@ class ComplexGenerator(AbstractProcessor):
:param num_cores: The number of complex generator cores to create.
"""
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
def _create_cores(self, num_cores: int) -> List[ComplexGeneratorCore]:
"""
Create a list of ComplexGeneratorCore.
"""
return [ComplexGeneratorCore() for _ in range(num_cores)]
def add_linear(
self,
@@ -127,6 +126,19 @@ class ComplexGenerator(AbstractProcessor):
data_limit,
)
def set_traffic_from_python_generator(
self, generator: Iterator[Any]
) -> None:
"""
Sets the traffic pattern defined by generator argument.
:param generator: A python generator object that creates traffic
patterns through calls to methods of PyTrafficGen.
"""
for core in self.cores:
core.set_traffic_from_python_generator(generator)
@overrides(AbstractGenerator)
def start_traffic(self) -> None:
"""
This function will start the traffic at the top of the traffic list. It

21
src/python/gem5/components/processors/complex_generator_core.py
View File

@@ -24,6 +24,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from typing import Iterator, Any
from m5.ticks import fromSeconds
from m5.util.convert import toLatency, toMemoryBandwidth
from m5.objects import PyTrafficGen, Port
@@ -178,6 +179,7 @@ class ComplexGeneratorCore(AbstractGeneratorCore):
self._traffic_params = self._traffic_params + [param]
self._traffic_set = False
@overrides(AbstractGeneratorCore)
def start_traffic(self) -> None:
"""
This function first checks if there are any pending traffics that
@@ -239,6 +241,25 @@ class ComplexGeneratorCore(AbstractGeneratorCore):
self._traffic_set = True
def set_traffic_from_python_generator(
self, python_generator: Iterator[Any]
) -> None:
"""
Function to set the traffic from a user defined python generator.
The generator should only only assume one input argument (positional)
for the actual PyTrafficGen object to create the traffic. This is possible
either through using a generator with hardcoded parameters in the
function calls to PyTrafficGen methods or by compiling a flexible
python generator into a generator object with only one
input argument (positional) using functools.partial.
:param generator: A python generator object that creates traffic
patterns through calls to methods of PyTrafficGen.
"""
if not self._traffic_set:
self._set_traffic()
self._traffic.append(python_generator(self.generator))
def _create_linear_traffic(
self,
duration: str,

19
src/python/gem5/components/processors/gups_generator.py
View File

@@ -26,16 +26,15 @@
from typing import Optional
from m5.objects import Addr
from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .abstract_processor import AbstractProcessor
from ..boards.abstract_board import AbstractBoard
from .abstract_generator import AbstractGenerator
from .gups_generator_core import GUPSGeneratorCore
class GUPSGenerator(AbstractProcessor):
class GUPSGenerator(AbstractGenerator):
def __init__(
self,
start_addr: Addr,
@@ -68,12 +67,10 @@ class GUPSGenerator(AbstractProcessor):
]
)
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@overrides(AbstractGenerator)
def start_traffic(self):
# This function should be implemented so that GUPSGenerator could be
# used in the same scripts that use LinearGenerator, RandomGenerator,
# and ComplexGenrator
"""
Since GUPSGeneratorCore does not need a call to start_traffic to
start generation. This function is just pass.
"""
pass

3
src/python/gem5/components/processors/gups_generator_core.py
View File

@@ -27,6 +27,7 @@
from typing import Optional
from ...utils.override import overrides
from .abstract_core import AbstractCore
from .abstract_generator_core import AbstractGeneratorCore
from m5.objects import Port, GUPSGen, Addr, SrcClockDomain, VoltageDomain
@@ -52,6 +53,6 @@ class GUPSGeneratorCore(AbstractGeneratorCore):
)
self.generator.clk_domain = clock_domain
@overrides(AbstractGeneratorCore)
@overrides(AbstractCore)
def connect_dcache(self, port: Port) -> None:
self.generator.port = port

20
src/python/gem5/components/processors/gups_generator_ep.py
View File

@@ -27,15 +27,13 @@
from typing import Optional
from m5.objects import Addr
from ..boards.mem_mode import MemMode
from ...utils.override import overrides
from m5.util.convert import toMemorySize
from .abstract_processor import AbstractProcessor
from ..boards.abstract_board import AbstractBoard
from .abstract_generator import AbstractGenerator
from .gups_generator_core import GUPSGeneratorCore
class GUPSGeneratorEP(AbstractProcessor):
class GUPSGeneratorEP(AbstractGenerator):
def __init__(
self,
num_cores: int,
@@ -73,7 +71,7 @@ class GUPSGeneratorEP(AbstractProcessor):
start_addr: Addr,
mem_size: str,
update_limit: int,
clk_freq: Optional[str],
clk_freq: str,
):
"""
Helper function to create cores.
@@ -91,12 +89,10 @@ class GUPSGeneratorEP(AbstractProcessor):
for i in range(num_cores)
]
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@overrides(AbstractGenerator)
def start_traffic(self):
# This function should be implemented so that GUPSGeneratorEP could be
# used in the same scripts that use LinearGenerator, RandomGenerator,
# and ComplexGenrator
"""
Since GUPSGeneratorCore does not need a call to start_traffic to
start generation. This function is just pass.
"""
pass

21
src/python/gem5/components/processors/gups_generator_par.py
View File

@@ -30,12 +30,12 @@ from m5.objects import Addr
from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .abstract_processor import AbstractProcessor
from .abstract_generator import AbstractGenerator
from ..boards.abstract_board import AbstractBoard
from .gups_generator_core import GUPSGeneratorCore
class GUPSGeneratorPAR(AbstractProcessor):
class GUPSGeneratorPAR(AbstractGenerator):
def __init__(
self,
num_cores: int,
@@ -73,11 +73,8 @@ class GUPSGeneratorPAR(AbstractProcessor):
start_addr: Addr,
mem_size: str,
update_limit: int,
clk_freq: Optional[str],
clk_freq: str,
):
"""
Helper function to create cores.
"""
return [
GUPSGeneratorCore(
start_addr=start_addr,
@@ -88,12 +85,10 @@ class GUPSGeneratorPAR(AbstractProcessor):
for _ in range(num_cores)
]
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@overrides(AbstractGenerator)
def start_traffic(self):
# This function should be implemented so that GUPSGeneratorPAR could be
# used in the same scripts that use LinearGenerator, RandomGenerator,
# and ComplexGenrator
"""
Since GUPSGeneratorCore does not need a call to start_traffic to
start generation. This function is just pass.
"""
pass

33
src/python/gem5/components/processors/linear_generator.py
View File

@@ -25,16 +25,13 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .linear_generator_core import LinearGeneratorCore
from .abstract_processor import AbstractProcessor
from ..boards.abstract_board import AbstractBoard
from .abstract_generator import AbstractGenerator
from typing import List
class LinearGenerator(AbstractProcessor):
class LinearGenerator(AbstractGenerator):
def __init__(
self,
num_cores: int = 1,
@@ -81,14 +78,14 @@ class LinearGenerator(AbstractProcessor):
def _create_cores(
self,
num_cores,
duration,
rate,
block_size,
min_addr,
max_addr,
rd_perc,
data_limit,
num_cores: int,
duration: str,
rate: str,
block_size: int,
min_addr: int,
max_addr: int,
rd_perc: int,
data_limit: int,
) -> List[LinearGeneratorCore]:
"""
The helper function to create the cores for the generator, it will use
@@ -104,16 +101,10 @@ class LinearGenerator(AbstractProcessor):
rd_perc=rd_perc,
data_limit=data_limit,
)
for i in range(num_cores)
for _ in range(num_cores)
]
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@overrides(AbstractGenerator)
def start_traffic(self) -> None:
"""
This function will start the assigned traffic to this generator.
"""
for core in self.cores:
core.start_traffic()

5
src/python/gem5/components/processors/linear_generator_core.py
View File

@@ -112,10 +112,7 @@ class LinearGeneratorCore(AbstractGeneratorCore):
)
yield self.generator.createExit(0)
@overrides(AbstractGeneratorCore)
def start_traffic(self) -> None:
"""
A call to this function will start generating the traffic, this call
should happen before m5.simulate() and after m5.instantiate()
"""
self._set_traffic()
self.generator.start(self._traffic)

27
src/python/gem5/components/processors/random_generator.py
View File

@@ -28,13 +28,13 @@ from ...utils.override import overrides
from ..boards.mem_mode import MemMode
from .random_generator_core import RandomGeneratorCore
from .abstract_processor import AbstractProcessor
from .abstract_generator import AbstractGenerator
from ..boards.abstract_board import AbstractBoard
from typing import List
class RandomGenerator(AbstractProcessor):
class RandomGenerator(AbstractGenerator):
def __init__(
self,
num_cores: int = 1,
@@ -81,14 +81,14 @@ class RandomGenerator(AbstractProcessor):
def _create_cores(
self,
num_cores,
duration,
rate,
block_size,
min_addr,
max_addr,
rd_perc,
data_limit,
num_cores: int,
duration: str,
rate: str,
block_size: int,
min_addr: int,
max_addr: int,
rd_perc: int,
data_limit: int,
) -> List[RandomGeneratorCore]:
"""
The helper function to create the cores for the generator, it will use
@@ -104,13 +104,10 @@ class RandomGenerator(AbstractProcessor):
rd_perc=rd_perc,
data_limit=data_limit,
)
for i in range(num_cores)
for _ in range(num_cores)
]
@overrides(AbstractProcessor)
def incorporate_processor(self, board: AbstractBoard) -> None:
board.set_mem_mode(MemMode.TIMING)
@overrides(AbstractGenerator)
def start_traffic(self) -> None:
"""
This function will start the assigned traffic to this generator.

5
src/python/gem5/components/processors/random_generator_core.py
View File

@@ -112,10 +112,7 @@ class RandomGeneratorCore(AbstractGeneratorCore):
)
yield self.generator.createExit(0)
@overrides(AbstractGeneratorCore)
def start_traffic(self) -> None:
"""
A call to this function will start generating the traffic, this call
should happen before m5.simulate() and after m5.instantiate().
"""
self._set_traffic()
self.generator.start(self._traffic)

67
src/python/gem5/components/processors/traffic_generator.py Normal file
View File

@@ -0,0 +1,67 @@
# Copyright (c) 2022 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from ...utils.override import overrides
from .traffic_generator_core import TrafficGeneratorCore
from .abstract_generator import AbstractGenerator
from typing import List
class TrafficGenerator(AbstractGenerator):
def __init__(
self,
config_file_list: List[str],
) -> None:
super().__init__(
cores=self._create_cores(config_file_list=config_file_list)
)
"""The traffic generator
This class defines an external interface to create a list of traffic
generator cores that could replace the processing cores in a board.
:param config_file_list: A list containing the path to configuration
file each describing the traffic pattern that should be created by
each core of the generator.
"""
def _create_cores(
self, config_file_list: List[str]
) -> List[TrafficGeneratorCore]:
"""
The helper function to create the cores for the generator, it will use
the same inputs as the constructor function.
"""
return [
TrafficGeneratorCore(config_file)
for config_file in config_file_list
]
@overrides(AbstractGenerator)
def start_traffic(self) -> None:
pass

55
src/python/gem5/components/processors/traffic_generator_core.py Normal file
View File

@@ -0,0 +1,55 @@
# Copyright (c) 2022 The Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from m5.objects import Port, TrafficGen
from .abstract_core import AbstractCore
from .abstract_generator_core import AbstractGeneratorCore
from ...utils.override import overrides
class TrafficGeneratorCore(AbstractGeneratorCore):
"""The traffic generator core interface.
This class defines the interface for a generator core that will create
a compound traffic specified by the parameters below. It uses
TrafficGen to create the traffic.
:param config_file: path to the configuration file specifying the
pattern of traffic.
"""
def __init__(self, config_file: str):
"""
Create a TrafficGen SimObject as the core of this component.
"""
super().__init__()
self.generator = TrafficGen(config_file=config_file)
@overrides(AbstractCore)
def connect_dcache(self, port: Port) -> None:
self.generator.port = port