derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

configs: Add disjoint VIPER configuration

Browse Source 
The disjoint VIPER configuration creates completely disconnected CPU and
GPU Ruby networks which can communicate only via the PCI bus. Either
garnet or simple network can be used. This copies most of the Ruby setup
from Ruby.py's create_system since creating disjoint networks is not
possible using Ruby.py.

Change-Id: Ibc23aa592f56554d088667d8e309ecdeb306da68
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/53072
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
This commit is contained in:
Matthew Poremba
2021-10-20 13:06:00 -05:00
parent ea9b7ef6a2
commit 66dd94a0ee
4 changed files with 315 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
configs/example/gpufs/DisjointNetwork.py Normal file
View File

@@ -0,0 +1,107 @@
# Copyright (c) 2021 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from this
# software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from m5.objects import *
from m5.util import fatal
from importlib import *
from network import Network
class DisjointSimple(SimpleNetwork):
def __init__(self, ruby_system):
super(DisjointSimple, self).__init__()
self.netifs = []
self.routers = []
self.int_links = []
self.ext_links = []
self.ruby_system = ruby_system
def connectCPU(self, opts, controllers):
# Setup parameters for makeTopology call for CPU network
topo_module = import_module("topologies.%s" % opts.cpu_topology)
topo_class = getattr(topo_module, opts.cpu_topology)
_topo = topo_class(controllers)
_topo.makeTopology(opts, self, SimpleIntLink,
SimpleExtLink, Switch)
self.initSimple(opts, self.int_links, self.ext_links)
def connectGPU(self, opts, controllers):
# Setup parameters for makeTopology call for GPU network
topo_module = import_module("topologies.%s" % opts.gpu_topology)
topo_class = getattr(topo_module, opts.gpu_topology)
_topo = topo_class(controllers)
_topo.makeTopology(opts, self, SimpleIntLink,
SimpleExtLink, Switch)
self.initSimple(opts, self.int_links, self.ext_links)
def initSimple(self, opts, int_links, ext_links):
# Attach links to network
self.int_links = int_links
self.ext_links = ext_links
self.setup_buffers()
class DisjointGarnet(GarnetNetwork):
def __init__(self, ruby_system):
super(DisjointGarnet, self).__init__()
self.netifs = []
self.ruby_system = ruby_system
def connectCPU(self, opts, controllers):
# Setup parameters for makeTopology call for CPU network
topo_module = import_module("topologies.%s" % opts.cpu_topology)
topo_class = getattr(topo_module, opts.cpu_topology)
_topo = topo_class(controllers)
_topo.makeTopology(opts, self, GarnetIntLink,
GarnetExtLink, GarnetRouter)
Network.init_network(opts, self, GarnetNetworkInterface)
def connectGPU(self, opts, controllers):
# Setup parameters for makeTopology call
topo_module = import_module("topologies.%s" % opts.gpu_topology)
topo_class = getattr(topo_module, opts.gpu_topology)
_topo = topo_class(controllers)
_topo.makeTopology(opts, self, GarnetIntLink,
GarnetExtLink, GarnetRouter)
Network.init_network(opts, self, GarnetNetworkInterface)

193
configs/example/gpufs/Disjoint_VIPER.py Normal file
View File

@@ -0,0 +1,193 @@
# Copyright (c) 2021 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from this
# software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
from m5.defines import buildEnv
from m5.objects import *
from m5.util import fatal
from example.gpufs.DisjointNetwork import *
from ruby.GPU_VIPER import *
from ruby import Ruby
class DummySystem():
def __init__(self, mem_ranges):
self.mem_ctrls = []
self.mem_ranges = mem_ranges
class Disjoint_VIPER(RubySystem):
def __init__(self):
if buildEnv['PROTOCOL'] != "GPU_VIPER":
fatal("This ruby config only supports the GPU_VIPER protocol")
super(Disjoint_VIPER, self).__init__()
def create(self, options, system, piobus, dma_devices):
# Disjoint network topology
if "garnet" in options.network:
self.network_cpu = DisjointGarnet(self)
self.network_gpu = DisjointGarnet(self)
else:
self.network_cpu = DisjointSimple(self)
self.network_gpu = DisjointSimple(self)
# Construct CPU controllers
cpu_dir_nodes = \
construct_dirs(options, system, self, self.network_cpu)
(cp_sequencers, cp_cntrl_nodes) = \
construct_corepairs(options, system, self, self.network_cpu)
# Construct GPU controllers
(tcp_sequencers, tcp_cntrl_nodes) = \
construct_tcps(options, system, self, self.network_gpu)
(sqc_sequencers, sqc_cntrl_nodes) = \
construct_sqcs(options, system, self, self.network_gpu)
(scalar_sequencers, scalar_cntrl_nodes) = \
construct_scalars(options, system, self, self.network_gpu)
tcc_cntrl_nodes = \
construct_tccs(options, system, self, self.network_gpu)
# Construct CPU memories
Ruby.setup_memory_controllers(system, self, cpu_dir_nodes, options)
# Construct GPU memories
(gpu_dir_nodes, gpu_mem_ctrls) = \
construct_gpudirs(options, system, self, self.network_gpu)
# Configure the directories based on which network they are in
for cpu_dir_node in cpu_dir_nodes:
cpu_dir_node.CPUonly = True
cpu_dir_node.GPUonly = False
for gpu_dir_node in gpu_dir_nodes:
gpu_dir_node.CPUonly = False
gpu_dir_node.GPUonly = True
# Set access backing store if specified
if options.access_backing_store:
self.access_backing_store = True
# Assign the memory controllers to the system
cpu_abstract_mems = []
for mem_ctrl in system.mem_ctrls:
cpu_abstract_mems.append(mem_ctrl.dram)
system.memories = cpu_abstract_mems
gpu_abstract_mems = []
for mem_ctrl in gpu_mem_ctrls:
gpu_abstract_mems.append(mem_ctrl.dram)
system.pc.south_bridge.gpu.memories = gpu_abstract_mems
# Setup DMA controllers
gpu_dma_types = ["VegaPagetableWalker", "AMDGPUMemoryManager"]
cpu_dma_ctrls = []
gpu_dma_ctrls = []
dma_cntrls = []
for i, dma_device in enumerate(dma_devices):
dma_seq = DMASequencer(version=i, ruby_system=self)
dma_cntrl = DMA_Controller(version=i, dma_sequencer=dma_seq,
ruby_system=self)
# Handle inconsistently named ports on various DMA devices:
if not hasattr(dma_device, 'type'):
# IDE doesn't have a .type but seems like everything else does.
dma_seq.in_ports = dma_device
elif dma_device.type in gpu_dma_types:
dma_seq.in_ports = dma_device.port
else:
dma_seq.in_ports = dma_device.dma
if hasattr(dma_device, 'type') and \
dma_device.type in gpu_dma_types:
dma_cntrl.requestToDir = MessageBuffer(buffer_size=0)
dma_cntrl.requestToDir.out_port = self.network_gpu.in_port
dma_cntrl.responseFromDir = MessageBuffer(buffer_size=0)
dma_cntrl.responseFromDir.in_port = self.network_gpu.out_port
dma_cntrl.mandatoryQueue = MessageBuffer(buffer_size = 0)
gpu_dma_ctrls.append(dma_cntrl)
else:
dma_cntrl.requestToDir = MessageBuffer(buffer_size=0)
dma_cntrl.requestToDir.out_port = self.network_cpu.in_port
dma_cntrl.responseFromDir = MessageBuffer(buffer_size=0)
dma_cntrl.responseFromDir.in_port = self.network_cpu.out_port
dma_cntrl.mandatoryQueue = MessageBuffer(buffer_size = 0)
cpu_dma_ctrls.append(dma_cntrl)
dma_cntrls.append(dma_cntrl)
system.dma_cntrls = dma_cntrls
# Collect CPU and GPU controllers into seperate lists
cpu_cntrls = cpu_dir_nodes + cp_cntrl_nodes + cpu_dma_ctrls
gpu_cntrls = tcp_cntrl_nodes + sqc_cntrl_nodes + \
scalar_cntrl_nodes + tcc_cntrl_nodes + gpu_dma_ctrls + \
gpu_dir_nodes
# Setup number of vnets
self.number_of_virtual_networks = 11
self.network_cpu.number_of_virtual_networks = 11
self.network_gpu.number_of_virtual_networks = 11
# Set up the disjoint topology
self.network_cpu.connectCPU(options, cpu_cntrls)
self.network_gpu.connectGPU(options, gpu_cntrls)
# Create port proxy for connecting system port. System port is used
# for loading from outside guest, e.g., binaries like vmlinux.
system.sys_port_proxy = RubyPortProxy(ruby_system = self)
system.sys_port_proxy.pio_request_port = piobus.cpu_side_ports
system.system_port = system.sys_port_proxy.in_ports
# Only CPU sequencers connect to PIO bus. This acts as the "default"
# destination for unknown address ranges. PCIe requests fall under
# this category.
for i in range(len(cp_sequencers)):
cp_sequencers[i].pio_request_port = piobus.cpu_side_ports
cp_sequencers[i].mem_request_port = piobus.cpu_side_ports
# Note: only used in X86
cp_sequencers[i].pio_response_port = piobus.mem_side_ports
# Setup ruby port. Both CPU and GPU are actually connected here.
all_sequencers = cp_sequencers + tcp_sequencers + \
sqc_sequencers + scalar_sequencers
self._cpu_ports = all_sequencers
self.num_of_sequencers = len(all_sequencers)

7
configs/example/gpufs/runfs.py
View File

@@ -71,7 +71,12 @@ def addRunFSOptions(parser):
help="Take a checkpoint before driver sends MMIOs. "
"This is used to switch out of KVM mode and into "
"timing mode required to read the VGA ROM on boot.")
parser.add_argument("--cpu-topology", type=str, default="Crossbar",
help="Network topology to use for CPU side. "
"Check configs/topologies for complete set")
parser.add_argument("--gpu-topology", type=str, default="Crossbar",
help="Network topology to use for GPU side. "
"Check configs/topologies for complete set")
def runGpuFSSystem(args):
'''

11
configs/example/gpufs/system/system.py
View File

@@ -36,6 +36,8 @@ from common.FSConfig import *
from common import Simulation
from ruby import Ruby
from example.gpufs.Disjoint_VIPER import *
def makeGpuFSSystem(args):
# Boot options are standard gem5 options plus:
# - Framebuffer device emulation 0 to reduce driver code paths.
@@ -133,6 +135,9 @@ def makeGpuFSSystem(args):
system._dma_ports.append(sdma1)
system._dma_ports.append(device_ih)
system._dma_ports.append(pm4_pkt_proc)
system._dma_ports.append(gpu_mem_mgr)
system._dma_ports.append(sdma0_pt_walker)
system._dma_ports.append(sdma1_pt_walker)
gpu_hsapp.pio = system.iobus.mem_side_ports
gpu_cmd_proc.pio = system.iobus.mem_side_ports
@@ -143,8 +148,10 @@ def makeGpuFSSystem(args):
pm4_pkt_proc.pio = system.iobus.mem_side_ports
# Create Ruby system using Ruby.py for now
Ruby.create_system(args, True, system, system.iobus,
system._dma_ports)
#Ruby.create_system(args, True, system, system.iobus,
# system._dma_ports)
system.ruby = Disjoint_VIPER()
system.ruby.create(args, system, system.iobus, system._dma_ports)
# Create a seperate clock domain for Ruby
system.ruby.clk_domain = SrcClockDomain(clock = args.ruby_clock,