diff --git a/configs/example/gpufs/DisjointNetwork.py b/configs/example/gpufs/DisjointNetwork.py new file mode 100644 index 0000000000..e1838bb209 --- /dev/null +++ b/configs/example/gpufs/DisjointNetwork.py @@ -0,0 +1,107 @@ +# Copyright (c) 2021 Advanced Micro Devices, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from m5.objects import * +from m5.util import fatal + +from importlib import * + +from network import Network + +class DisjointSimple(SimpleNetwork): + + def __init__(self, ruby_system): + super(DisjointSimple, self).__init__() + + self.netifs = [] + self.routers = [] + self.int_links = [] + self.ext_links = [] + self.ruby_system = ruby_system + + def connectCPU(self, opts, controllers): + + # Setup parameters for makeTopology call for CPU network + topo_module = import_module("topologies.%s" % opts.cpu_topology) + topo_class = getattr(topo_module, opts.cpu_topology) + _topo = topo_class(controllers) + _topo.makeTopology(opts, self, SimpleIntLink, + SimpleExtLink, Switch) + + self.initSimple(opts, self.int_links, self.ext_links) + + def connectGPU(self, opts, controllers): + + # Setup parameters for makeTopology call for GPU network + topo_module = import_module("topologies.%s" % opts.gpu_topology) + topo_class = getattr(topo_module, opts.gpu_topology) + _topo = topo_class(controllers) + _topo.makeTopology(opts, self, SimpleIntLink, + SimpleExtLink, Switch) + + self.initSimple(opts, self.int_links, self.ext_links) + + + def initSimple(self, opts, int_links, ext_links): + + # Attach links to network + self.int_links = int_links + self.ext_links = ext_links + + self.setup_buffers() + +class DisjointGarnet(GarnetNetwork): + + def __init__(self, ruby_system): + super(DisjointGarnet, self).__init__() + + self.netifs = [] + self.ruby_system = ruby_system + + def connectCPU(self, opts, controllers): + + # Setup parameters for makeTopology call for CPU network + topo_module = import_module("topologies.%s" % opts.cpu_topology) + topo_class = getattr(topo_module, opts.cpu_topology) + _topo = topo_class(controllers) + _topo.makeTopology(opts, self, GarnetIntLink, + GarnetExtLink, GarnetRouter) + + Network.init_network(opts, self, GarnetNetworkInterface) + + def connectGPU(self, opts, controllers): + + # Setup parameters for makeTopology call + topo_module = import_module("topologies.%s" % opts.gpu_topology) + topo_class = getattr(topo_module, opts.gpu_topology) + _topo = topo_class(controllers) + _topo.makeTopology(opts, self, GarnetIntLink, + GarnetExtLink, GarnetRouter) + + Network.init_network(opts, self, GarnetNetworkInterface) diff --git a/configs/example/gpufs/Disjoint_VIPER.py b/configs/example/gpufs/Disjoint_VIPER.py new file mode 100644 index 0000000000..17729b0823 --- /dev/null +++ b/configs/example/gpufs/Disjoint_VIPER.py @@ -0,0 +1,193 @@ +# Copyright (c) 2021 Advanced Micro Devices, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from m5.defines import buildEnv +from m5.objects import * +from m5.util import fatal + +from example.gpufs.DisjointNetwork import * +from ruby.GPU_VIPER import * +from ruby import Ruby + + +class DummySystem(): + def __init__(self, mem_ranges): + + self.mem_ctrls = [] + self.mem_ranges = mem_ranges + + +class Disjoint_VIPER(RubySystem): + def __init__(self): + if buildEnv['PROTOCOL'] != "GPU_VIPER": + fatal("This ruby config only supports the GPU_VIPER protocol") + + super(Disjoint_VIPER, self).__init__() + + def create(self, options, system, piobus, dma_devices): + + # Disjoint network topology + if "garnet" in options.network: + self.network_cpu = DisjointGarnet(self) + self.network_gpu = DisjointGarnet(self) + else: + self.network_cpu = DisjointSimple(self) + self.network_gpu = DisjointSimple(self) + + + # Construct CPU controllers + cpu_dir_nodes = \ + construct_dirs(options, system, self, self.network_cpu) + (cp_sequencers, cp_cntrl_nodes) = \ + construct_corepairs(options, system, self, self.network_cpu) + + # Construct GPU controllers + (tcp_sequencers, tcp_cntrl_nodes) = \ + construct_tcps(options, system, self, self.network_gpu) + (sqc_sequencers, sqc_cntrl_nodes) = \ + construct_sqcs(options, system, self, self.network_gpu) + (scalar_sequencers, scalar_cntrl_nodes) = \ + construct_scalars(options, system, self, self.network_gpu) + tcc_cntrl_nodes = \ + construct_tccs(options, system, self, self.network_gpu) + + # Construct CPU memories + Ruby.setup_memory_controllers(system, self, cpu_dir_nodes, options) + + # Construct GPU memories + (gpu_dir_nodes, gpu_mem_ctrls) = \ + construct_gpudirs(options, system, self, self.network_gpu) + + # Configure the directories based on which network they are in + for cpu_dir_node in cpu_dir_nodes: + cpu_dir_node.CPUonly = True + cpu_dir_node.GPUonly = False + for gpu_dir_node in gpu_dir_nodes: + gpu_dir_node.CPUonly = False + gpu_dir_node.GPUonly = True + + # Set access backing store if specified + if options.access_backing_store: + self.access_backing_store = True + + # Assign the memory controllers to the system + cpu_abstract_mems = [] + for mem_ctrl in system.mem_ctrls: + cpu_abstract_mems.append(mem_ctrl.dram) + system.memories = cpu_abstract_mems + + gpu_abstract_mems = [] + for mem_ctrl in gpu_mem_ctrls: + gpu_abstract_mems.append(mem_ctrl.dram) + system.pc.south_bridge.gpu.memories = gpu_abstract_mems + + # Setup DMA controllers + gpu_dma_types = ["VegaPagetableWalker", "AMDGPUMemoryManager"] + + cpu_dma_ctrls = [] + gpu_dma_ctrls = [] + dma_cntrls = [] + for i, dma_device in enumerate(dma_devices): + dma_seq = DMASequencer(version=i, ruby_system=self) + dma_cntrl = DMA_Controller(version=i, dma_sequencer=dma_seq, + ruby_system=self) + + # Handle inconsistently named ports on various DMA devices: + if not hasattr(dma_device, 'type'): + # IDE doesn't have a .type but seems like everything else does. + dma_seq.in_ports = dma_device + elif dma_device.type in gpu_dma_types: + dma_seq.in_ports = dma_device.port + else: + dma_seq.in_ports = dma_device.dma + + if hasattr(dma_device, 'type') and \ + dma_device.type in gpu_dma_types: + dma_cntrl.requestToDir = MessageBuffer(buffer_size=0) + dma_cntrl.requestToDir.out_port = self.network_gpu.in_port + dma_cntrl.responseFromDir = MessageBuffer(buffer_size=0) + dma_cntrl.responseFromDir.in_port = self.network_gpu.out_port + dma_cntrl.mandatoryQueue = MessageBuffer(buffer_size = 0) + + gpu_dma_ctrls.append(dma_cntrl) + else: + dma_cntrl.requestToDir = MessageBuffer(buffer_size=0) + dma_cntrl.requestToDir.out_port = self.network_cpu.in_port + dma_cntrl.responseFromDir = MessageBuffer(buffer_size=0) + dma_cntrl.responseFromDir.in_port = self.network_cpu.out_port + dma_cntrl.mandatoryQueue = MessageBuffer(buffer_size = 0) + + cpu_dma_ctrls.append(dma_cntrl) + + dma_cntrls.append(dma_cntrl) + + system.dma_cntrls = dma_cntrls + + + # Collect CPU and GPU controllers into seperate lists + cpu_cntrls = cpu_dir_nodes + cp_cntrl_nodes + cpu_dma_ctrls + gpu_cntrls = tcp_cntrl_nodes + sqc_cntrl_nodes + \ + scalar_cntrl_nodes + tcc_cntrl_nodes + gpu_dma_ctrls + \ + gpu_dir_nodes + + + # Setup number of vnets + self.number_of_virtual_networks = 11 + self.network_cpu.number_of_virtual_networks = 11 + self.network_gpu.number_of_virtual_networks = 11 + + + # Set up the disjoint topology + self.network_cpu.connectCPU(options, cpu_cntrls) + self.network_gpu.connectGPU(options, gpu_cntrls) + + + # Create port proxy for connecting system port. System port is used + # for loading from outside guest, e.g., binaries like vmlinux. + system.sys_port_proxy = RubyPortProxy(ruby_system = self) + system.sys_port_proxy.pio_request_port = piobus.cpu_side_ports + system.system_port = system.sys_port_proxy.in_ports + + + # Only CPU sequencers connect to PIO bus. This acts as the "default" + # destination for unknown address ranges. PCIe requests fall under + # this category. + for i in range(len(cp_sequencers)): + cp_sequencers[i].pio_request_port = piobus.cpu_side_ports + cp_sequencers[i].mem_request_port = piobus.cpu_side_ports + + # Note: only used in X86 + cp_sequencers[i].pio_response_port = piobus.mem_side_ports + + + # Setup ruby port. Both CPU and GPU are actually connected here. + all_sequencers = cp_sequencers + tcp_sequencers + \ + sqc_sequencers + scalar_sequencers + self._cpu_ports = all_sequencers + self.num_of_sequencers = len(all_sequencers) diff --git a/configs/example/gpufs/runfs.py b/configs/example/gpufs/runfs.py index e5acf05372..8edf54404f 100644 --- a/configs/example/gpufs/runfs.py +++ b/configs/example/gpufs/runfs.py @@ -71,7 +71,12 @@ def addRunFSOptions(parser): help="Take a checkpoint before driver sends MMIOs. " "This is used to switch out of KVM mode and into " "timing mode required to read the VGA ROM on boot.") - + parser.add_argument("--cpu-topology", type=str, default="Crossbar", + help="Network topology to use for CPU side. " + "Check configs/topologies for complete set") + parser.add_argument("--gpu-topology", type=str, default="Crossbar", + help="Network topology to use for GPU side. " + "Check configs/topologies for complete set") def runGpuFSSystem(args): ''' diff --git a/configs/example/gpufs/system/system.py b/configs/example/gpufs/system/system.py index 8ea2cd5d54..8af423dc89 100644 --- a/configs/example/gpufs/system/system.py +++ b/configs/example/gpufs/system/system.py @@ -36,6 +36,8 @@ from common.FSConfig import * from common import Simulation from ruby import Ruby +from example.gpufs.Disjoint_VIPER import * + def makeGpuFSSystem(args): # Boot options are standard gem5 options plus: # - Framebuffer device emulation 0 to reduce driver code paths. @@ -133,6 +135,9 @@ def makeGpuFSSystem(args): system._dma_ports.append(sdma1) system._dma_ports.append(device_ih) system._dma_ports.append(pm4_pkt_proc) + system._dma_ports.append(gpu_mem_mgr) + system._dma_ports.append(sdma0_pt_walker) + system._dma_ports.append(sdma1_pt_walker) gpu_hsapp.pio = system.iobus.mem_side_ports gpu_cmd_proc.pio = system.iobus.mem_side_ports @@ -143,8 +148,10 @@ def makeGpuFSSystem(args): pm4_pkt_proc.pio = system.iobus.mem_side_ports # Create Ruby system using Ruby.py for now - Ruby.create_system(args, True, system, system.iobus, - system._dma_ports) + #Ruby.create_system(args, True, system, system.iobus, + # system._dma_ports) + system.ruby = Disjoint_VIPER() + system.ruby.create(args, system, system.iobus, system._dma_ports) # Create a seperate clock domain for Ruby system.ruby.clk_domain = SrcClockDomain(clock = args.ruby_clock,