arch-arm, configs: Treat the bootloader rom as cacheable memory

Prior to this changeset the bootloader rom (instantiated as a
SimpleMemory) in ruby Arm systems was treated as an IO device and it
was fronted by a DMA controller. This changeset moves the bootloader
rom and adds it to the system as another memory with a dedicated
directory controller.

Change-Id: I094fed031cdef7f77a939d94f948d967b349b7e0
Reviewed-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/8741
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>

configs/common/FSConfig.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2010-2012, 2015-2017 ARM Limited
+# Copyright (c) 2010-2012, 2015-2018 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -326,8 +326,10 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
         # iobus, as gem5's membus is only used for initialization and
         # SST doesn't use it.  Attaching nvmem to iobus solves this issue.
         # During initialization, system_port -> membus -> iobus -> nvmem.
-        if external_memory or ruby:
+        if external_memory:
             self.realview.setupBootLoader(self.iobus,  self, binary)
+        elif ruby:
+            self.realview.setupBootLoader(None, self, binary)
         else:
             self.realview.setupBootLoader(self.membus, self, binary)
         self.gic_cpu_addr = self.realview.gic.cpu_addr
@@ -386,8 +388,6 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None,
     elif ruby:
         self._dma_ports = [ ]
         self.realview.attachOnChipIO(self.iobus, dma_ports=self._dma_ports)
-        # Force Ruby to treat the boot ROM as an IO device.
-        self.realview.nvmem.in_addr_map = False
         self.realview.attachIO(self.iobus, dma_ports=self._dma_ports)
     else:
         self.realview.attachOnChipIO(self.membus, self.bridge)

configs/example/fs.py

@@ -152,8 +152,9 @@ def build_test_system(np):
         test_sys.kvm_vm = KvmVM()
 
     if options.ruby:
+        bootmem = getattr(test_sys, 'bootmem', None)
         Ruby.create_system(options, True, test_sys, test_sys.iobus,
-                           test_sys._dma_ports)
+                           test_sys._dma_ports, bootmem)
 
         # Create a seperate clock domain for Ruby
         test_sys.ruby.clk_domain = SrcClockDomain(clock = options.ruby_clock,

configs/ruby/GPU_RfO.py

@@ -427,7 +427,8 @@ def define_options(parser):
     parser.add_option("--tcc-dir-factor", type='int', default=4,
                       help="TCCdir size = factor *(TCPs + TCC)")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_RfO':
         panic("This script requires the GPU_RfO protocol to be built.")
 

configs/ruby/GPU_VIPER.py

@@ -390,7 +390,8 @@ def define_options(parser):
     parser.add_option("--noL1", action = "store_true", default = False,
                       help = "bypassL1")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER':
         panic("This script requires the GPU_VIPER protocol to be built.")
 

configs/ruby/GPU_VIPER_Baseline.py

@@ -373,7 +373,8 @@ def define_options(parser):
     parser.add_option("--noL2", action = "store_true", default = False,
                       help = "bypassL2")
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER_Baseline':
         panic("This script requires the" \
         "GPU_VIPER_Baseline protocol to be built.")

configs/ruby/GPU_VIPER_Region.py

@@ -442,7 +442,8 @@ def define_options(parser):
         action="store_true", default=False)
     parser.add_option("--use-L3-on-WT", action="store_true", default=False)
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'GPU_VIPER_Region':
         panic("This script requires the GPU_VIPER_Region protocol to be built.")
 

configs/ruby/Garnet_standalone.py

@@ -42,7 +42,8 @@ class L1Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'Garnet_standalone':
         panic("This script requires Garnet_standalone protocol to be built.")
 
@@ -99,9 +100,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
         l1_cntrl.responseFromCache = MessageBuffer()
         l1_cntrl.forwardFromCache = MessageBuffer()
 
-
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+        options, system.mem_ranges, bootmem, ruby_system, system)
-                                         ruby_system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -112,4 +115,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
     all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MESI_Three_Level.py

@@ -49,7 +49,8 @@ def define_options(parser):
             caches private to clusters")
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MESI_Three_Level':
         fatal("This script requires the MESI_Three_Level protocol to be\
@@ -199,8 +200,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
     ruby_system.memctrl_clk_domain = DerivedClockDomain(
             clk_domain = ruby_system.clk_domain, clk_divider = 3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -259,4 +263,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MESI_Two_Level.py

@@ -43,7 +43,8 @@ class L2Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MESI_Two_Level':
         fatal("This script requires the MESI_Two_Level protocol to be built.")
@@ -173,8 +174,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain = ruby_system.clk_domain,
                                           clk_divider = 3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -231,4 +235,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MI_example.py

@@ -42,7 +42,8 @@ class L1Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MI_example':
         panic("This script requires the MI_example protocol to be built.")
@@ -125,8 +126,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer(ordered = True)
@@ -188,4 +192,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 5
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MOESI_AMD_Base.py

@@ -209,7 +209,8 @@ def define_options(parser):
     parser.add_option("--num-tbes", type="int", default=256)
     parser.add_option("--l2-latency", type="int", default=50) # load to use
 
-def create_system(options, full_system, system, dma_devices, ruby_system):
+def create_system(options, full_system, system, dma_devices, bootmem,
+                  ruby_system):
     if buildEnv['PROTOCOL'] != 'MOESI_AMD_Base':
         panic("This script requires the MOESI_AMD_Base protocol.")
 

configs/ruby/MOESI_CMP_directory.py

@@ -43,7 +43,8 @@ class L2Cache(RubyCache): pass
 def define_options(parser):
     return
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_CMP_directory':
         panic("This script requires the MOESI_CMP_directory protocol to be built.")
@@ -165,8 +166,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_divider=3)
 
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         # Connect the directory controllers and the network
         dir_cntrl.requestToDir = MessageBuffer()
@@ -236,4 +240,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 3
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MOESI_CMP_token.py

@@ -50,7 +50,8 @@ def define_options(parser):
     parser.add_option("--allow-atomic-migration", action="store_true",
           help="allow migratory sharing for atomic only accessed blocks")
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_CMP_token':
         panic("This script requires the MOESI_CMP_token protocol to be built.")
@@ -190,8 +191,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         dir_cntrl.l2_select_num_bits = l2_bits
         # Connect the directory controllers and the network
@@ -264,4 +268,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 6
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/MOESI_hammer.py

@@ -52,7 +52,8 @@ def define_options(parser):
     parser.add_option("--dir-on", action="store_true",
           help="Hammer: enable Full-bit Directory")
 
-def create_system(options, full_system, system, dma_ports, ruby_system):
+def create_system(options, full_system, system, dma_ports, bootmem,
+                  ruby_system):
 
     if buildEnv['PROTOCOL'] != 'MOESI_hammer':
         panic("This script requires the MOESI_hammer protocol to be built.")
@@ -172,8 +173,11 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
                                           clk_domain=ruby_system.clk_domain,
                                           clk_divider=3)
 
-    dir_cntrl_nodes = create_directories(options, system.mem_ranges,
+    mem_dir_cntrl_nodes, rom_dir_cntrl_node = create_directories(
-                                         ruby_system)
+        options, system.mem_ranges, bootmem, ruby_system, system)
+    dir_cntrl_nodes = mem_dir_cntrl_nodes[:]
+    if rom_dir_cntrl_node is not None:
+        dir_cntrl_nodes.append(rom_dir_cntrl_node)
     for dir_cntrl in dir_cntrl_nodes:
         pf = ProbeFilter(size = pf_size, assoc = 4,
                          start_index_bit = pf_start_bit)
@@ -254,4 +258,4 @@ def create_system(options, full_system, system, dma_ports, ruby_system):
 
     ruby_system.network.number_of_virtual_networks = 6
     topology = create_topology(all_cntrls, options)
-    return (cpu_sequencers, dir_cntrl_nodes, topology)
+    return (cpu_sequencers, mem_dir_cntrl_nodes, topology)

configs/ruby/Ruby.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2012, 2017 ARM Limited
+# Copyright (c) 2012, 2017-2018 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -135,7 +135,8 @@ def create_topology(controllers, options):
     topology = eval("Topo.%s(controllers)" % options.topology)
     return topology
 
-def create_system(options, full_system, system, piobus = None, dma_ports = []):
+def create_system(options, full_system, system, piobus = None, dma_ports = [],
+                  bootmem=None):
 
     system.ruby = RubySystem()
     ruby = system.ruby
@@ -150,7 +151,7 @@ def create_system(options, full_system, system, piobus = None, dma_ports = []):
     try:
         (cpu_sequencers, dir_cntrls, topology) = \
              eval("%s.create_system(options, full_system, system, dma_ports,\
-                                    ruby)"
+                                    bootmem, ruby)"
                   % protocol)
     except:
         print("Error: could not create sytem for ruby protocol %s" % protocol)
@@ -198,7 +199,8 @@ def create_system(options, full_system, system, piobus = None, dma_ports = []):
         ruby.phys_mem = SimpleMemory(range=system.mem_ranges[0],
                                      in_addr_map=False)
 
-def create_directories(options, mem_ranges, ruby_system):
+def create_directories(options, mem_ranges, bootmem, ruby_system,
+                       system):
     dir_cntrl_nodes = []
     if options.numa_high_bit:
         numa_bit = options.numa_high_bit
@@ -227,7 +229,17 @@ def create_directories(options, mem_ranges, ruby_system):
 
         exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
         dir_cntrl_nodes.append(dir_cntrl)
-    return dir_cntrl_nodes
+
+    if bootmem is not None:
+        rom_dir_cntrl = Directory_Controller()
+        rom_dir_cntrl.directory = RubyDirectoryMemory()
+        rom_dir_cntrl.ruby_system = ruby_system
+        rom_dir_cntrl.version = i + 1
+        rom_dir_cntrl.memory = bootmem.port
+        rom_dir_cntrl.addr_ranges = bootmem.range
+        return (dir_cntrl_nodes, rom_dir_cntrl)
+
+    return (dir_cntrl_nodes, None)
 
 def send_evicts(options):
     # currently, 2 scenarios warrant forwarding evictions to the CPU:

src/dev/arm/RealView.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2009-2017 ARM Limited
+# Copyright (c) 2009-2018 ARM Limited
 # All rights reserved.
 #
 # The license below extends only to copyright in the software and shall
@@ -580,9 +580,11 @@ class RealView(Platform):
         self._attach_io(self._off_chip_devices(), *args, **kwargs)
 
     def setupBootLoader(self, mem_bus, cur_sys, loc):
-        self.nvmem = SimpleMemory(range = AddrRange('2GB', size = '64MB'),
+        cur_sys.bootmem = SimpleMemory(
-                                  conf_table_reported = False)
+            range = AddrRange('2GB', size = '64MB'),
-        self.nvmem.port = mem_bus.master
+            conf_table_reported = False)
+        if mem_bus is not None:
+            cur_sys.bootmem.port = mem_bus.master
         cur_sys.boot_loader = loc('boot.arm')
         cur_sys.atags_addr = 0x100
         cur_sys.load_offset = 0
@@ -971,9 +973,10 @@ class VExpress_EMM(RealView):
         self.gicv2m.frames = [Gicv2mFrame(spi_base=256, spi_len=64, addr=0x2C1C0000)]
 
     def setupBootLoader(self, mem_bus, cur_sys, loc):
-        self.nvmem = SimpleMemory(range = AddrRange('64MB'),
+        cur_sys.bootmem = SimpleMemory(range = AddrRange('64MB'),
-                                  conf_table_reported = False)
+                                       conf_table_reported = False)
-        self.nvmem.port = mem_bus.master
+        if mem_bus is not None:
+            cur_sys.bootmem.port = mem_bus.master
         if not cur_sys.boot_loader:
             cur_sys.boot_loader = loc('boot_emm.arm')
         cur_sys.atags_addr = 0x8000000
@@ -988,15 +991,15 @@ class VExpress_EMM64(VExpress_EMM):
         pci_pio_base=0x2f000000)
 
     def setupBootLoader(self, mem_bus, cur_sys, loc):
-        self.nvmem = SimpleMemory(range=AddrRange(0, size='64MB'),
+        cur_sys.bootmem = SimpleMemory(range=AddrRange(0, size='64MB'),
-                                  conf_table_reported=False)
+                                       conf_table_reported=False)
-        self.nvmem.port = mem_bus.master
+        if mem_bus is not None:
+            cur_sys.bootmem.port = mem_bus.master
         if not cur_sys.boot_loader:
             cur_sys.boot_loader = loc('boot_emm.arm64')
         cur_sys.atags_addr = 0x8000000
         cur_sys.load_offset = 0x80000000
 
-
 class VExpress_GEM5_V1(RealView):
     """
 The VExpress gem5 memory map is loosely based on a modified
@@ -1164,9 +1167,10 @@ Interrupts:
         self._attach_device(device, *args, **kwargs)
 
     def setupBootLoader(self, mem_bus, cur_sys, loc):
-        self.nvmem = SimpleMemory(range=AddrRange(0, size='64MB'),
+        cur_sys.bootmem = SimpleMemory(range=AddrRange(0, size='64MB'),
-                                  conf_table_reported=False)
+                                       conf_table_reported=False)
-        self.nvmem.port = mem_bus.master
+        if mem_bus is not None:
+            cur_sys.bootmem.port = mem_bus.master
         if not cur_sys.boot_loader:
             cur_sys.boot_loader = [ loc('boot_emm.arm64'), loc('boot_emm.arm') ]
         cur_sys.atags_addr = 0x8000000

tests/configs/base_config.py

@@ -170,8 +170,9 @@ class BaseSystem(object):
             options.num_cpus = self.num_cpus
             options.num_dirs = 2
 
+            bootmem = getattr(system, 'bootmem', None)
             Ruby.create_system(options, True, system, system.iobus,
-                               system._dma_ports)
+                               system._dma_ports, bootmem)
 
             # Create a seperate clock domain for Ruby
             system.ruby.clk_domain = SrcClockDomain(