dev-amdgpu,configs: checkpoint before MMIOs

The flow for Full System amdgpu is the use KVM to boot linux and begin
loading the driver module. However, the amdgpu module requires reading
the VGA ROM located at 0xc0000 in X86. KVM does not support having a
small 128KiB hole at this location, therefore we take a checkpoint and
switch to a timing CPU to continue loading the drivers before the VGA
ROM is read.

This creates a checkpoint just before the first MMIOs. This is indicated
by three interrupts being sent to the PCI device. After three interrupts
in a row are counted a checkpoint exit event occurs. The interrupt
counter is reset if a non-interrupt PCI read is seen.

Change-Id: I23b320abe81ff6e766cb3f604eca2979339938e5
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/46161
Reviewed-by: Matt Sinclair <mattdsinclair@gmail.com>
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Matt Sinclair <mattdsinclair@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

configs/example/gpufs/runfs.py

@@ -58,6 +58,8 @@ def addRunFSOptions(parser):
     parser.add_argument("--host-parallel", default=False,
                         action="store_true",
                         help="Run multiple host threads in KVM mode")
+    parser.add_argument("--restore-dir", type=str, default=None,
+                        help="Directory to restore checkpoints from")
     parser.add_argument("--disk-image", default="",
                         help="The boot disk image to mount (/dev/sda)")
     parser.add_argument("--second-disk", default=None,
@@ -66,6 +68,11 @@ def addRunFSOptions(parser):
     parser.add_argument("--gpu-rom", default=None, help="GPU BIOS to load")
     parser.add_argument("--gpu-mmio-trace", default=None,
                         help="GPU MMIO trace to load")
+    parser.add_argument("--checkpoint-before-mmios", default=False,
+                        action="store_true",
+                        help="Take a checkpoint before driver sends MMIOs. "
+                        "This is used to switch out of KVM mode and into "
+                        "timing mode required to read the VGA ROM on boot.")
 
 
 def runGpuFSSystem(args):
@@ -89,7 +96,10 @@ def runGpuFSSystem(args):
     if args.script is not None:
         system.readfile = args.script
 
-    m5.instantiate()
+    if args.restore_dir is None:
+        m5.instantiate()
+    else:
+        m5.instantiate(args.restore_dir)
 
 
     print("Running the simulation")
@@ -97,6 +107,20 @@ def runGpuFSSystem(args):
 
     exit_event = m5.simulate(sim_ticks)
 
+    # Keep executing while there is something to do
+    while True:
+        if exit_event.getCause() == "m5_exit instruction encountered" or \
+            exit_event.getCause() == "user interrupt received" or \
+            exit_event.getCause() == "simulate() limit reached":
+            break
+        elif "checkpoint" in exit_event.getCause():
+            assert(args.checkpoint_dir is not None)
+            m5.checkpoint(args.checkpoint_dir)
+            break
+        else:
+            print('Unknown exit event: %s. Continuing...'
+                    % exit_event.getCause())
+
     print('Exiting @ tick %i because %s' %
           (m5.curTick(), exit_event.getCause()))
 

configs/example/gpufs/system/amdgpu.py

@@ -148,3 +148,5 @@ def connectGPU(system, args):
 
     system.pc.south_bridge.gpu.trace_file = args.gpu_mmio_trace
     system.pc.south_bridge.gpu.rom_binary = args.gpu_rom
+    system.pc.south_bridge.gpu.checkpoint_before_mmios = \
+        args.checkpoint_before_mmios

src/dev/amdgpu/AMDGPU.py

@@ -70,3 +70,5 @@ class AMDGPUDevice(PciDevice):
 
     rom_binary = Param.String("ROM binary dumped from hardware")
     trace_file = Param.String("MMIO trace collected on hardware")
+    checkpoint_before_mmios = Param.Bool(False, "Take a checkpoint before the"
+                                                " device begins sending MMIOs")

src/dev/amdgpu/amdgpu_device.cc

@@ -40,9 +40,11 @@
 #include "mem/packet_access.hh"
 #include "params/AMDGPUDevice.hh"
 #include "sim/byteswap.hh"
+#include "sim/sim_exit.hh"
 
 AMDGPUDevice::AMDGPUDevice(const AMDGPUDeviceParams &p)
-    : PciDevice(p)
+    : PciDevice(p), checkpoint_before_mmios(p.checkpoint_before_mmios),
+      init_interrupt_count(0)
 {
     // Loading the rom binary dumped from hardware.
     std::ifstream romBin;
@@ -100,7 +102,25 @@ AMDGPUDevice::readConfig(PacketPtr pkt)
     DPRINTF(AMDGPUDevice, "Read Config: from offset: %#x size: %#x "
             "data: %#x\n", offset, pkt->getSize(), config.data[offset]);
 
-    return PciDevice::readConfig(pkt);
+    Tick delay = PciDevice::readConfig(pkt);
+
+    // Before sending MMIOs the driver sends three interrupts in a row.
+    // Use this to trigger creating a checkpoint to restore in timing mode.
+    // This is only necessary until we can create a "hole" in the KVM VM
+    // around the VGA ROM region such that KVM exits and sends requests to
+    // this device rather than the KVM VM.
+    if (checkpoint_before_mmios) {
+        if (offset == PCI0_INTERRUPT_PIN) {
+            if (++init_interrupt_count == 3) {
+                DPRINTF(AMDGPUDevice, "Checkpointing before first MMIO\n");
+                exitSimLoop("checkpoint", 0, curTick() + delay + 1);
+            }
+        } else {
+            init_interrupt_count = 0;
+        }
+    }
+
+    return delay;
 }
 
 Tick

src/dev/amdgpu/amdgpu_device.hh

@@ -102,6 +102,9 @@ class AMDGPUDevice : public PciDevice
      */
     std::unordered_map<uint32_t, uint64_t> regs;
 
+    bool checkpoint_before_mmios;
+    int init_interrupt_count;
+
   public:
     AMDGPUDevice(const AMDGPUDeviceParams &p);