From c493d2c4ad3a4c7a54f129b883ab429a5728e6af Mon Sep 17 00:00:00 2001
From: Matthew Poremba <matthew.poremba@amd.com>
Date: Thu, 3 Jun 2021 12:22:50 -0500
Subject: [PATCH] sim,mem-ruby: Handle interleaved device memory

Device memories are used for PCI devices which have their own pools of
backing store memory such as amdgpu device. The check for an address
being in device memory previously did not handle multiple interleaved
memory devices with the same address range. Therefore, the device memory
check would fail if the interleaving masks did not match. This updates
the method to iterate through all device memories that handle the
RequestorID and returns true if any of the device memories contain the
packet address.

Change-Id: I9339d39c1cb54a5b9075c4a122c118fe61dc6fdb
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/46381
Reviewed-by: Daniel Carvalho <odanrc@yahoo.com.br>
Reviewed-by: Matt Sinclair <mattdsinclair@gmail.com>
Maintainer: Matt Sinclair <mattdsinclair@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
---
 src/mem/ruby/system/RubyPort.cc |  2 +-
 src/sim/system.cc               | 30 +++++++++++++++++++-----------
 src/sim/system.hh               |  7 ++++---
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/src/mem/ruby/system/RubyPort.cc b/src/mem/ruby/system/RubyPort.cc
index 741a41023f..d1305c4fe6 100644
--- a/src/mem/ruby/system/RubyPort.cc
+++ b/src/mem/ruby/system/RubyPort.cc
@@ -565,7 +565,7 @@ RubyPort::MemResponsePort::hitCallback(PacketPtr pkt)
         // We must check device memory first in case it overlaps with the
         // system memory range.
         if (ruby_port->system->isDeviceMemAddr(pkt)) {
-            auto dmem = ruby_port->system->getDeviceMemory(pkt->requestorId());
+            auto dmem = ruby_port->system->getDeviceMemory(pkt);
             dmem->access(pkt);
         } else if (ruby_port->system->isMemAddr(pkt->getAddr())) {
             rs->getPhysMem()->access(pkt);
diff --git a/src/sim/system.cc b/src/sim/system.cc
index 3c244d822d..04dce5e9fc 100644
--- a/src/sim/system.cc
+++ b/src/sim/system.cc
@@ -378,26 +378,34 @@ System::isMemAddr(Addr addr) const
 void
 System::addDeviceMemory(RequestorID requestor_id, AbstractMemory *deviceMemory)
 {
-    if (!deviceMemMap.count(requestor_id)) {
-        deviceMemMap.insert(std::make_pair(requestor_id, deviceMemory));
-    }
+    deviceMemMap[requestor_id].push_back(deviceMemory);
 }
 
 bool
-System::isDeviceMemAddr(PacketPtr pkt) const
+System::isDeviceMemAddr(const PacketPtr& pkt) const
 {
-    const RequestorID& id = pkt->requestorId();
+    if (!deviceMemMap.count(pkt->requestorId())) {
+        return false;
+    }
 
-    return (deviceMemMap.count(id) &&
-            deviceMemMap.at(id)->getAddrRange().contains(pkt->getAddr()));
+    return (getDeviceMemory(pkt) != nullptr);
 }
 
 AbstractMemory *
-System::getDeviceMemory(RequestorID id) const
+System::getDeviceMemory(const PacketPtr& pkt) const
 {
-    panic_if(!deviceMemMap.count(id),
-             "No device memory found for RequestorID %d\n", id);
-    return deviceMemMap.at(id);
+    const RequestorID& rid = pkt->requestorId();
+
+    panic_if(!deviceMemMap.count(rid),
+             "No device memory found for Requestor %d\n", rid);
+
+    for (auto& mem : deviceMemMap.at(rid)) {
+        if (pkt->getAddrRange().isSubset(mem->getAddrRange())) {
+            return mem;
+        }
+    }
+
+    return nullptr;
 }
 
 void
diff --git a/src/sim/system.hh b/src/sim/system.hh
index 09798c3855..66e138e39f 100644
--- a/src/sim/system.hh
+++ b/src/sim/system.hh
@@ -108,7 +108,8 @@ class System : public SimObject, public PCEventScope
     SystemPort _systemPort;
 
     // Map of memory address ranges for devices with their own backing stores
-    std::unordered_map<RequestorID, AbstractMemory *> deviceMemMap;
+    std::unordered_map<RequestorID, std::vector<AbstractMemory *>>
+        deviceMemMap;
 
   public:
 
@@ -371,12 +372,12 @@ class System : public SimObject, public PCEventScope
      * of the packet match an address range of a device corresponding to the
      * RequestorId of the request.
      */
-    bool isDeviceMemAddr(PacketPtr pkt) const;
+    bool isDeviceMemAddr(const PacketPtr& pkt) const;
 
     /**
      * Return a pointer to the device memory.
      */
-    AbstractMemory *getDeviceMemory(RequestorID _id) const;
+    AbstractMemory *getDeviceMemory(const PacketPtr& pkt) const;
 
     /*
      * Return the list of address ranges backed by a shadowed ROM.