sim,mem-ruby: Handle interleaved device memory

Device memories are used for PCI devices which have their own pools of
backing store memory such as amdgpu device. The check for an address
being in device memory previously did not handle multiple interleaved
memory devices with the same address range. Therefore, the device memory
check would fail if the interleaving masks did not match. This updates
the method to iterate through all device memories that handle the
RequestorID and returns true if any of the device memories contain the
packet address.

Change-Id: I9339d39c1cb54a5b9075c4a122c118fe61dc6fdb
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/46381
Reviewed-by: Daniel Carvalho <odanrc@yahoo.com.br>
Reviewed-by: Matt Sinclair <mattdsinclair@gmail.com>
Maintainer: Matt Sinclair <mattdsinclair@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/mem/ruby/system/RubyPort.cc

@@ -565,7 +565,7 @@ RubyPort::MemResponsePort::hitCallback(PacketPtr pkt)
         // We must check device memory first in case it overlaps with the
         // system memory range.
         if (ruby_port->system->isDeviceMemAddr(pkt)) {
-            auto dmem = ruby_port->system->getDeviceMemory(pkt->requestorId());
+            auto dmem = ruby_port->system->getDeviceMemory(pkt);
             dmem->access(pkt);
         } else if (ruby_port->system->isMemAddr(pkt->getAddr())) {
             rs->getPhysMem()->access(pkt);

src/sim/system.cc

@@ -378,26 +378,34 @@ System::isMemAddr(Addr addr) const
 void
 System::addDeviceMemory(RequestorID requestor_id, AbstractMemory *deviceMemory)
 {
-    if (!deviceMemMap.count(requestor_id)) {
-        deviceMemMap.insert(std::make_pair(requestor_id, deviceMemory));
-    }
+    deviceMemMap[requestor_id].push_back(deviceMemory);
 }
 
 bool
-System::isDeviceMemAddr(PacketPtr pkt) const
+System::isDeviceMemAddr(const PacketPtr& pkt) const
 {
-    const RequestorID& id = pkt->requestorId();
+    if (!deviceMemMap.count(pkt->requestorId())) {
+        return false;
+    }
 
-    return (deviceMemMap.count(id) &&
-            deviceMemMap.at(id)->getAddrRange().contains(pkt->getAddr()));
+    return (getDeviceMemory(pkt) != nullptr);
 }
 
 AbstractMemory *
-System::getDeviceMemory(RequestorID id) const
+System::getDeviceMemory(const PacketPtr& pkt) const
 {
-    panic_if(!deviceMemMap.count(id),
-             "No device memory found for RequestorID %d\n", id);
-    return deviceMemMap.at(id);
+    const RequestorID& rid = pkt->requestorId();
+
+    panic_if(!deviceMemMap.count(rid),
+             "No device memory found for Requestor %d\n", rid);
+
+    for (auto& mem : deviceMemMap.at(rid)) {
+        if (pkt->getAddrRange().isSubset(mem->getAddrRange())) {
+            return mem;
+        }
+    }
+
+    return nullptr;
 }
 
 void

src/sim/system.hh

@@ -108,7 +108,8 @@ class System : public SimObject, public PCEventScope
     SystemPort _systemPort;
 
     // Map of memory address ranges for devices with their own backing stores
-    std::unordered_map<RequestorID, AbstractMemory *> deviceMemMap;
+    std::unordered_map<RequestorID, std::vector<AbstractMemory *>>
+        deviceMemMap;
 
   public:
 
@@ -371,12 +372,12 @@ class System : public SimObject, public PCEventScope
      * of the packet match an address range of a device corresponding to the
      * RequestorId of the request.
      */
-    bool isDeviceMemAddr(PacketPtr pkt) const;
+    bool isDeviceMemAddr(const PacketPtr& pkt) const;
 
     /**
      * Return a pointer to the device memory.
      */
-    AbstractMemory *getDeviceMemory(RequestorID _id) const;
+    AbstractMemory *getDeviceMemory(const PacketPtr& pkt) const;
 
     /*
      * Return the list of address ranges backed by a shadowed ROM.