dev-amdgpu: Check privledge bit for SDMA RLC queues (#792)

By default all SDMA queues are privileged queues, meaning the addresses
in SDMA packets use the privileged translation tables. RLC queues
(sometimes called user queues) are not necessarily privileged and might
use user translation tables. RLC queues are used more often in ROCm 6.0
exposing an issue with invalid translations with RLC queues.

This changeset checks the priv bit in the SDMA MQD when an RLC queue is
mapped. Each packet type which uses an address then checks the bit
before performing translation. Tested with daily/weekly tests with a
ROCm 6.0 disk image and tests are passing.

Change-Id: I6122fbc194e8d6f5d38e81f1b0e11646d90e0ea0

src/dev/amdgpu/sdma_engine.cc

@@ -184,6 +184,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
     Addr rptr_wb_addr = mqd->sdmax_rlcx_rb_rptr_addr_hi;
     rptr_wb_addr <<= 32;
     rptr_wb_addr |= mqd->sdmax_rlcx_rb_rptr_addr_lo;
+    bool priv = bits(mqd->sdmax_rlcx_rb_cntl, 23, 23);
 
     // Get first free RLC
     if (!rlc0.valid()) {
@@ -199,6 +200,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
         rlc0.processing(false);
         rlc0.setMQD(mqd);
         rlc0.setMQDAddr(mqdAddr);
+        rlc0.setPriv(priv);
     } else if (!rlc1.valid()) {
         DPRINTF(SDMAEngine, "Doorbell %lx mapped to RLC1\n", doorbell);
         rlcInfo[1] = doorbell;
@@ -212,6 +214,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
         rlc1.processing(false);
         rlc1.setMQD(mqd);
         rlc1.setMQDAddr(mqdAddr);
+        rlc1.setPriv(priv);
     } else {
         panic("No free RLCs. Check they are properly unmapped.");
     }
@@ -622,8 +625,9 @@ SDMAEngine::writeReadData(SDMAQueue *q, sdmaWrite *pkt, uint32_t *dmaBuffer)
         gpuDevice->getMemMgr()->writeRequest(mmhubAddr, (uint8_t *)dmaBuffer,
                                            bufferSize, 0, cb);
     } else {
-        // TODO: getGARTAddr?
-        pkt->dest = getGARTAddr(pkt->dest);
+        if (q->priv()) {
+            pkt->dest = getGARTAddr(pkt->dest);
+        }
         auto cb = new DmaVirtCallback<uint32_t>(
             [ = ] (const uint64_t &) { writeDone(q, pkt, dmaBuffer); });
         dmaWriteVirt(pkt->dest, bufferSize, cb, (void *)dmaBuffer);
@@ -650,9 +654,11 @@ SDMAEngine::copy(SDMAQueue *q, sdmaCopy *pkt)
     q->incRptr(sizeof(sdmaCopy));
     // count represents the number of bytes - 1 to be copied
     pkt->count++;
-    DPRINTF(SDMAEngine, "Getting GART addr for %lx\n", pkt->source);
-    pkt->source = getGARTAddr(pkt->source);
-    DPRINTF(SDMAEngine, "GART addr %lx\n", pkt->source);
+    if (q->priv()) {
+        DPRINTF(SDMAEngine, "Getting GART addr for %lx\n", pkt->source);
+        pkt->source = getGARTAddr(pkt->source);
+        DPRINTF(SDMAEngine, "GART addr %lx\n", pkt->source);
+    }
 
     // Read data from the source first, then call the copyReadData method
     uint8_t *dmaBuffer = new uint8_t[pkt->count];
@@ -745,7 +751,11 @@ SDMAEngine::copyDone(SDMAQueue *q, sdmaCopy *pkt, uint8_t *dmaBuffer)
 void
 SDMAEngine::indirectBuffer(SDMAQueue *q, sdmaIndirectBuffer *pkt)
 {
-    q->ib()->base(getGARTAddr(pkt->base));
+    if (q->priv()) {
+        q->ib()->base(getGARTAddr(pkt->base));
+    } else {
+        q->ib()->base(pkt->base);
+    }
     q->ib()->rptr(0);
     q->ib()->size(pkt->size * sizeof(uint32_t) + 1);
     q->ib()->setWptr(pkt->size * sizeof(uint32_t));
@@ -761,7 +771,9 @@ void
 SDMAEngine::fence(SDMAQueue *q, sdmaFence *pkt)
 {
     q->incRptr(sizeof(sdmaFence));
-    pkt->dest = getGARTAddr(pkt->dest);
+    if (q->priv()) {
+        pkt->dest = getGARTAddr(pkt->dest);
+    }
 
     // Writing the data from the fence packet to the destination address.
     auto cb = new DmaVirtCallback<uint32_t>(

src/dev/amdgpu/sdma_engine.hh

@@ -68,6 +68,7 @@ class SDMAEngine : public DmaVirtDevice
         SDMAType _type;
         SDMAQueueDesc *_mqd;
         Addr _mqd_addr = 0;
+        bool _priv = true; // Only used for RLC queues. True otherwise.
       public:
         SDMAQueue() : _rptr(0), _wptr(0), _valid(false), _processing(false),
             _parent(nullptr), _ib(nullptr), _type(SDMAGfx), _mqd(nullptr) {}
@@ -87,6 +88,7 @@ class SDMAEngine : public DmaVirtDevice
         SDMAType queueType() { return _type; }
         SDMAQueueDesc* getMQD() { return _mqd; }
         Addr getMQDAddr() { return _mqd_addr; }
+        bool priv() { return _priv; }
 
         void base(Addr value) { _base = value; }
 
@@ -121,6 +123,7 @@ class SDMAEngine : public DmaVirtDevice
         void queueType(SDMAType type) { _type = type; }
         void setMQD(SDMAQueueDesc *mqd) { _mqd = mqd; }
         void setMQDAddr(Addr mqdAddr) { _mqd_addr = mqdAddr; }
+        void setPriv(bool priv) { _priv = priv; }
     };
 
     /* SDMA Engine ID */