From 0ac110ac9570458b07d69dc4459e70efcd9d44cc Mon Sep 17 00:00:00 2001
From: Matthew Poremba <matthew.poremba@amd.com>
Date: Wed, 24 Jan 2024 07:25:43 -0800
Subject: [PATCH] dev-amdgpu: Check privledge bit for SDMA RLC queues (#792)

By default all SDMA queues are privileged queues, meaning the addresses
in SDMA packets use the privileged translation tables. RLC queues
(sometimes called user queues) are not necessarily privileged and might
use user translation tables. RLC queues are used more often in ROCm 6.0
exposing an issue with invalid translations with RLC queues.

This changeset checks the priv bit in the SDMA MQD when an RLC queue is
mapped. Each packet type which uses an address then checks the bit
before performing translation. Tested with daily/weekly tests with a
ROCm 6.0 disk image and tests are passing.

Change-Id: I6122fbc194e8d6f5d38e81f1b0e11646d90e0ea0
---
 src/dev/amdgpu/sdma_engine.cc | 26 +++++++++++++++++++-------
 src/dev/amdgpu/sdma_engine.hh |  3 +++
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/dev/amdgpu/sdma_engine.cc b/src/dev/amdgpu/sdma_engine.cc
index 0202f583e6..24c228b9c6 100644
--- a/src/dev/amdgpu/sdma_engine.cc
+++ b/src/dev/amdgpu/sdma_engine.cc
@@ -184,6 +184,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
     Addr rptr_wb_addr = mqd->sdmax_rlcx_rb_rptr_addr_hi;
     rptr_wb_addr <<= 32;
     rptr_wb_addr |= mqd->sdmax_rlcx_rb_rptr_addr_lo;
+    bool priv = bits(mqd->sdmax_rlcx_rb_cntl, 23, 23);
 
     // Get first free RLC
     if (!rlc0.valid()) {
@@ -199,6 +200,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
         rlc0.processing(false);
         rlc0.setMQD(mqd);
         rlc0.setMQDAddr(mqdAddr);
+        rlc0.setPriv(priv);
     } else if (!rlc1.valid()) {
         DPRINTF(SDMAEngine, "Doorbell %lx mapped to RLC1\n", doorbell);
         rlcInfo[1] = doorbell;
@@ -212,6 +214,7 @@ SDMAEngine::registerRLCQueue(Addr doorbell, Addr mqdAddr, SDMAQueueDesc *mqd)
         rlc1.processing(false);
         rlc1.setMQD(mqd);
         rlc1.setMQDAddr(mqdAddr);
+        rlc1.setPriv(priv);
     } else {
         panic("No free RLCs. Check they are properly unmapped.");
     }
@@ -622,8 +625,9 @@ SDMAEngine::writeReadData(SDMAQueue *q, sdmaWrite *pkt, uint32_t *dmaBuffer)
         gpuDevice->getMemMgr()->writeRequest(mmhubAddr, (uint8_t *)dmaBuffer,
                                            bufferSize, 0, cb);
     } else {
-        // TODO: getGARTAddr?
-        pkt->dest = getGARTAddr(pkt->dest);
+        if (q->priv()) {
+            pkt->dest = getGARTAddr(pkt->dest);
+        }
         auto cb = new DmaVirtCallback<uint32_t>(
             [ = ] (const uint64_t &) { writeDone(q, pkt, dmaBuffer); });
         dmaWriteVirt(pkt->dest, bufferSize, cb, (void *)dmaBuffer);
@@ -650,9 +654,11 @@ SDMAEngine::copy(SDMAQueue *q, sdmaCopy *pkt)
     q->incRptr(sizeof(sdmaCopy));
     // count represents the number of bytes - 1 to be copied
     pkt->count++;
-    DPRINTF(SDMAEngine, "Getting GART addr for %lx\n", pkt->source);
-    pkt->source = getGARTAddr(pkt->source);
-    DPRINTF(SDMAEngine, "GART addr %lx\n", pkt->source);
+    if (q->priv()) {
+        DPRINTF(SDMAEngine, "Getting GART addr for %lx\n", pkt->source);
+        pkt->source = getGARTAddr(pkt->source);
+        DPRINTF(SDMAEngine, "GART addr %lx\n", pkt->source);
+    }
 
     // Read data from the source first, then call the copyReadData method
     uint8_t *dmaBuffer = new uint8_t[pkt->count];
@@ -745,7 +751,11 @@ SDMAEngine::copyDone(SDMAQueue *q, sdmaCopy *pkt, uint8_t *dmaBuffer)
 void
 SDMAEngine::indirectBuffer(SDMAQueue *q, sdmaIndirectBuffer *pkt)
 {
-    q->ib()->base(getGARTAddr(pkt->base));
+    if (q->priv()) {
+        q->ib()->base(getGARTAddr(pkt->base));
+    } else {
+        q->ib()->base(pkt->base);
+    }
     q->ib()->rptr(0);
     q->ib()->size(pkt->size * sizeof(uint32_t) + 1);
     q->ib()->setWptr(pkt->size * sizeof(uint32_t));
@@ -761,7 +771,9 @@ void
 SDMAEngine::fence(SDMAQueue *q, sdmaFence *pkt)
 {
     q->incRptr(sizeof(sdmaFence));
-    pkt->dest = getGARTAddr(pkt->dest);
+    if (q->priv()) {
+        pkt->dest = getGARTAddr(pkt->dest);
+    }
 
     // Writing the data from the fence packet to the destination address.
     auto cb = new DmaVirtCallback<uint32_t>(
diff --git a/src/dev/amdgpu/sdma_engine.hh b/src/dev/amdgpu/sdma_engine.hh
index 5abe63fcc6..d8ab31bbde 100644
--- a/src/dev/amdgpu/sdma_engine.hh
+++ b/src/dev/amdgpu/sdma_engine.hh
@@ -68,6 +68,7 @@ class SDMAEngine : public DmaVirtDevice
         SDMAType _type;
         SDMAQueueDesc *_mqd;
         Addr _mqd_addr = 0;
+        bool _priv = true; // Only used for RLC queues. True otherwise.
       public:
         SDMAQueue() : _rptr(0), _wptr(0), _valid(false), _processing(false),
             _parent(nullptr), _ib(nullptr), _type(SDMAGfx), _mqd(nullptr) {}
@@ -87,6 +88,7 @@ class SDMAEngine : public DmaVirtDevice
         SDMAType queueType() { return _type; }
         SDMAQueueDesc* getMQD() { return _mqd; }
         Addr getMQDAddr() { return _mqd_addr; }
+        bool priv() { return _priv; }
 
         void base(Addr value) { _base = value; }
 
@@ -121,6 +123,7 @@ class SDMAEngine : public DmaVirtDevice
         void queueType(SDMAType type) { _type = type; }
         void setMQD(SDMAQueueDesc *mqd) { _mqd = mqd; }
         void setMQDAddr(Addr mqdAddr) { _mqd_addr = mqdAddr; }
+        void setPriv(bool priv) { _priv = priv; }
     };
 
     /* SDMA Engine ID */