mem: Handle DRAM write queue drain and disabled power down

Write queue drain logic seems off currently. An event is scheduled if
the write queue is empty instead of non-empty. There is no check to see
if draining is complete when bus is in write mode. Finally the power
down check on drain always fails if DRAM powerdown is disabled.

This changeset reverses the drain conditional for the write queue to
schedule an event if the write queue is *not* empty and checks in the
event processing method that the queues are all empty so that
signalDrainDone can be called. Lastly the powerdown state is ignored if
DRAM powerdown is disabled. Powerdown is disabled in the GPU_VIPER
protocol by default. This changeset successfully drains and checkpoints
a GPUFS simulation using GPU_VIPER protocol.

Change-Id: I5459856a694c9054b28677049a06b99b9ad91bbb
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/69917
Tested-by: kokoro <noreply+kokoro@google.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>

src/mem/dram_interface.hh

@@ -380,7 +380,18 @@ class DRAMInterface : public MemInterface
          * @param Return true if the rank is idle from a bank
          *        and power point of view
          */
-        bool inPwrIdleState() const { return pwrState == PWR_IDLE; }
+        bool
+        inPwrIdleState() const
+        {
+            // If powerdown is not enabled, then the ranks never go to idle
+            // states. In that case return true here to prevent checkpointing
+            // from getting stuck waiting for DRAM to be idle.
+            if (!dram.enableDRAMPowerdown) {
+                return true;
+            }
+
+            return pwrState == PWR_IDLE;
+        }
 
         /**
          * Trigger a self-refresh exit if there are entries enqueued

src/mem/mem_ctrl.cc

@@ -908,6 +908,13 @@ MemCtrl::processNextReqEvent(MemInterface* mem_intr,
         }
     }
 
+    if (drainState() == DrainState::Draining && !totalWriteQueueSize &&
+        !totalReadQueueSize && respQEmpty() && allIntfDrained()) {
+
+        DPRINTF(Drain, "MemCtrl controller done draining\n");
+        signalDrainDone();
+    }
+
     // updates current state
     busState = busStateNext;
 
@@ -1411,8 +1418,8 @@ MemCtrl::drain()
 {
     // if there is anything in any of our internal queues, keep track
     // of that as well
-    if (!(!totalWriteQueueSize && !totalReadQueueSize && respQueue.empty() &&
-          allIntfDrained())) {
+    if (totalWriteQueueSize || totalReadQueueSize || !respQueue.empty() ||
+          !allIntfDrained()) {
 
         DPRINTF(Drain, "Memory controller not drained, write: %d, read: %d,"
                 " resp: %d\n", totalWriteQueueSize, totalReadQueueSize,
@@ -1420,7 +1427,8 @@ MemCtrl::drain()
 
         // the only queue that is not drained automatically over time
         // is the write queue, thus kick things into action if needed
-        if (!totalWriteQueueSize && !nextReqEvent.scheduled()) {
+        if (totalWriteQueueSize && !nextReqEvent.scheduled()) {
+            DPRINTF(Drain,"Scheduling nextReqEvent from drain\n");
             schedule(nextReqEvent, curTick());
         }