mem-ruby: Copyback UD_RU line when evicted in CHI protocol (#945)

This is a followed up fix to #791 mem-ruby: Fix possible dirty line loss
in CHI when ReadShared hit on UD line.
UD_RU line may have stale data since the upstream could have updated the
line, so its local cache line data is treated as invalid
(dataValid=false). But when the line is evicted, it must be written back
to downstream because the upstream may have the line in clean state
(UC). This change fixes it by performing copy back the UD_RU line while
keeping its dataValid as false.

Example error case:
- L3 was in UD_RSC and being evicted without back-invalidation. LLC (HN)
was in RU state.
- Because there's still upstream sharer, L3 sends WriteClean.
- Because the data state was unique and dirty, L3 sends CBWrData_UD_PD.
- LLC becomes UD_RU.
- When the line is evicted from LLC (LocalHN_Eviction), the line is just
dropped, causing the loss of the dirty copy

Co-authored-by: Minje Jun <minje.jun@samsung.com>

src/mem/ruby/protocol/chi/CHI-cache-actions.sm

@@ -401,6 +401,12 @@ action(Initiate_Replacement, desc="") {
   }
   copyCacheAndDir(cache_entry, getDirEntry(address), tbe, initial);
 
+  // UD_RU line needs to be written back because the upstream may be in UC state.
+  if (initial == State:UD_RU) {
+    tbe.dataBlk := cache_entry.DataBlk;
+    tbe.dataBlkValid.fillMask();
+  }
+
   // model the initial tag array read
   tbe.actions.pushNB(Event:TagArrayRead);
 
@@ -1553,7 +1559,8 @@ action(Send_InvSnpResp, desc="") {
 action(Send_WriteBackOrWriteEvict, desc="") {
   assert(is_valid(tbe));
   assert(tbe.dataBlkValid.isFull());
-  assert(tbe.dataValid);
+  assert(tbe.dataValid ||
+         (tbe.dir_ownerIsExcl && tbe.dataUnique && tbe.dataDirty));
   assert(is_HN == false);
 
   assert(tbe.dataUnique || tbe.dataDirty);
@@ -1576,7 +1583,9 @@ action(Send_WriteBackOrWriteEvict, desc="") {
 action(Send_WriteCleanFull, desc="") {
   assert(is_valid(tbe));
   assert(tbe.dataBlkValid.isFull());
-  assert(tbe.dataValid);
+  // Data must be valid or the line was in UD_RU state.
+  assert(tbe.dataValid ||
+         (tbe.dir_ownerIsExcl && tbe.dataUnique && tbe.dataDirty));
   assert(is_HN == false);
   assert(tbe.dataDirty);
 
@@ -2654,14 +2663,20 @@ action(Send_WBData, desc="") {
   if (is_HN) {
     assert(tbe.dataBlkValid.isFull());
     assert(tbe.dataDirty);
-    assert(tbe.dataValid);
+    assert(tbe.dataValid ||
+           (tbe.dir_ownerIsExcl && tbe.dataUnique && tbe.dataDirty));
     tbe.snd_msgType := CHIDataType:NCBWrData;
   } else {
     if (tbe.dataValid == false) {
-      // only possible when the WB was made stale by a snoop
-      assert(tbe.is_stale);
-      tbe.dataBlkValid.fillMask();
-      tbe.snd_msgType := CHIDataType:CBWrData_I;
+      // only possible when the WB was made stale by a snoop or
+      // Writeback on UD_RU line.
+      if (tbe.dir_ownerIsExcl && tbe.dataUnique && tbe.dataDirty) {
+        tbe.snd_msgType := CHIDataType:CBWrData_UD_PD;
+      } else {
+        assert(tbe.is_stale);
+        tbe.dataBlkValid.fillMask();
+        tbe.snd_msgType := CHIDataType:CBWrData_I;
+      }
     } else if (tbe.dataUnique) {
       assert(tbe.dataBlkValid.isFull());
       if (tbe.dataDirty) {

src/mem/ruby/protocol/chi/CHI-cache-transitions.sm

@@ -713,7 +713,7 @@ transition({UD_RSC, UC_RSC, SC_RSC, SD_RSC, UD, RU, RSD, RUSD, RUSC, UD_RSD, SD_
 // When in UD_RU,UC_RU,UD_RSD,SD_RSD we also just drop the line since an upstream
 // cache has an up-to-data line that it will either WriteBack or WriteEvict
 transition({SC,UC,SC_RSC,UC_RSC,
-            UD_RU,UC_RU,UD_RSD,SD_RSD}, LocalHN_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
+            UC_RU,UD_RSD,SD_RSD}, LocalHN_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
   Initiate_Replacement;
   Initiate_Replacement_JustDrop;
   Profile_Eviction;
@@ -722,7 +722,7 @@ transition({SC,UC,SC_RSC,UC_RSC,
   ProcessNextState;
 }
 
-transition({UD,SD,UD_RSC,SD_RSC}, LocalHN_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
+transition({UD,SD,UD_RU,UD_RSC,SD_RSC}, LocalHN_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
   Initiate_Replacement;
   Initiate_Replacement_WB;
   Profile_Eviction;
@@ -749,7 +749,7 @@ transition({UD,SD,UC}, Local_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
   ProcessNextState;
 }
 
-transition({UD_RU,UC_RU,UD_RSD,SD_RSD,SC_RSC,UC_RSC}, Local_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
+transition({UC_RU,UD_RSD,SD_RSD,SC_RSC,UC_RSC}, Local_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
   Initiate_Replacement;
   Initiate_Replacement_JustDrop;
   Profile_Eviction;
@@ -758,7 +758,7 @@ transition({UD_RU,UC_RU,UD_RSD,SD_RSD,SC_RSC,UC_RSC}, Local_Eviction, BUSY_BLKD)
   ProcessNextState;
 }
 
-transition({UD_RSC,SD_RSC}, Local_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
+transition({UD_RU, UD_RSC,SD_RSC}, Local_Eviction, BUSY_BLKD) {ReplTBEAvailable} {
   Initiate_Replacement;
   Initiate_Replacement_WB;
   Profile_Eviction;