cache: fail store conditionals when upgrade loses race

Requires new "SCUpgradeReq" message that marks upgrades
for store conditionals, so downstream caches can fail
these when they run into invalidations.
See http://www.m5sim.org/flyspray/task/197

src/mem/cache/cache_impl.hh

@@ -196,7 +196,7 @@ Cache<TagStore>::satisfyCpuSideRequest(PacketPtr pkt, BlkType *blk)
         // Not a read or write... must be an upgrade.  it's OK
         // to just ack those as long as we have an exclusive
         // copy at this level.
-        assert(pkt->cmd == MemCmd::UpgradeReq);
+        assert(pkt->isUpgrade());
         tags->invalidateBlk(blk);
     }
 }
@@ -536,8 +536,8 @@ Cache<TagStore>::getBusPacket(PacketPtr cpu_pkt, BlkType *blk,
         return NULL;
     }
 
-    if (!blkValid && (cpu_pkt->cmd == MemCmd::Writeback ||
-                      cpu_pkt->cmd == MemCmd::UpgradeReq)) {
+    if (!blkValid &&
+        (cpu_pkt->cmd == MemCmd::Writeback || cpu_pkt->isUpgrade())) {
         // Writebacks that weren't allocated in access() and upgrades
         // from upper-level caches that missed completely just go
         // through.
@@ -556,7 +556,7 @@ Cache<TagStore>::getBusPacket(PacketPtr cpu_pkt, BlkType *blk,
         // only reason to be here is that blk is shared
         // (read-only) and we need exclusive
         assert(needsExclusive && !blk->isWritable());
-        cmd = MemCmd::UpgradeReq;
+        cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
     } else {
         // block is invalid
         cmd = needsExclusive ? MemCmd::ReadExReq : MemCmd::ReadReq;
@@ -814,6 +814,11 @@ Cache<TagStore>::handleResponse(PacketPtr pkt)
                 assert(!target->pkt->req->isUncacheable());
                 missLatency[target->pkt->cmdToIndex()][0/*pkt->req->threadId()*/] +=
                     completion_time - target->recvTime;
+            } else if (target->pkt->cmd == MemCmd::StoreCondReq &&
+                       pkt->cmd == MemCmd::UpgradeFailResp) {
+                // failed StoreCond upgrade
+                completion_time = tags->getHitLatency() + pkt->finishTime;
+                target->pkt->req->setExtraData(0);
             } else {
                 // not a cache fill, just forwarding response
                 completion_time = tags->getHitLatency() + pkt->finishTime;
@@ -1336,7 +1341,15 @@ Cache<TagStore>::getTimingPacket()
     PacketPtr tgt_pkt = mshr->getTarget()->pkt;
     PacketPtr pkt = NULL;
 
-    if (mshr->isForwardNoResponse()) {
+    if (tgt_pkt->cmd == MemCmd::SCUpgradeFailReq) {
+        // SCUpgradeReq saw invalidation while queued in MSHR, so now
+        // that we are getting around to processing it, just treat it
+        // as if we got a failure response
+        pkt = new Packet(tgt_pkt);
+        pkt->cmd = MemCmd::UpgradeFailResp;
+        handleResponse(pkt);
+        return NULL;
+    } else if (mshr->isForwardNoResponse()) {
         // no response expected, just forward packet as it is
         assert(tags->findBlock(mshr->addr) == NULL);
         pkt = tgt_pkt;

src/mem/cache/mshr.cc

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2002-2005 The Regents of The University of Michigan
+ * Copyright (c) 2010 Advancec Micro Devices, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -71,7 +72,7 @@ MSHR::TargetList::add(PacketPtr pkt, Tick readyTime,
             needsExclusive = true;
         }
 
-        if (pkt->cmd == MemCmd::UpgradeReq) {
+        if (pkt->isUpgrade()) {
             hasUpgrade = true;
         }
     }
@@ -99,6 +100,9 @@ MSHR::TargetList::replaceUpgrades()
         if (i->pkt->cmd == MemCmd::UpgradeReq) {
             i->pkt->cmd = MemCmd::ReadExReq;
             DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n");
+        } else if (i->pkt->cmd == MemCmd::SCUpgradeReq) {
+            i->pkt->cmd = MemCmd::SCUpgradeFailReq;
+            DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n");
         }
     }
 

src/mem/packet.cc

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2006 The Regents of The University of Michigan
+ * Copyright (c) 2010 Advancec Micro Devices, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -92,11 +93,22 @@ MemCmd::commandInfo[] =
     { SET3(IsWrite, NeedsExclusive, IsResponse),
             InvalidCmd, "WriteInvalidateResp" },
     /* UpgradeReq */
-    { SET4(IsInvalidate, NeedsExclusive, IsRequest, NeedsResponse),
+    { SET5(IsInvalidate, NeedsExclusive, IsUpgrade, IsRequest, NeedsResponse),
             UpgradeResp, "UpgradeReq" },
+    /* SCUpgradeReq: response could be UpgradeResp or UpgradeFailResp */
+    { SET6(IsInvalidate, NeedsExclusive, IsUpgrade, IsLlsc,
+           IsRequest, NeedsResponse),
+            UpgradeResp, "SCUpgradeReq" },
     /* UpgradeResp */
-    { SET2(NeedsExclusive, IsResponse),
+    { SET3(NeedsExclusive, IsUpgrade, IsResponse),
             InvalidCmd, "UpgradeResp" },
+    /* SCUpgradeFailReq: generates UpgradeFailResp ASAP */
+    { SET5(IsInvalidate, NeedsExclusive, IsLlsc,
+           IsRequest, NeedsResponse),
+            UpgradeFailResp, "SCUpgradeFailReq" },
+    /* UpgradeFailResp */
+    { SET2(NeedsExclusive, IsResponse),
+            InvalidCmd, "UpgradeFailResp" },
     /* ReadExReq */
     { SET5(IsRead, NeedsExclusive, IsInvalidate, IsRequest, NeedsResponse),
             ReadExResp, "ReadExReq" },

src/mem/packet.hh

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2006 The Regents of The University of Michigan
+ * Copyright (c) 2010 Advancec Micro Devices, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,7 +82,10 @@ class MemCmd
         WriteInvalidateReq,
         WriteInvalidateResp,
         UpgradeReq,
+        SCUpgradeReq,           // Special "weak" upgrade for StoreCond
         UpgradeResp,
+        SCUpgradeFailReq,       // Failed SCUpgradeReq in MSHR (never sent)
+        UpgradeFailResp,        // Valid for SCUpgradeReq only
         ReadExReq,
         ReadExResp,
         LoadLockedReq,
@@ -111,6 +115,7 @@ class MemCmd
     {
         IsRead,         //!< Data flows from responder to requester
         IsWrite,        //!< Data flows from requester to responder
+        IsUpgrade,
         IsPrefetch,     //!< Not a demand access
         IsInvalidate,
         NeedsExclusive, //!< Requires exclusive copy to complete in-cache
@@ -157,7 +162,8 @@ class MemCmd
   public:
 
     bool isRead() const         { return testCmdAttrib(IsRead); }
-    bool isWrite()  const       { return testCmdAttrib(IsWrite); }
+    bool isWrite() const        { return testCmdAttrib(IsWrite); }
+    bool isUpgrade() const      { return testCmdAttrib(IsUpgrade); }
     bool isRequest() const      { return testCmdAttrib(IsRequest); }
     bool isResponse() const     { return testCmdAttrib(IsResponse); }
     bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); }
@@ -392,7 +398,8 @@ class Packet : public FastAlloc, public Printable
     inline int cmdToIndex() const { return cmd.toInt(); }
 
     bool isRead() const         { return cmd.isRead(); }
-    bool isWrite()  const       { return cmd.isWrite(); }
+    bool isWrite() const        { return cmd.isWrite(); }
+    bool isUpgrade()  const     { return cmd.isUpgrade(); }
     bool isRequest() const      { return cmd.isRequest(); }
     bool isResponse() const     { return cmd.isResponse(); }
     bool needsExclusive() const { return cmd.needsExclusive(); }