diff --git a/src/mem/ruby/protocol/chi/CHI-cache-actions.sm b/src/mem/ruby/protocol/chi/CHI-cache-actions.sm index 3db54934c6..b062d56c28 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-actions.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-actions.sm @@ -682,6 +682,42 @@ action(Initiate_ReadUnique_Hit_InvUpstream, desc="") { tbe.actions.pushNB(Event:TagArrayWrite); } +action(Initiate_MakeReadUnique_Hit_InvUpstream, desc="") { + assert(tbe.reqType == CHIRequestType:MakeReadUnique); + tbe.actions.push(Event:ReadHitPipe); + if (tbe.dir_sharers.count() > 1) { + tbe.actions.push(Event:SendSnpCleanInvalidNoReq); + } + if (!tbe.dataUnique && !is_HN) { + tbe.actions.push(Event:SendMakeReadUnique); + } + tbe.actions.pushNB(Event:DataArrayRead); + tbe.actions.push(Event:FinishMakeReadUnique); +} + +action(Finish_MakeReadUnique, desc="") { + // Choosing whether to send or not data back (CompData vs Comp) + // to the requestor. Data will be forwarded if the requestor + // lost the cacheline while the transaction was in progress + if (tbe.dataValid && tbe.dir_sharers.isElement(tbe.requestor) == false) { + tbe.actions.pushNB(Event:SendCompData); + } else { + if (tbe.dataDirty) { + // Got dirty data from snoopees + tbe.actions.pushNB(Event:SendCompUDResp); + } else { + tbe.actions.pushNB(Event:SendCompUCResp); + } + } + tbe.dataUnique := true; + tbe.dataMaybeDirtyUpstream := true; + tbe.requestorToBeExclusiveOwner := true; + tbe.dir_ownerExists := false; + + tbe.actions.pushNB(Event:TagArrayWrite); + tbe.actions.push(Event:WaitCompAck); +} + action(Initiate_CleanUnique, desc="") { tbe.actions.push(Event:ReadMissPipe); // TODO need another latency pipe ?? @@ -1587,6 +1623,28 @@ action(Send_ReadUnique, desc="") { } } +action(Send_MakeReadUnique, desc="") { + assert((tbe.dataValid && tbe.dataUnique) == false); + + assert(tbe.expected_req_resp.hasExpected() == false); + clearExpectedReqResp(tbe); + tbe.expected_req_resp.addExpectedDataType(CHIDataType:DataSepResp_UC); + tbe.expected_req_resp.addExpectedDataType(CHIDataType:CompData_UC); + tbe.expected_req_resp.addExpectedDataType(CHIDataType:CompData_UD_PD); + // NOTE: the first CompData received counts as RespSepData + tbe.expected_req_resp.addExpectedRespType(CHIResponseType:RespSepData); + tbe.expected_req_resp.addExpectedRespType(CHIResponseType:Comp_UC); + tbe.expected_req_resp.addExpectedRespType(CHIResponseType:Comp_UD_PD); + tbe.expected_req_resp.setExpectedCount(2); + + enqueue(reqOutPort, CHIRequestMsg, request_latency) { + prepareRequest(tbe, CHIRequestType:MakeReadUnique, out_msg); + out_msg.Destination.add(mapAddressToDownstreamMachine(tbe.addr)); + out_msg.dataToFwdRequestor := false; + allowRequestRetry(tbe, out_msg); + } +} + action(Send_CleanUnique, desc="") { assert(tbe.dataValid || (tbe.dir_sharers.count() > 0)); assert(tbe.dataUnique == false); @@ -2670,7 +2728,8 @@ action(Send_CompData, desc="") { bool is_rd_shared := (tbe.reqType == CHIRequestType:ReadShared) || (tbe.reqType == CHIRequestType:ReadNotSharedDirty); bool is_rd_nsd := tbe.reqType == CHIRequestType:ReadNotSharedDirty; - bool is_rd_unique := tbe.reqType == CHIRequestType:ReadUnique; + bool is_rd_unique := tbe.reqType == CHIRequestType:ReadUnique || + tbe.reqType == CHIRequestType:MakeReadUnique; // Send UC/UD on ReadShared or ReadNotSharedDirty if the line has no sharers // and one of the followings are met @@ -3095,6 +3154,18 @@ action(Send_CompUC_Stale, desc="") { } } +action(Send_CompUD_PD, desc="") { + assert(is_valid(tbe)); + enqueue(rspOutPort, CHIResponseMsg, response_latency) { + out_msg.addr := address; + out_msg.type := CHIResponseType:Comp_UD_PD; + out_msg.responder := machineID; + out_msg.Destination.add(tbe.requestor); + out_msg.txnId := tbe.txnId; + out_msg.dbid := tbe.txnId; + } +} + action(Send_CompAck, desc="") { assert(is_valid(tbe)); enqueue(rspOutPort, CHIResponseMsg, response_latency) { diff --git a/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm b/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm index 9fda5ba052..d2272d341e 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm @@ -798,13 +798,15 @@ bool needCacheEntry(CHIRequestType req_type, (enable_DMT && is_invalid(dir_entry) && ((req_type == CHIRequestType:ReadShared) || (req_type == CHIRequestType:ReadUnique) || + (req_type == CHIRequestType:MakeReadUnique) || (req_type == CHIRequestType:ReadOnce)))) { return false; } else { return is_prefetch || (alloc_on_readshared && ((req_type == CHIRequestType:ReadShared) || (req_type == CHIRequestType:ReadNotSharedDirty))) || - (alloc_on_readunique && (req_type == CHIRequestType:ReadUnique)) || + (alloc_on_readunique && ((req_type == CHIRequestType:ReadUnique) || + (req_type == CHIRequestType:MakeReadUnique))) || (alloc_on_readonce && (req_type == CHIRequestType:ReadOnce)) || (alloc_on_writeback && ((req_type == CHIRequestType:WriteBackFull) || (req_type == CHIRequestType:WriteCleanFull) || @@ -824,6 +826,7 @@ bool needDeallocCacheEntry(CHIRequestType req_type) { return (dealloc_on_shared && ((req_type == CHIRequestType:ReadShared) || (req_type == CHIRequestType:ReadNotSharedDirty))) || (dealloc_on_unique && ((req_type == CHIRequestType:ReadUnique) || + (req_type == CHIRequestType:MakeReadUnique) || (req_type == CHIRequestType:CleanUnique))); } @@ -1185,6 +1188,7 @@ bool isReadReqType(CHIRequestType type) { if (type == CHIRequestType:Load || type == CHIRequestType:ReadShared || type == CHIRequestType:ReadNotSharedDirty || + type == CHIRequestType:MakeReadUnique || type == CHIRequestType:ReadOnce) { return true; } @@ -1243,6 +1247,8 @@ Event reqToEvent(CHIRequestType type, bool is_prefetch) { return Event:CleanUnique; } else if (type == CHIRequestType:ReadOnce) { return Event:ReadOnce; + } else if (type == CHIRequestType:MakeReadUnique) { + return Event:MakeReadUnique; } else if (type == CHIRequestType:Evict) { return Event:Evict; } else if (type == CHIRequestType:WriteBackFull) { diff --git a/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm b/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm index 012e5f4ce5..2e19245d36 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2023 Arm Limited + * Copyright (c) 2021-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -332,6 +332,29 @@ transition({SC_RSC, SD_RSC, RSC, SD_RSD, RSD}, ReadUnique, BUSY_BLKD) { ProcessNextState; } +// MakeReadUnique +transition({UD_RSC,UC_RSC,UD_RSD}, MakeReadUnique, BUSY_BLKD) { + Initiate_Request; + Initiate_MakeReadUnique_Hit_InvUpstream; + Profile_Hit; + Pop_ReqRdyQueue; + ProcessNextState; +} + +transition({RUSD,RUSC,RSC,RSD,SC_RSC,SD_RSC,SD_RSD}, MakeReadUnique, BUSY_BLKD) { + Initiate_Request; + Initiate_MakeReadUnique_Hit_InvUpstream; + Profile_Miss; + Pop_ReqRdyQueue; + ProcessNextState; +} + +transition({BUSY_BLKD, BUSY_INTR}, FinishMakeReadUnique, BUSY_BLKD) { + Pop_TriggerQueue; + Finish_MakeReadUnique; + ProcessNextState_ClearPending; +} + // CleanUnique transition({I, SC, UC, SD, UD, RU, RSC, RSD, RUSD, RUSC, @@ -927,6 +950,7 @@ transition({BUSY_BLKD,BUSY_INTR}, {ReadShared, ReadNotSharedDirty, ReadUnique, ReadUnique_PoC, ReadOnce, CleanUnique, CleanUnique_Stale, Load, Store, AtomicLoad, AtomicStore, Prefetch, + MakeReadUnique, WriteBackFull, WriteBackFull_Stale, WriteEvictFull, WriteEvictFull_Stale, WriteCleanFull, WriteCleanFull_Stale, @@ -1151,6 +1175,12 @@ transition(BUSY_BLKD, SendCompUCRespStale) { ProcessNextState_ClearPending; } +transition(BUSY_BLKD, SendCompUDResp) { + Pop_TriggerQueue; + Send_CompUD_PD; + ProcessNextState_ClearPending; +} + transition(BUSY_BLKD, SendRespSepData) { Pop_TriggerQueue; Send_RespSepData; @@ -1189,6 +1219,13 @@ transition(BUSY_BLKD, SendReadUnique, BUSY_INTR) {DestinationAvailable} { ProcessNextState_ClearPending; } +transition(BUSY_BLKD, SendMakeReadUnique, BUSY_INTR) {DestinationAvailable} { + Pop_TriggerQueue; + Send_MakeReadUnique; + Profile_OutgoingStart; + ProcessNextState_ClearPending; +} + transition(BUSY_BLKD, SendCleanUnique, BUSY_INTR) {DestinationAvailable} { Pop_TriggerQueue; Send_CleanUnique; diff --git a/src/mem/ruby/protocol/chi/CHI-cache.sm b/src/mem/ruby/protocol/chi/CHI-cache.sm index 6a74045c23..4e0efd85f7 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache.sm @@ -313,6 +313,7 @@ machine(MachineType:Cache, "Cache coherency protocol") : ReadUnique_PoC, desc="", in_trans="yes"; ReadOnce, desc="", in_trans="yes"; CleanUnique, desc="", in_trans="yes"; + MakeReadUnique, desc="", in_trans="yes"; Evict, desc="", in_trans="yes"; WriteBackFull, desc="", in_trans="yes"; WriteEvictFull, desc="", in_trans="yes"; @@ -464,6 +465,7 @@ machine(MachineType:Cache, "Cache coherency protocol") : SendReadNoSnp, out_trans="yes", desc="Send a SendReadNoSnp"; SendReadNoSnpDMT, out_trans="yes", desc="Send a SendReadNoSnp using DMT"; SendReadUnique, out_trans="yes", desc="Send a ReadUnique"; + SendMakeReadUnique, out_trans="yes", desc="Send a MakeReadUnique"; SendCompAck, desc="Send CompAck"; // Read handling at the completer SendCompData, desc="Send CompData"; @@ -505,6 +507,7 @@ machine(MachineType:Cache, "Cache coherency protocol") : SendCleanUnique,out_trans="yes", desc="Send a CleanUnique"; SendCompUCResp, desc="Ack CleanUnique with Comp_UC"; SendCompUCRespStale, desc="Ack stale CleanUnique with Comp_UC"; + SendCompUDResp, desc="Ack MakeReadUnique with Comp_UD_PD"; // Checks if an upgrade using a CleanUnique was sucessfull CheckUpgrade_FromStore, desc="Upgrade needed by a Store"; @@ -552,6 +555,7 @@ machine(MachineType:Cache, "Cache coherency protocol") : TX_Data, desc="Transmit pending data messages"; MaintainCoherence, desc="Queues a WriteBack or Evict before droping the only valid copy of the block"; FinishCleanUnique, desc="Sends acks and perform any writeback after a CleanUnique"; + FinishMakeReadUnique, desc="Return Comp or CompData to requestor at the end of MakeReadUnique"; FinishCopyBack_Stale, desc="Check if a Evict needs to be sent"; ActionStalledOnHazard, desc="Stall a trigger action because until finish handling snoop hazard"; WriteZero, desc="Stall a trigger action because until finish handling snoop hazard"; diff --git a/src/mem/ruby/protocol/chi/CHI-msg.sm b/src/mem/ruby/protocol/chi/CHI-msg.sm index eacd29a4b3..ce0d3d88cf 100644 --- a/src/mem/ruby/protocol/chi/CHI-msg.sm +++ b/src/mem/ruby/protocol/chi/CHI-msg.sm @@ -59,6 +59,7 @@ enumeration(CHIRequestType, desc="") { ReadUnique; ReadOnce; CleanUnique; + MakeReadUnique; Evict; @@ -134,6 +135,7 @@ enumeration(CHIResponseType, desc="...") { // CHI response types Comp_I; Comp_UC; + Comp_UD_PD; Comp_SC; CompAck; CompDBIDResp;