mem-ruby: Make DMASequencer aware of Atomics

Add handling for issuing atomic packet types, setting the WriteMask and
AtomicOpFunctor in makeRequest. Add an atomicCallback to handle atomic
packet type responses.

Change-Id: I9775fc110bb99a1740089746f0d1b3deb124b9f5
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/33716
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Reviewed-by: Matt Sinclair <mattdsinclair@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Matt Sinclair <mattdsinclair@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/mem/ruby/protocol/RubySlicc_Exports.sm

@@ -312,6 +312,7 @@ structure(SequencerMsg, desc="...", interface="Message") {
   SequencerRequestType Type,     desc="Type of request (LD, ST, etc)";
   Addr ProgramCounter,    desc="Program counter of the instruction that caused the miss";
   RubyAccessMode AccessMode, desc="user/supervisor access type";
+  WriteMask writeMask,       desc="WriteMask for atomics";
   DataBlock DataBlk,         desc="Data";
   int Len,                   desc="size in bytes of access";
   PrefetchBit Prefetch,      desc="Is this a prefetch request";

src/mem/ruby/protocol/RubySlicc_Types.sm

@@ -222,6 +222,7 @@ structure (WireBuffer, inport="yes", outport="yes", external = "yes") {
 structure (DMASequencer, external = "yes") {
   void ackCallback(Addr);
   void dataCallback(DataBlock,Addr);
+  void atomicCallback(DataBlock,Addr);
   void recordRequestType(CacheRequestType);
 }
 

src/mem/ruby/system/DMASequencer.cc

@@ -108,7 +108,35 @@ DMASequencer::makeRequest(PacketPtr pkt)
         std::make_shared<SequencerMsg>(clockEdge());
     msg->getPhysicalAddress() = paddr;
     msg->getLineAddress() = line_addr;
-    msg->getType() = write ? SequencerRequestType_ST : SequencerRequestType_LD;
+
+    if (pkt->req->isAtomic()) {
+        msg->setType(SequencerRequestType_ATOMIC);
+
+        // While regular LD/ST can support DMAs spanning multiple cache lines,
+        // atomic requests are only supported within a single cache line. The
+        // atomic request will end upon atomicCallback and not call issueNext.
+        int block_size = m_ruby_system->getBlockSizeBytes();
+        int atomic_offset = pkt->getAddr() - line_addr;
+        std::vector<bool> access_mask(block_size, false);
+        assert(atomic_offset + pkt->getSize() <= block_size);
+
+        for (int idx = 0; idx < pkt->getSize(); ++idx) {
+            access_mask[atomic_offset + idx] = true;
+        }
+
+        std::vector<std::pair<int, AtomicOpFunctor*>> atomic_ops;
+        std::pair<int, AtomicOpFunctor*>
+            atomic_op(atomic_offset, pkt->getAtomicOp());
+
+        atomic_ops.emplace_back(atomic_op);
+        msg->getwriteMask().setAtomicOps(atomic_ops);
+    } else if (write) {
+        msg->setType(SequencerRequestType_ST);
+    } else {
+        assert(pkt->isRead());
+        msg->setType(SequencerRequestType_LD);
+    }
+
     int offset = paddr & m_data_block_mask;
 
     msg->getLen() = (offset + len) <= RubySystem::getBlockSizeBytes() ?
@@ -208,6 +236,25 @@ DMASequencer::ackCallback(const Addr& address)
     issueNext(address);
 }
 
+void
+DMASequencer::atomicCallback(const DataBlock& dblk, const Addr& address)
+{
+    RequestTable::iterator i = m_RequestTable.find(address);
+    assert(i != m_RequestTable.end());
+
+    DMARequest &active_request = i->second;
+    PacketPtr pkt = active_request.pkt;
+
+    int offset = active_request.start_paddr & m_data_block_mask;
+    memcpy(pkt->getPtr<uint8_t>(), dblk.getData(offset, pkt->getSize()),
+           pkt->getSize());
+
+    ruby_hit_callback(pkt);
+
+    m_outstanding_count--;
+    m_RequestTable.erase(i);
+}
+
 void
 DMASequencer::recordRequestType(DMASequencerRequestType requestType)
 {

src/mem/ruby/system/DMASequencer.hh

@@ -70,6 +70,7 @@ class DMASequencer : public RubyPort
     /* SLICC callback */
     void dataCallback(const DataBlock &dblk, const Addr &addr);
     void ackCallback(const Addr &addr);
+    void atomicCallback(const DataBlock &dblk, const Addr &addr);
 
     void recordRequestType(DMASequencerRequestType requestType);