mem-ruby: alternative interface for func. reads

A single functionalRead may not be able to get the whole latest
copy of the block in protocols that have features such as:

- a cache line can be partially present and dirty in a controller
- a cache line can be transferred over the network using multiple
  protocol-level messages

To support these cases, this patch adds an alternative function:

bool functionalRead(PacketPtr, WriteMask&)

Protocols that implement this function can partially update
the packet and use the WriteMask to mark updated bytes.
The top-level RubySystem:functionalRead then issues functionalRead
to controllers until the whole block is read.
This patch implements functionalRead(PacketPtr, WriteMask&) for all the
common messages and SimpleNetwork. A protocol-specific implementation
will be provided in a future patch.

The new interface is compiled only if required by the protocol (see
src/mem/ruby/system/SConscript). Otherwise the original interface is
used thus maintaining compatibility with previous protocols.

Change-Id: I4600d5f1d7cc170bd7b09ccd09bfd3bb6605f86b
Signed-off-by: Tiago Mück <tiago.muck@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/31416
Reviewed-by: Matthew Poremba <matthew.poremba@amd.com>
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/mem/ruby/network/MessageBuffer.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019,2020 ARM Limited
+ * Copyright (c) 2019-2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -486,7 +486,7 @@ MessageBuffer::isReady(Tick current_time) const
 }
 
 uint32_t
-MessageBuffer::functionalAccess(Packet *pkt, bool is_read)
+MessageBuffer::functionalAccess(Packet *pkt, bool is_read, WriteMask *mask)
 {
     DPRINTF(RubyQueue, "functional %s for %#x\n",
             is_read ? "read" : "write", pkt->getAddr());
@@ -497,8 +497,10 @@ MessageBuffer::functionalAccess(Packet *pkt, bool is_read)
     // correspond to the address in the packet.
     for (unsigned int i = 0; i < m_prio_heap.size(); ++i) {
         Message *msg = m_prio_heap[i].get();
-        if (is_read && msg->functionalRead(pkt))
+        if (is_read && !mask && msg->functionalRead(pkt))
             return 1;
+        else if (is_read && mask && msg->functionalRead(pkt, *mask))
+            num_functional_accesses++;
         else if (!is_read && msg->functionalWrite(pkt))
             num_functional_accesses++;
     }
@@ -513,8 +515,10 @@ MessageBuffer::functionalAccess(Packet *pkt, bool is_read)
             it != (map_iter->second).end(); ++it) {
 
             Message *msg = (*it).get();
-            if (is_read && msg->functionalRead(pkt))
+            if (is_read && !mask && msg->functionalRead(pkt))
                 return 1;
+            else if (is_read && mask && msg->functionalRead(pkt, *mask))
+                num_functional_accesses++;
             else if (!is_read && msg->functionalWrite(pkt))
                 num_functional_accesses++;
         }

src/mem/ruby/network/MessageBuffer.hh

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019,2020 ARM Limited
+ * Copyright (c) 2019-2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -160,7 +160,7 @@ class MessageBuffer : public SimObject
     // Return value indicates the number of messages that were updated.
     uint32_t functionalWrite(Packet *pkt)
     {
-        return functionalAccess(pkt, false);
+        return functionalAccess(pkt, false, nullptr);
     }
 
     // Function for figuring if message in the buffer has valid data for
@@ -169,13 +169,19 @@ class MessageBuffer : public SimObject
     // read was performed.
     bool functionalRead(Packet *pkt)
     {
-        return functionalAccess(pkt, true) == 1;
+        return functionalAccess(pkt, true, nullptr) == 1;
+    }
+
+    // Functional read with mask
+    bool functionalRead(Packet *pkt, WriteMask &mask)
+    {
+        return functionalAccess(pkt, true, &mask) == 1;
     }
 
   private:
     void reanalyzeList(std::list<MsgPtr> &, Tick);
 
-    uint32_t functionalAccess(Packet *pkt, bool is_read);
+    uint32_t functionalAccess(Packet *pkt, bool is_read, WriteMask *mask);
 
   private:
     // Data Members (m_ prefix)

src/mem/ruby/network/Network.hh

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 ARM Limited
+ * Copyright (c) 2017,2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -119,6 +119,8 @@ class Network : public ClockedObject
      */
     virtual bool functionalRead(Packet *pkt)
     { fatal("Functional read not implemented.\n"); }
+    virtual bool functionalRead(Packet *pkt, WriteMask& mask)
+    { fatal("Masked functional read not implemented.\n"); }
     virtual uint32_t functionalWrite(Packet *pkt)
     { fatal("Functional write not implemented.\n"); }
 

src/mem/ruby/network/simple/SimpleNetwork.cc

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2020 Advanced Micro Devices, Inc.
- * Copyright (c) 2019 ARM Limited
+ * Copyright (c) 2019,2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -203,6 +203,21 @@ SimpleNetwork::functionalRead(Packet *pkt)
     return false;
 }
 
+bool
+SimpleNetwork::functionalRead(Packet *pkt, WriteMask &mask)
+{
+    bool read = false;
+    for (unsigned int i = 0; i < m_switches.size(); i++) {
+        if (m_switches[i]->functionalRead(pkt, mask))
+            read = true;
+    }
+    for (unsigned int i = 0; i < m_int_link_buffers.size(); ++i) {
+        if (m_int_link_buffers[i]->functionalRead(pkt, mask))
+            read = true;
+    }
+    return read;
+}
+
 uint32_t
 SimpleNetwork::functionalWrite(Packet *pkt)
 {

src/mem/ruby/network/simple/SimpleNetwork.hh

@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2021 ARM Limited
+ * All rights reserved.
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
  * All rights reserved.
  *
@@ -71,6 +83,7 @@ class SimpleNetwork : public Network
     void print(std::ostream& out) const;
 
     bool functionalRead(Packet *pkt);
+    bool functionalRead(Packet *pkt, WriteMask &mask);
     uint32_t functionalWrite(Packet *pkt);
 
   private:

src/mem/ruby/network/simple/Switch.cc

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2020 Inria
- * Copyright (c) 2019 ARM Limited
+ * Copyright (c) 2019,2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -176,6 +176,17 @@ Switch::functionalRead(Packet *pkt)
     return false;
 }
 
+bool
+Switch::functionalRead(Packet *pkt, WriteMask &mask)
+{
+    bool read = false;
+    for (unsigned int i = 0; i < m_port_buffers.size(); ++i) {
+        if (m_port_buffers[i]->functionalRead(pkt, mask))
+            read = true;
+    }
+    return read;
+}
+
 uint32_t
 Switch::functionalWrite(Packet *pkt)
 {

src/mem/ruby/network/simple/Switch.hh

@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2021 ARM Limited
+ * All rights reserved.
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 2020 Inria
  * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
  * All rights reserved.
@@ -79,6 +91,7 @@ class Switch : public BasicRouter
     void init_net_ptr(SimpleNetwork* net_ptr) { m_network_ptr = net_ptr; }
 
     bool functionalRead(Packet *);
+    bool functionalRead(Packet *, WriteMask&);
     uint32_t functionalWrite(Packet *);
 
   private:

src/mem/ruby/protocol/RubySlicc_Exports.sm

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 ARM Limited
+ * Copyright (c) 2020-2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -107,6 +107,7 @@ enumeration(AccessPermission, desc="...", default="AccessPermission_NotPresent")
   // This is not supposed to be used in directory or token protocols where
   // memory/NB has an idea of what is going on in the whole system.
   Backing_Store, desc="for memory in Broadcast/Snoop protocols";
+  Backing_Store_Busy, desc="Backing_Store + cntrl is busy waiting for data";
 
   // Invalid data
   Invalid,    desc="block is in an Invalid base state";

src/mem/ruby/protocol/RubySlicc_MemControl.sm

@@ -1,5 +1,17 @@
 
 /*
+ * Copyright (c) 2021 ARM Limited
+ * All rights reserved
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 1999-2005 Mark D. Hill and David A. Wood
  * All rights reserved.
  *
@@ -64,7 +76,29 @@ structure(MemoryMsg, desc="...", interface="Message") {
   int Acks,                     desc="How many acks to expect";
 
   bool functionalRead(Packet *pkt) {
-    return testAndRead(addr, DataBlk, pkt);
+    if ((MessageSize == MessageSizeType:Response_Data) ||
+        (MessageSize == MessageSizeType:Writeback_Data))  {
+      return testAndRead(addr, DataBlk, pkt);
+    }
+    return false;
+  }
+
+  bool functionalRead(Packet *pkt, WriteMask &mask) {
+    if ((MessageSize == MessageSizeType:Response_Data) ||
+        (MessageSize == MessageSizeType:Writeback_Data))  {
+      WriteMask read_mask;
+      read_mask.setMask(addressOffset(addr, makeLineAddress(addr)), Len, true);
+      if (MessageSize != MessageSizeType:Writeback_Data) {
+        read_mask.setInvertedMask(mask);
+      }
+      if (read_mask.isEmpty()) {
+        return false;
+      } else if (testAndReadMask(addr, DataBlk, read_mask, pkt)) {
+        mask.orMask(read_mask);
+        return true;
+      }
+    }
+    return false;
   }
 
   bool functionalWrite(Packet *pkt) {

src/mem/ruby/slicc_interface/AbstractController.hh

@@ -117,7 +117,16 @@ class AbstractController : public ClockedObject, public Consumer
     //! These functions are used by ruby system to read/write the data blocks
     //! that exist with in the controller.
     virtual bool functionalReadBuffers(PacketPtr&) = 0;
-    virtual void functionalRead(const Addr &addr, PacketPtr) = 0;
+    virtual void functionalRead(const Addr &addr, PacketPtr)
+    { panic("functionalRead(Addr,PacketPtr) not implemented"); }
+
+    //! Functional read that reads only blocks not present in the mask.
+    //! Return number of bytes read.
+    virtual bool functionalReadBuffers(PacketPtr&, WriteMask &mask) = 0;
+    virtual void functionalRead(const Addr &addr, PacketPtr pkt,
+                               WriteMask &mask)
+    { panic("functionalRead(Addr,PacketPtr,WriteMask) not implemented"); }
+
     void functionalMemoryRead(PacketPtr);
     //! The return value indicates the number of messages written with the
     //! data from the packet.

src/mem/ruby/slicc_interface/Message.hh

@@ -1,4 +1,16 @@
 /*
+ * Copyright (c) 2021 ARM Limited
+ * All rights reserved.
+ *
+ * The license below extends only to copyright in the software and shall
+ * not be construed as granting a license to any other intellectual
+ * property including but not limited to intellectual property relating
+ * to a hardware implementation of the functionality of the software
+ * licensed hereunder.  You may use the software subject to the license
+ * terms below provided that you ensure that this notice is replicated
+ * unmodified and in its entirety in all distributions of the software,
+ * modified or unmodified, in source code or in binary form.
+ *
  * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood
  * All rights reserved.
  *
@@ -35,6 +47,7 @@
 
 #include "mem/packet.hh"
 #include "mem/ruby/common/NetDest.hh"
+#include "mem/ruby/common/WriteMask.hh"
 #include "mem/ruby/protocol/MessageSizeType.hh"
 
 class Message;
@@ -73,8 +86,12 @@ class Message
      * class that can be potentially searched for the address needs to
      * implement these methods.
      */
-    virtual bool functionalRead(Packet *pkt) = 0;
-    virtual bool functionalWrite(Packet *pkt) = 0;
+    virtual bool functionalRead(Packet *pkt)
+    { panic("functionalRead(Packet) not implemented"); }
+    virtual bool functionalRead(Packet *pkt, WriteMask &mask)
+    { panic("functionalRead(Packet,WriteMask) not implemented"); }
+    virtual bool functionalWrite(Packet *pkt)
+    { panic("functionalWrite(Packet) not implemented"); }
 
     //! Update the delay this message has experienced so far.
     void updateDelayedTicks(Tick curTime)

src/mem/ruby/slicc_interface/RubyRequest.cc

@@ -69,6 +69,12 @@ RubyRequest::functionalRead(Packet *pkt)
     return false;
 }
 
+bool
+RubyRequest::functionalRead(Packet *pkt, WriteMask &mask)
+{
+    return false;
+}
+
 bool
 RubyRequest::functionalWrite(Packet *pkt)
 {

src/mem/ruby/slicc_interface/RubyRequest.hh

@@ -157,6 +157,7 @@ class RubyRequest : public Message
 
     void print(std::ostream& out) const;
     bool functionalRead(Packet *pkt);
+    bool functionalRead(Packet *pkt, WriteMask &mask);
     bool functionalWrite(Packet *pkt);
 };
 

src/mem/ruby/system/RubySystem.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 ARM Limited
+ * Copyright (c) 2019,2021 ARM Limited
  * All rights reserved.
  *
  * The license below extends only to copyright in the software and shall
@@ -472,6 +472,7 @@ RubySystem::resetStats()
     }
 }
 
+#ifndef PARTIAL_FUNC_READS
 bool
 RubySystem::functionalRead(PacketPtr pkt)
 {
@@ -587,6 +588,95 @@ RubySystem::functionalRead(PacketPtr pkt)
 
     return false;
 }
+#else
+bool
+RubySystem::functionalRead(PacketPtr pkt)
+{
+    Addr address(pkt->getAddr());
+    Addr line_address = makeLineAddress(address);
+
+    DPRINTF(RubySystem, "Functional Read request for %#x\n", address);
+
+    std::vector<AbstractController*> ctrl_ro;
+    std::vector<AbstractController*> ctrl_busy;
+    std::vector<AbstractController*> ctrl_others;
+    AbstractController *ctrl_rw = nullptr;
+    AbstractController *ctrl_bs = nullptr;
+
+    // Build lists of controllers that have line
+    for (auto ctrl : m_abs_cntrl_vec) {
+        switch(ctrl->getAccessPermission(line_address)) {
+            case AccessPermission_Read_Only:
+                ctrl_ro.push_back(ctrl);
+                break;
+            case AccessPermission_Busy:
+                ctrl_busy.push_back(ctrl);
+                break;
+            case AccessPermission_Read_Write:
+                assert(ctrl_rw == nullptr);
+                ctrl_rw = ctrl;
+                break;
+            case AccessPermission_Backing_Store:
+                assert(ctrl_bs == nullptr);
+                ctrl_bs = ctrl;
+                break;
+            case AccessPermission_Backing_Store_Busy:
+                assert(ctrl_bs == nullptr);
+                ctrl_bs = ctrl;
+                ctrl_busy.push_back(ctrl);
+                break;
+            default:
+                ctrl_others.push_back(ctrl);
+                break;
+        }
+    }
+
+    DPRINTF(RubySystem, "num_ro=%d, num_busy=%d , has_rw=%d, "
+                        "backing_store=%d\n",
+                ctrl_ro.size(), ctrl_busy.size(),
+                ctrl_rw != nullptr, ctrl_bs != nullptr);
+
+    // Issue functional reads to all controllers found in a stable state
+    // until we get a full copy of the line
+    WriteMask bytes;
+    if (ctrl_rw != nullptr) {
+        ctrl_rw->functionalRead(line_address, pkt, bytes);
+        // if a RW controllter has the full line that's all uptodate
+        if (bytes.isFull())
+            return true;
+    }
+
+    // Get data from RO and BS
+    for (auto ctrl : ctrl_ro)
+        ctrl->functionalRead(line_address, pkt, bytes);
+
+    ctrl_bs->functionalRead(line_address, pkt, bytes);
+
+    // if there is any busy controller or bytes still not set, then a partial
+    // and/or dirty copy of the line might be in a message buffer or the
+    // network
+    if (!ctrl_busy.empty() || !bytes.isFull()) {
+        DPRINTF(RubySystem, "Reading from busy controllers and network\n");
+        for (auto ctrl : ctrl_busy) {
+            ctrl->functionalRead(line_address, pkt, bytes);
+            ctrl->functionalReadBuffers(pkt, bytes);
+        }
+        for (auto& network : m_networks) {
+            network->functionalRead(pkt, bytes);
+        }
+        for (auto ctrl : ctrl_others) {
+            ctrl->functionalRead(line_address, pkt, bytes);
+            ctrl->functionalReadBuffers(pkt, bytes);
+        }
+    }
+    // we either got the full line or couldn't find anything at this point
+    panic_if(!(bytes.isFull() || bytes.isEmpty()),
+            "Inconsistent state on functional read for %#x %s\n",
+            address, bytes);
+
+    return bytes.isFull();
+}
+#endif
 
 // The function searches through all the buffers that exist in different
 // cache, directory and memory controllers, and in the network components

src/mem/ruby/system/SConscript

@@ -45,6 +45,12 @@ if env['PROTOCOL'] == 'None':
 
 env.Append(CPPDEFINES=['PROTOCOL_' + env['PROTOCOL']])
 
+# list of protocols that require the partial functional read interface
+need_partial_func_reads = []
+
+if env['PROTOCOL'] in need_partial_func_reads:
+    env.Append(CPPDEFINES=['PARTIAL_FUNC_READS'])
+
 if env['BUILD_GPU']:
     SimObject('GPUCoalescer.py')
 SimObject('RubySystem.py')

src/mem/slicc/symbols/StateMachine.py

@@ -328,6 +328,7 @@ class $c_ident : public AbstractController
     GPUCoalescer* getGPUCoalescer() const;
 
     bool functionalReadBuffers(PacketPtr&);
+    bool functionalReadBuffers(PacketPtr&, WriteMask&);
     int functionalWriteBuffers(PacketPtr&);
 
     void countTransition(${ident}_State state, ${ident}_Event event);
@@ -1182,6 +1183,27 @@ $c_ident::functionalReadBuffers(PacketPtr& pkt)
         code('''
     return false;
 }
+
+bool
+$c_ident::functionalReadBuffers(PacketPtr& pkt, WriteMask &mask)
+{
+    bool read = false;
+''')
+        for var in self.objects:
+            vtype = var.type
+            if vtype.isBuffer:
+                vid = "m_%s_ptr" % var.ident
+                code('if ($vid->functionalRead(pkt, mask)) read = true;')
+
+        for var in self.config_parameters:
+            vtype = var.type_ast.type
+            if vtype.isBuffer:
+                vid = "m_%s_ptr" % var.ident
+                code('if ($vid->functionalRead(pkt, mask)) read = true;')
+
+        code('''
+    return read;
+}
 ''')
 
         code.write(path, "%s.cc" % c_ident)