diff --git a/src/mem/ruby/network/MessageBuffer.cc b/src/mem/ruby/network/MessageBuffer.cc index 1dc3e843c0..d67b678f5a 100644 --- a/src/mem/ruby/network/MessageBuffer.cc +++ b/src/mem/ruby/network/MessageBuffer.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019,2020 ARM Limited + * Copyright (c) 2019-2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -486,7 +486,7 @@ MessageBuffer::isReady(Tick current_time) const } uint32_t -MessageBuffer::functionalAccess(Packet *pkt, bool is_read) +MessageBuffer::functionalAccess(Packet *pkt, bool is_read, WriteMask *mask) { DPRINTF(RubyQueue, "functional %s for %#x\n", is_read ? "read" : "write", pkt->getAddr()); @@ -497,8 +497,10 @@ MessageBuffer::functionalAccess(Packet *pkt, bool is_read) // correspond to the address in the packet. for (unsigned int i = 0; i < m_prio_heap.size(); ++i) { Message *msg = m_prio_heap[i].get(); - if (is_read && msg->functionalRead(pkt)) + if (is_read && !mask && msg->functionalRead(pkt)) return 1; + else if (is_read && mask && msg->functionalRead(pkt, *mask)) + num_functional_accesses++; else if (!is_read && msg->functionalWrite(pkt)) num_functional_accesses++; } @@ -513,8 +515,10 @@ MessageBuffer::functionalAccess(Packet *pkt, bool is_read) it != (map_iter->second).end(); ++it) { Message *msg = (*it).get(); - if (is_read && msg->functionalRead(pkt)) + if (is_read && !mask && msg->functionalRead(pkt)) return 1; + else if (is_read && mask && msg->functionalRead(pkt, *mask)) + num_functional_accesses++; else if (!is_read && msg->functionalWrite(pkt)) num_functional_accesses++; } diff --git a/src/mem/ruby/network/MessageBuffer.hh b/src/mem/ruby/network/MessageBuffer.hh index b09cb8ab1f..4d70f3088e 100644 --- a/src/mem/ruby/network/MessageBuffer.hh +++ b/src/mem/ruby/network/MessageBuffer.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019,2020 ARM Limited + * Copyright (c) 2019-2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -160,7 +160,7 @@ class MessageBuffer : public SimObject // Return value indicates the number of messages that were updated. uint32_t functionalWrite(Packet *pkt) { - return functionalAccess(pkt, false); + return functionalAccess(pkt, false, nullptr); } // Function for figuring if message in the buffer has valid data for @@ -169,13 +169,19 @@ class MessageBuffer : public SimObject // read was performed. bool functionalRead(Packet *pkt) { - return functionalAccess(pkt, true) == 1; + return functionalAccess(pkt, true, nullptr) == 1; + } + + // Functional read with mask + bool functionalRead(Packet *pkt, WriteMask &mask) + { + return functionalAccess(pkt, true, &mask) == 1; } private: void reanalyzeList(std::list &, Tick); - uint32_t functionalAccess(Packet *pkt, bool is_read); + uint32_t functionalAccess(Packet *pkt, bool is_read, WriteMask *mask); private: // Data Members (m_ prefix) diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh index 8af1610a18..9fb94c3ef2 100644 --- a/src/mem/ruby/network/Network.hh +++ b/src/mem/ruby/network/Network.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited + * Copyright (c) 2017,2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -119,6 +119,8 @@ class Network : public ClockedObject */ virtual bool functionalRead(Packet *pkt) { fatal("Functional read not implemented.\n"); } + virtual bool functionalRead(Packet *pkt, WriteMask& mask) + { fatal("Masked functional read not implemented.\n"); } virtual uint32_t functionalWrite(Packet *pkt) { fatal("Functional write not implemented.\n"); } diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc index dc1551db2e..0f9056513a 100644 --- a/src/mem/ruby/network/simple/SimpleNetwork.cc +++ b/src/mem/ruby/network/simple/SimpleNetwork.cc @@ -1,6 +1,6 @@ /* * Copyright (c) 2020 Advanced Micro Devices, Inc. - * Copyright (c) 2019 ARM Limited + * Copyright (c) 2019,2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -203,6 +203,21 @@ SimpleNetwork::functionalRead(Packet *pkt) return false; } +bool +SimpleNetwork::functionalRead(Packet *pkt, WriteMask &mask) +{ + bool read = false; + for (unsigned int i = 0; i < m_switches.size(); i++) { + if (m_switches[i]->functionalRead(pkt, mask)) + read = true; + } + for (unsigned int i = 0; i < m_int_link_buffers.size(); ++i) { + if (m_int_link_buffers[i]->functionalRead(pkt, mask)) + read = true; + } + return read; +} + uint32_t SimpleNetwork::functionalWrite(Packet *pkt) { diff --git a/src/mem/ruby/network/simple/SimpleNetwork.hh b/src/mem/ruby/network/simple/SimpleNetwork.hh index e76b7d1be9..55546a086b 100644 --- a/src/mem/ruby/network/simple/SimpleNetwork.hh +++ b/src/mem/ruby/network/simple/SimpleNetwork.hh @@ -1,4 +1,16 @@ /* + * Copyright (c) 2021 ARM Limited + * All rights reserved. + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood * All rights reserved. * @@ -71,6 +83,7 @@ class SimpleNetwork : public Network void print(std::ostream& out) const; bool functionalRead(Packet *pkt); + bool functionalRead(Packet *pkt, WriteMask &mask); uint32_t functionalWrite(Packet *pkt); private: diff --git a/src/mem/ruby/network/simple/Switch.cc b/src/mem/ruby/network/simple/Switch.cc index c3b583553a..2eeddb6663 100644 --- a/src/mem/ruby/network/simple/Switch.cc +++ b/src/mem/ruby/network/simple/Switch.cc @@ -1,6 +1,6 @@ /* * Copyright (c) 2020 Inria - * Copyright (c) 2019 ARM Limited + * Copyright (c) 2019,2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -176,6 +176,17 @@ Switch::functionalRead(Packet *pkt) return false; } +bool +Switch::functionalRead(Packet *pkt, WriteMask &mask) +{ + bool read = false; + for (unsigned int i = 0; i < m_port_buffers.size(); ++i) { + if (m_port_buffers[i]->functionalRead(pkt, mask)) + read = true; + } + return read; +} + uint32_t Switch::functionalWrite(Packet *pkt) { diff --git a/src/mem/ruby/network/simple/Switch.hh b/src/mem/ruby/network/simple/Switch.hh index a9502b946a..271d090750 100644 --- a/src/mem/ruby/network/simple/Switch.hh +++ b/src/mem/ruby/network/simple/Switch.hh @@ -1,4 +1,16 @@ /* + * Copyright (c) 2021 ARM Limited + * All rights reserved. + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * * Copyright (c) 2020 Inria * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood * All rights reserved. @@ -79,6 +91,7 @@ class Switch : public BasicRouter void init_net_ptr(SimpleNetwork* net_ptr) { m_network_ptr = net_ptr; } bool functionalRead(Packet *); + bool functionalRead(Packet *, WriteMask&); uint32_t functionalWrite(Packet *); private: diff --git a/src/mem/ruby/protocol/RubySlicc_Exports.sm b/src/mem/ruby/protocol/RubySlicc_Exports.sm index a3cbafa6a5..0eb10a7aa0 100644 --- a/src/mem/ruby/protocol/RubySlicc_Exports.sm +++ b/src/mem/ruby/protocol/RubySlicc_Exports.sm @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 ARM Limited + * Copyright (c) 2020-2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -107,6 +107,7 @@ enumeration(AccessPermission, desc="...", default="AccessPermission_NotPresent") // This is not supposed to be used in directory or token protocols where // memory/NB has an idea of what is going on in the whole system. Backing_Store, desc="for memory in Broadcast/Snoop protocols"; + Backing_Store_Busy, desc="Backing_Store + cntrl is busy waiting for data"; // Invalid data Invalid, desc="block is in an Invalid base state"; diff --git a/src/mem/ruby/protocol/RubySlicc_MemControl.sm b/src/mem/ruby/protocol/RubySlicc_MemControl.sm index 801a7bb75c..e8517a4a07 100644 --- a/src/mem/ruby/protocol/RubySlicc_MemControl.sm +++ b/src/mem/ruby/protocol/RubySlicc_MemControl.sm @@ -1,5 +1,17 @@ /* + * Copyright (c) 2021 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * * Copyright (c) 1999-2005 Mark D. Hill and David A. Wood * All rights reserved. * @@ -64,7 +76,29 @@ structure(MemoryMsg, desc="...", interface="Message") { int Acks, desc="How many acks to expect"; bool functionalRead(Packet *pkt) { - return testAndRead(addr, DataBlk, pkt); + if ((MessageSize == MessageSizeType:Response_Data) || + (MessageSize == MessageSizeType:Writeback_Data)) { + return testAndRead(addr, DataBlk, pkt); + } + return false; + } + + bool functionalRead(Packet *pkt, WriteMask &mask) { + if ((MessageSize == MessageSizeType:Response_Data) || + (MessageSize == MessageSizeType:Writeback_Data)) { + WriteMask read_mask; + read_mask.setMask(addressOffset(addr, makeLineAddress(addr)), Len, true); + if (MessageSize != MessageSizeType:Writeback_Data) { + read_mask.setInvertedMask(mask); + } + if (read_mask.isEmpty()) { + return false; + } else if (testAndReadMask(addr, DataBlk, read_mask, pkt)) { + mask.orMask(read_mask); + return true; + } + } + return false; } bool functionalWrite(Packet *pkt) { diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh index 95be144988..93337398c0 100644 --- a/src/mem/ruby/slicc_interface/AbstractController.hh +++ b/src/mem/ruby/slicc_interface/AbstractController.hh @@ -117,7 +117,16 @@ class AbstractController : public ClockedObject, public Consumer //! These functions are used by ruby system to read/write the data blocks //! that exist with in the controller. virtual bool functionalReadBuffers(PacketPtr&) = 0; - virtual void functionalRead(const Addr &addr, PacketPtr) = 0; + virtual void functionalRead(const Addr &addr, PacketPtr) + { panic("functionalRead(Addr,PacketPtr) not implemented"); } + + //! Functional read that reads only blocks not present in the mask. + //! Return number of bytes read. + virtual bool functionalReadBuffers(PacketPtr&, WriteMask &mask) = 0; + virtual void functionalRead(const Addr &addr, PacketPtr pkt, + WriteMask &mask) + { panic("functionalRead(Addr,PacketPtr,WriteMask) not implemented"); } + void functionalMemoryRead(PacketPtr); //! The return value indicates the number of messages written with the //! data from the packet. diff --git a/src/mem/ruby/slicc_interface/Message.hh b/src/mem/ruby/slicc_interface/Message.hh index 1044fe022c..d7acd2c015 100644 --- a/src/mem/ruby/slicc_interface/Message.hh +++ b/src/mem/ruby/slicc_interface/Message.hh @@ -1,4 +1,16 @@ /* + * Copyright (c) 2021 ARM Limited + * All rights reserved. + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * * Copyright (c) 1999-2008 Mark D. Hill and David A. Wood * All rights reserved. * @@ -35,6 +47,7 @@ #include "mem/packet.hh" #include "mem/ruby/common/NetDest.hh" +#include "mem/ruby/common/WriteMask.hh" #include "mem/ruby/protocol/MessageSizeType.hh" class Message; @@ -73,8 +86,12 @@ class Message * class that can be potentially searched for the address needs to * implement these methods. */ - virtual bool functionalRead(Packet *pkt) = 0; - virtual bool functionalWrite(Packet *pkt) = 0; + virtual bool functionalRead(Packet *pkt) + { panic("functionalRead(Packet) not implemented"); } + virtual bool functionalRead(Packet *pkt, WriteMask &mask) + { panic("functionalRead(Packet,WriteMask) not implemented"); } + virtual bool functionalWrite(Packet *pkt) + { panic("functionalWrite(Packet) not implemented"); } //! Update the delay this message has experienced so far. void updateDelayedTicks(Tick curTime) diff --git a/src/mem/ruby/slicc_interface/RubyRequest.cc b/src/mem/ruby/slicc_interface/RubyRequest.cc index 05ed26397a..af91acac32 100644 --- a/src/mem/ruby/slicc_interface/RubyRequest.cc +++ b/src/mem/ruby/slicc_interface/RubyRequest.cc @@ -69,6 +69,12 @@ RubyRequest::functionalRead(Packet *pkt) return false; } +bool +RubyRequest::functionalRead(Packet *pkt, WriteMask &mask) +{ + return false; +} + bool RubyRequest::functionalWrite(Packet *pkt) { diff --git a/src/mem/ruby/slicc_interface/RubyRequest.hh b/src/mem/ruby/slicc_interface/RubyRequest.hh index 14318c7cd9..55b645e936 100644 --- a/src/mem/ruby/slicc_interface/RubyRequest.hh +++ b/src/mem/ruby/slicc_interface/RubyRequest.hh @@ -157,6 +157,7 @@ class RubyRequest : public Message void print(std::ostream& out) const; bool functionalRead(Packet *pkt); + bool functionalRead(Packet *pkt, WriteMask &mask); bool functionalWrite(Packet *pkt); }; diff --git a/src/mem/ruby/system/RubySystem.cc b/src/mem/ruby/system/RubySystem.cc index 5aca4a8c5d..18ebc7ff42 100644 --- a/src/mem/ruby/system/RubySystem.cc +++ b/src/mem/ruby/system/RubySystem.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited + * Copyright (c) 2019,2021 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -472,6 +472,7 @@ RubySystem::resetStats() } } +#ifndef PARTIAL_FUNC_READS bool RubySystem::functionalRead(PacketPtr pkt) { @@ -587,6 +588,95 @@ RubySystem::functionalRead(PacketPtr pkt) return false; } +#else +bool +RubySystem::functionalRead(PacketPtr pkt) +{ + Addr address(pkt->getAddr()); + Addr line_address = makeLineAddress(address); + + DPRINTF(RubySystem, "Functional Read request for %#x\n", address); + + std::vector ctrl_ro; + std::vector ctrl_busy; + std::vector ctrl_others; + AbstractController *ctrl_rw = nullptr; + AbstractController *ctrl_bs = nullptr; + + // Build lists of controllers that have line + for (auto ctrl : m_abs_cntrl_vec) { + switch(ctrl->getAccessPermission(line_address)) { + case AccessPermission_Read_Only: + ctrl_ro.push_back(ctrl); + break; + case AccessPermission_Busy: + ctrl_busy.push_back(ctrl); + break; + case AccessPermission_Read_Write: + assert(ctrl_rw == nullptr); + ctrl_rw = ctrl; + break; + case AccessPermission_Backing_Store: + assert(ctrl_bs == nullptr); + ctrl_bs = ctrl; + break; + case AccessPermission_Backing_Store_Busy: + assert(ctrl_bs == nullptr); + ctrl_bs = ctrl; + ctrl_busy.push_back(ctrl); + break; + default: + ctrl_others.push_back(ctrl); + break; + } + } + + DPRINTF(RubySystem, "num_ro=%d, num_busy=%d , has_rw=%d, " + "backing_store=%d\n", + ctrl_ro.size(), ctrl_busy.size(), + ctrl_rw != nullptr, ctrl_bs != nullptr); + + // Issue functional reads to all controllers found in a stable state + // until we get a full copy of the line + WriteMask bytes; + if (ctrl_rw != nullptr) { + ctrl_rw->functionalRead(line_address, pkt, bytes); + // if a RW controllter has the full line that's all uptodate + if (bytes.isFull()) + return true; + } + + // Get data from RO and BS + for (auto ctrl : ctrl_ro) + ctrl->functionalRead(line_address, pkt, bytes); + + ctrl_bs->functionalRead(line_address, pkt, bytes); + + // if there is any busy controller or bytes still not set, then a partial + // and/or dirty copy of the line might be in a message buffer or the + // network + if (!ctrl_busy.empty() || !bytes.isFull()) { + DPRINTF(RubySystem, "Reading from busy controllers and network\n"); + for (auto ctrl : ctrl_busy) { + ctrl->functionalRead(line_address, pkt, bytes); + ctrl->functionalReadBuffers(pkt, bytes); + } + for (auto& network : m_networks) { + network->functionalRead(pkt, bytes); + } + for (auto ctrl : ctrl_others) { + ctrl->functionalRead(line_address, pkt, bytes); + ctrl->functionalReadBuffers(pkt, bytes); + } + } + // we either got the full line or couldn't find anything at this point + panic_if(!(bytes.isFull() || bytes.isEmpty()), + "Inconsistent state on functional read for %#x %s\n", + address, bytes); + + return bytes.isFull(); +} +#endif // The function searches through all the buffers that exist in different // cache, directory and memory controllers, and in the network components diff --git a/src/mem/ruby/system/SConscript b/src/mem/ruby/system/SConscript index a5d2fb11ea..a2708ae9db 100644 --- a/src/mem/ruby/system/SConscript +++ b/src/mem/ruby/system/SConscript @@ -45,6 +45,12 @@ if env['PROTOCOL'] == 'None': env.Append(CPPDEFINES=['PROTOCOL_' + env['PROTOCOL']]) +# list of protocols that require the partial functional read interface +need_partial_func_reads = [] + +if env['PROTOCOL'] in need_partial_func_reads: + env.Append(CPPDEFINES=['PARTIAL_FUNC_READS']) + if env['BUILD_GPU']: SimObject('GPUCoalescer.py') SimObject('RubySystem.py') diff --git a/src/mem/slicc/symbols/StateMachine.py b/src/mem/slicc/symbols/StateMachine.py index 012441d99c..59e54a8994 100644 --- a/src/mem/slicc/symbols/StateMachine.py +++ b/src/mem/slicc/symbols/StateMachine.py @@ -328,6 +328,7 @@ class $c_ident : public AbstractController GPUCoalescer* getGPUCoalescer() const; bool functionalReadBuffers(PacketPtr&); + bool functionalReadBuffers(PacketPtr&, WriteMask&); int functionalWriteBuffers(PacketPtr&); void countTransition(${ident}_State state, ${ident}_Event event); @@ -1182,6 +1183,27 @@ $c_ident::functionalReadBuffers(PacketPtr& pkt) code(''' return false; } + +bool +$c_ident::functionalReadBuffers(PacketPtr& pkt, WriteMask &mask) +{ + bool read = false; +''') + for var in self.objects: + vtype = var.type + if vtype.isBuffer: + vid = "m_%s_ptr" % var.ident + code('if ($vid->functionalRead(pkt, mask)) read = true;') + + for var in self.config_parameters: + vtype = var.type_ast.type + if vtype.isBuffer: + vid = "m_%s_ptr" % var.ident + code('if ($vid->functionalRead(pkt, mask)) read = true;') + + code(''' + return read; +} ''') code.write(path, "%s.cc" % c_ident)