mem-ruby,mem-garnet: Multiple networks per RubySystem

Add support for multiple networks per RubySystem. This is done by
introducing local IDs to each network and translating from a global ID
passed around through Ruby and SLICC code. The local IDs represents the
NodeID of a MachineType in the network and are ordered the same way
that NodeIDs are ordered using MachineType_base_number. If there are
not multiple networks in a RubySystem the local and global IDs are the
same value.

This is useful in cases where multiple isolated networks are needed to
support devices with Ruby caches which do not interact with other
networks. For example, a dGPU device will have a cache hierarchy that
will not interact with the CPU cache hierachy.

Change-Id: I33a917b3a394eec84b16fbf001c3c2c44c047f66
JIRA: https://gem5.atlassian.net/browse/GEM5-445
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/27927
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Reviewed-by: Bradford Beckmann <brad.beckmann@amd.com>
Maintainer: Bradford Beckmann <brad.beckmann@amd.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/mem/ruby/network/Network.cc

@@ -55,9 +55,35 @@ Network::Network(const Params *p)
     m_virtual_networks = p->number_of_virtual_networks;
     m_control_msg_size = p->control_msg_size;
 
-    // Total nodes/controllers in network
+    // Populate localNodeVersions with the version of each MachineType in
+    // this network. This will be used to compute a global to local ID.
+    // Do this by looking at the ext_node for each ext_link. There is one
+    // ext_node per ext_link and it points to an AbstractController.
+    // For RubySystems with one network global and local ID are the same.
+    std::unordered_map<MachineType, std::vector<NodeID>> localNodeVersions;
+    for (auto &it : params()->ext_links) {
+        AbstractController *cntrl = it->params()->ext_node;
+        localNodeVersions[cntrl->getType()].push_back(cntrl->getVersion());
+    }
+
+    // Compute a local ID for each MachineType using the same order as SLICC
+    NodeID local_node_id = 0;
+    for (int i = 0; i < MachineType_base_level(MachineType_NUM); ++i) {
+        MachineType mach = static_cast<MachineType>(i);
+        if (localNodeVersions.count(mach)) {
+            for (auto &ver : localNodeVersions.at(mach)) {
+                // Get the global ID Ruby will pass around
+                NodeID global_node_id = MachineType_base_number(mach) + ver;
+                globalToLocalMap.emplace(global_node_id, local_node_id);
+                ++local_node_id;
+            }
+        }
+    }
+
+    // Total nodes/controllers in network is equal to the local node count
     // Must make sure this is called after the State Machine constructors
-    m_nodes = MachineType_base_number(MachineType_NUM);
+    m_nodes = local_node_id;
+
     assert(m_nodes != 0);
     assert(m_virtual_networks != 0);
 
@@ -158,11 +184,11 @@ Network::MessageSizeType_to_int(MessageSizeType size_type)
 }
 
 void
-Network::checkNetworkAllocation(NodeID id, bool ordered,
+Network::checkNetworkAllocation(NodeID local_id, bool ordered,
                                         int network_num,
                                         std::string vnet_type)
 {
-    fatal_if(id >= m_nodes, "Node ID is out of range");
+    fatal_if(local_id >= m_nodes, "Node ID is out of range");
     fatal_if(network_num >= m_virtual_networks, "Network id is out of range");
 
     if (ordered) {
@@ -174,25 +200,29 @@ Network::checkNetworkAllocation(NodeID id, bool ordered,
 
 
 void
-Network::setToNetQueue(NodeID id, bool ordered, int network_num,
+Network::setToNetQueue(NodeID global_id, bool ordered, int network_num,
                                  std::string vnet_type, MessageBuffer *b)
 {
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_toNetQueues[id].size() <= network_num) {
-        m_toNetQueues[id].push_back(nullptr);
+    NodeID local_id = getLocalNodeID(global_id);
+    checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+    while (m_toNetQueues[local_id].size() <= network_num) {
+        m_toNetQueues[local_id].push_back(nullptr);
     }
-    m_toNetQueues[id][network_num] = b;
+    m_toNetQueues[local_id][network_num] = b;
 }
 
 void
-Network::setFromNetQueue(NodeID id, bool ordered, int network_num,
+Network::setFromNetQueue(NodeID global_id, bool ordered, int network_num,
                                    std::string vnet_type, MessageBuffer *b)
 {
-    checkNetworkAllocation(id, ordered, network_num, vnet_type);
-    while (m_fromNetQueues[id].size() <= network_num) {
-        m_fromNetQueues[id].push_back(nullptr);
+    NodeID local_id = getLocalNodeID(global_id);
+    checkNetworkAllocation(local_id, ordered, network_num, vnet_type);
+
+    while (m_fromNetQueues[local_id].size() <= network_num) {
+        m_fromNetQueues[local_id].push_back(nullptr);
     }
-    m_fromNetQueues[id][network_num] = b;
+    m_fromNetQueues[local_id][network_num] = b;
 }
 
 NodeID
@@ -212,3 +242,10 @@ Network::addressToNodeID(Addr addr, MachineType mtype)
     }
     return MachineType_base_count(mtype);
 }
+
+NodeID
+Network::getLocalNodeID(NodeID global_id) const
+{
+    assert(globalToLocalMap.count(global_id));
+    return globalToLocalMap.at(global_id);
+}

src/mem/ruby/network/Network.hh

@@ -90,13 +90,13 @@ class Network : public ClockedObject
     static uint32_t MessageSizeType_to_int(MessageSizeType size_type);
 
     // returns the queue requested for the given component
-    void setToNetQueue(NodeID id, bool ordered, int netNumber,
+    void setToNetQueue(NodeID global_id, bool ordered, int netNumber,
                                std::string vnet_type, MessageBuffer *b);
-    virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber,
+    virtual void setFromNetQueue(NodeID global_id, bool ordered, int netNumber,
                                  std::string vnet_type, MessageBuffer *b);
 
-    virtual void checkNetworkAllocation(NodeID id, bool ordered,
-        int network_num, std::string vnet_type);
+    virtual void checkNetworkAllocation(NodeID local_id, bool ordered,
+                                       int network_num, std::string vnet_type);
 
     virtual void makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
                              const NetDest& routing_table_entry) = 0;
@@ -140,6 +140,8 @@ class Network : public ClockedObject
         return RubyDummyPort::instance();
     }
 
+    NodeID getLocalNodeID(NodeID global_id) const;
+
   protected:
     // Private copy constructor and assignment operator
     Network(const Network& obj);
@@ -182,6 +184,10 @@ class Network : public ClockedObject
         AddrRangeList ranges;
     };
     std::unordered_multimap<MachineType, AddrMapNode> addrMap;
+
+    // Global NodeID to local node map. If there are not multiple networks in
+    // the same RubySystem, this is a one-to-one mapping of global to local.
+    std::unordered_map<NodeID, NodeID> globalToLocalMap;
 };
 
 inline std::ostream&

src/mem/ruby/network/garnet2.0/GarnetNetwork.cc

@@ -146,10 +146,11 @@ GarnetNetwork::init()
 */
 
 void
-GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
                             const NetDest& routing_table_entry)
 {
-    assert(src < m_nodes);
+    NodeID local_src = getLocalNodeID(global_src);
+    assert(local_src < m_nodes);
 
     GarnetExtLink* garnet_link = safe_cast<GarnetExtLink*>(link);
 
@@ -163,7 +164,7 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
 
     PortDirection dst_inport_dirn = "Local";
     m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link);
-    m_nis[src]->addOutPort(net_link, credit_link, dest);
+    m_nis[local_src]->addOutPort(net_link, credit_link, dest);
 }
 
 /*
@@ -173,10 +174,12 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
 */
 
 void
-GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
-                             const NetDest& routing_table_entry)
+GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+                              BasicLink* link,
+                              const NetDest& routing_table_entry)
 {
-    assert(dest < m_nodes);
+    NodeID local_dest = getLocalNodeID(global_dest);
+    assert(local_dest < m_nodes);
     assert(src < m_routers.size());
     assert(m_routers[src] != NULL);
 
@@ -194,7 +197,7 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
     m_routers[src]->addOutPort(src_outport_dirn, net_link,
                                routing_table_entry,
                                link->m_weight, credit_link);
-    m_nis[dest]->addInPort(net_link, credit_link);
+    m_nis[local_dest]->addInPort(net_link, credit_link);
 }
 
 /*
@@ -233,9 +236,11 @@ GarnetNetwork::getNumRouters()
 
 // Get ID of router connected to a NI.
 int
-GarnetNetwork::get_router_id(int ni)
+GarnetNetwork::get_router_id(int global_ni)
 {
-    return m_nis[ni]->get_router_id();
+    NodeID local_ni = getLocalNodeID(global_ni);
+
+    return m_nis[local_ni]->get_router_id();
 }
 
 void

src/mem/ruby/network/simple/SimpleNetwork.cc

@@ -83,27 +83,30 @@ SimpleNetwork::init()
 
 // From a switch to an endpoint node
 void
-SimpleNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link,
-                           const NetDest& routing_table_entry)
+SimpleNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
+                              BasicLink* link,
+                              const NetDest& routing_table_entry)
 {
-    assert(dest < m_nodes);
+    NodeID local_dest = getLocalNodeID(global_dest);
+    assert(local_dest < m_nodes);
     assert(src < m_switches.size());
     assert(m_switches[src] != NULL);
 
     SimpleExtLink *simple_link = safe_cast<SimpleExtLink*>(link);
 
-    m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry,
-                                simple_link->m_latency,
+    m_switches[src]->addOutPort(m_fromNetQueues[local_dest],
+                                routing_table_entry, simple_link->m_latency,
                                 simple_link->m_bw_multiplier);
 }
 
 // From an endpoint node to a switch
 void
-SimpleNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link,
+SimpleNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
                           const NetDest& routing_table_entry)
 {
-    assert(src < m_nodes);
-    m_switches[dest]->addInPort(m_toNetQueues[src]);
+    NodeID local_src = getLocalNodeID(global_src);
+    assert(local_src < m_nodes);
+    m_switches[dest]->addInPort(m_toNetQueues[local_src]);
 }
 
 // From a switch to a switch

src/mem/ruby/system/RubySystem.cc

@@ -94,7 +94,7 @@ RubySystem::RubySystem(const Params *p)
 void
 RubySystem::registerNetwork(Network* network_ptr)
 {
-    m_network = network_ptr;
+    m_networks.emplace_back(network_ptr);
 }
 
 void
@@ -108,7 +108,6 @@ RubySystem::registerAbstractController(AbstractController* cntrl)
 
 RubySystem::~RubySystem()
 {
-    delete m_network;
     delete m_profiler;
 }
 
@@ -407,7 +406,9 @@ void
 RubySystem::resetStats()
 {
     m_start_cycle = curCycle();
-    m_network->resetStats();
+    for (auto& network : m_networks) {
+        network->resetStats();
+    }
 }
 
 bool
@@ -512,8 +513,10 @@ RubySystem::functionalRead(PacketPtr pkt)
         DPRINTF(RubySystem, "Network functionalRead lookup "
                             "(num_maybe_stale=%d, num_busy = %d)\n",
                 num_maybe_stale, num_busy);
-        if (m_network->functionalRead(pkt))
-            return true;
+        for (auto& network : m_networks) {
+            if (network->functionalRead(pkt))
+                return true;
+        }
     }
 
     return false;
@@ -558,7 +561,9 @@ RubySystem::functionalWrite(PacketPtr pkt)
         }
     }
 
-    num_functional_writes += m_network->functionalWrite(pkt);
+    for (auto& network : m_networks) {
+        num_functional_writes += network->functionalWrite(pkt);
+    }
     DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes);
 
     return true;

src/mem/ruby/system/RubySystem.hh

@@ -130,7 +130,8 @@ class RubySystem : public ClockedObject
     SimpleMemory *m_phys_mem;
     const bool m_access_backing_store;
 
-    Network* m_network;
+    //std::vector<Network *> m_networks;
+    std::vector<std::unique_ptr<Network>> m_networks;
     std::vector<AbstractController *> m_abs_cntrl_vec;
     Cycles m_start_cycle;