diff --git a/src/mem/ruby/network/Network.cc b/src/mem/ruby/network/Network.cc index 57834f2e2a..982b57e27b 100644 --- a/src/mem/ruby/network/Network.cc +++ b/src/mem/ruby/network/Network.cc @@ -55,9 +55,35 @@ Network::Network(const Params *p) m_virtual_networks = p->number_of_virtual_networks; m_control_msg_size = p->control_msg_size; - // Total nodes/controllers in network + // Populate localNodeVersions with the version of each MachineType in + // this network. This will be used to compute a global to local ID. + // Do this by looking at the ext_node for each ext_link. There is one + // ext_node per ext_link and it points to an AbstractController. + // For RubySystems with one network global and local ID are the same. + std::unordered_map> localNodeVersions; + for (auto &it : params()->ext_links) { + AbstractController *cntrl = it->params()->ext_node; + localNodeVersions[cntrl->getType()].push_back(cntrl->getVersion()); + } + + // Compute a local ID for each MachineType using the same order as SLICC + NodeID local_node_id = 0; + for (int i = 0; i < MachineType_base_level(MachineType_NUM); ++i) { + MachineType mach = static_cast(i); + if (localNodeVersions.count(mach)) { + for (auto &ver : localNodeVersions.at(mach)) { + // Get the global ID Ruby will pass around + NodeID global_node_id = MachineType_base_number(mach) + ver; + globalToLocalMap.emplace(global_node_id, local_node_id); + ++local_node_id; + } + } + } + + // Total nodes/controllers in network is equal to the local node count // Must make sure this is called after the State Machine constructors - m_nodes = MachineType_base_number(MachineType_NUM); + m_nodes = local_node_id; + assert(m_nodes != 0); assert(m_virtual_networks != 0); @@ -158,11 +184,11 @@ Network::MessageSizeType_to_int(MessageSizeType size_type) } void -Network::checkNetworkAllocation(NodeID id, bool ordered, +Network::checkNetworkAllocation(NodeID local_id, bool ordered, int network_num, std::string vnet_type) { - fatal_if(id >= m_nodes, "Node ID is out of range"); + fatal_if(local_id >= m_nodes, "Node ID is out of range"); fatal_if(network_num >= m_virtual_networks, "Network id is out of range"); if (ordered) { @@ -174,25 +200,29 @@ Network::checkNetworkAllocation(NodeID id, bool ordered, void -Network::setToNetQueue(NodeID id, bool ordered, int network_num, +Network::setToNetQueue(NodeID global_id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { - checkNetworkAllocation(id, ordered, network_num, vnet_type); - while (m_toNetQueues[id].size() <= network_num) { - m_toNetQueues[id].push_back(nullptr); + NodeID local_id = getLocalNodeID(global_id); + checkNetworkAllocation(local_id, ordered, network_num, vnet_type); + + while (m_toNetQueues[local_id].size() <= network_num) { + m_toNetQueues[local_id].push_back(nullptr); } - m_toNetQueues[id][network_num] = b; + m_toNetQueues[local_id][network_num] = b; } void -Network::setFromNetQueue(NodeID id, bool ordered, int network_num, +Network::setFromNetQueue(NodeID global_id, bool ordered, int network_num, std::string vnet_type, MessageBuffer *b) { - checkNetworkAllocation(id, ordered, network_num, vnet_type); - while (m_fromNetQueues[id].size() <= network_num) { - m_fromNetQueues[id].push_back(nullptr); + NodeID local_id = getLocalNodeID(global_id); + checkNetworkAllocation(local_id, ordered, network_num, vnet_type); + + while (m_fromNetQueues[local_id].size() <= network_num) { + m_fromNetQueues[local_id].push_back(nullptr); } - m_fromNetQueues[id][network_num] = b; + m_fromNetQueues[local_id][network_num] = b; } NodeID @@ -212,3 +242,10 @@ Network::addressToNodeID(Addr addr, MachineType mtype) } return MachineType_base_count(mtype); } + +NodeID +Network::getLocalNodeID(NodeID global_id) const +{ + assert(globalToLocalMap.count(global_id)); + return globalToLocalMap.at(global_id); +} diff --git a/src/mem/ruby/network/Network.hh b/src/mem/ruby/network/Network.hh index 606e6704bf..bba0c5e7f1 100644 --- a/src/mem/ruby/network/Network.hh +++ b/src/mem/ruby/network/Network.hh @@ -90,13 +90,13 @@ class Network : public ClockedObject static uint32_t MessageSizeType_to_int(MessageSizeType size_type); // returns the queue requested for the given component - void setToNetQueue(NodeID id, bool ordered, int netNumber, + void setToNetQueue(NodeID global_id, bool ordered, int netNumber, std::string vnet_type, MessageBuffer *b); - virtual void setFromNetQueue(NodeID id, bool ordered, int netNumber, + virtual void setFromNetQueue(NodeID global_id, bool ordered, int netNumber, std::string vnet_type, MessageBuffer *b); - virtual void checkNetworkAllocation(NodeID id, bool ordered, - int network_num, std::string vnet_type); + virtual void checkNetworkAllocation(NodeID local_id, bool ordered, + int network_num, std::string vnet_type); virtual void makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, const NetDest& routing_table_entry) = 0; @@ -140,6 +140,8 @@ class Network : public ClockedObject return RubyDummyPort::instance(); } + NodeID getLocalNodeID(NodeID global_id) const; + protected: // Private copy constructor and assignment operator Network(const Network& obj); @@ -182,6 +184,10 @@ class Network : public ClockedObject AddrRangeList ranges; }; std::unordered_multimap addrMap; + + // Global NodeID to local node map. If there are not multiple networks in + // the same RubySystem, this is a one-to-one mapping of global to local. + std::unordered_map globalToLocalMap; }; inline std::ostream& diff --git a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc index 8f226b1de2..a88302b6c4 100644 --- a/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc +++ b/src/mem/ruby/network/garnet2.0/GarnetNetwork.cc @@ -146,10 +146,11 @@ GarnetNetwork::init() */ void -GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, +GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link, const NetDest& routing_table_entry) { - assert(src < m_nodes); + NodeID local_src = getLocalNodeID(global_src); + assert(local_src < m_nodes); GarnetExtLink* garnet_link = safe_cast(link); @@ -163,7 +164,7 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, PortDirection dst_inport_dirn = "Local"; m_routers[dest]->addInPort(dst_inport_dirn, net_link, credit_link); - m_nis[src]->addOutPort(net_link, credit_link, dest); + m_nis[local_src]->addOutPort(net_link, credit_link, dest); } /* @@ -173,10 +174,12 @@ GarnetNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, */ void -GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, - const NetDest& routing_table_entry) +GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest, + BasicLink* link, + const NetDest& routing_table_entry) { - assert(dest < m_nodes); + NodeID local_dest = getLocalNodeID(global_dest); + assert(local_dest < m_nodes); assert(src < m_routers.size()); assert(m_routers[src] != NULL); @@ -194,7 +197,7 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, m_routers[src]->addOutPort(src_outport_dirn, net_link, routing_table_entry, link->m_weight, credit_link); - m_nis[dest]->addInPort(net_link, credit_link); + m_nis[local_dest]->addInPort(net_link, credit_link); } /* @@ -233,9 +236,11 @@ GarnetNetwork::getNumRouters() // Get ID of router connected to a NI. int -GarnetNetwork::get_router_id(int ni) +GarnetNetwork::get_router_id(int global_ni) { - return m_nis[ni]->get_router_id(); + NodeID local_ni = getLocalNodeID(global_ni); + + return m_nis[local_ni]->get_router_id(); } void diff --git a/src/mem/ruby/network/simple/SimpleNetwork.cc b/src/mem/ruby/network/simple/SimpleNetwork.cc index 84817e444b..d3b5515123 100644 --- a/src/mem/ruby/network/simple/SimpleNetwork.cc +++ b/src/mem/ruby/network/simple/SimpleNetwork.cc @@ -83,27 +83,30 @@ SimpleNetwork::init() // From a switch to an endpoint node void -SimpleNetwork::makeExtOutLink(SwitchID src, NodeID dest, BasicLink* link, - const NetDest& routing_table_entry) +SimpleNetwork::makeExtOutLink(SwitchID src, NodeID global_dest, + BasicLink* link, + const NetDest& routing_table_entry) { - assert(dest < m_nodes); + NodeID local_dest = getLocalNodeID(global_dest); + assert(local_dest < m_nodes); assert(src < m_switches.size()); assert(m_switches[src] != NULL); SimpleExtLink *simple_link = safe_cast(link); - m_switches[src]->addOutPort(m_fromNetQueues[dest], routing_table_entry, - simple_link->m_latency, + m_switches[src]->addOutPort(m_fromNetQueues[local_dest], + routing_table_entry, simple_link->m_latency, simple_link->m_bw_multiplier); } // From an endpoint node to a switch void -SimpleNetwork::makeExtInLink(NodeID src, SwitchID dest, BasicLink* link, +SimpleNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link, const NetDest& routing_table_entry) { - assert(src < m_nodes); - m_switches[dest]->addInPort(m_toNetQueues[src]); + NodeID local_src = getLocalNodeID(global_src); + assert(local_src < m_nodes); + m_switches[dest]->addInPort(m_toNetQueues[local_src]); } // From a switch to a switch diff --git a/src/mem/ruby/system/RubySystem.cc b/src/mem/ruby/system/RubySystem.cc index 2aeff8caeb..c9da20656b 100644 --- a/src/mem/ruby/system/RubySystem.cc +++ b/src/mem/ruby/system/RubySystem.cc @@ -94,7 +94,7 @@ RubySystem::RubySystem(const Params *p) void RubySystem::registerNetwork(Network* network_ptr) { - m_network = network_ptr; + m_networks.emplace_back(network_ptr); } void @@ -108,7 +108,6 @@ RubySystem::registerAbstractController(AbstractController* cntrl) RubySystem::~RubySystem() { - delete m_network; delete m_profiler; } @@ -407,7 +406,9 @@ void RubySystem::resetStats() { m_start_cycle = curCycle(); - m_network->resetStats(); + for (auto& network : m_networks) { + network->resetStats(); + } } bool @@ -512,8 +513,10 @@ RubySystem::functionalRead(PacketPtr pkt) DPRINTF(RubySystem, "Network functionalRead lookup " "(num_maybe_stale=%d, num_busy = %d)\n", num_maybe_stale, num_busy); - if (m_network->functionalRead(pkt)) - return true; + for (auto& network : m_networks) { + if (network->functionalRead(pkt)) + return true; + } } return false; @@ -558,7 +561,9 @@ RubySystem::functionalWrite(PacketPtr pkt) } } - num_functional_writes += m_network->functionalWrite(pkt); + for (auto& network : m_networks) { + num_functional_writes += network->functionalWrite(pkt); + } DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes); return true; diff --git a/src/mem/ruby/system/RubySystem.hh b/src/mem/ruby/system/RubySystem.hh index 5d10991aae..2407072850 100644 --- a/src/mem/ruby/system/RubySystem.hh +++ b/src/mem/ruby/system/RubySystem.hh @@ -130,7 +130,8 @@ class RubySystem : public ClockedObject SimpleMemory *m_phys_mem; const bool m_access_backing_store; - Network* m_network; + //std::vector m_networks; + std::vector> m_networks; std::vector m_abs_cntrl_vec; Cycles m_start_cycle;