derek/gem5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mem-garnet: Flexible VCs per Vnet for each router

Browse Source 
This change allows configuring each router with a certain number
of VCs for each VNET. This is beneficial when dealing with
heterogenous link widths in a system. Configuring VCs
for each router allows one to ensure equal throughput
within the network while avoiding head-of-line blocking.
Changing a router's VCs number can be done in topology files
using the vcs_per_vnet value argument of router.

Change-Id: Icf4f510248128429a1a11f19f9802ee96f340611
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/32599
Reviewed-by: Jason Lowe-Power <power.jg@gmail.com>
Maintainer: Jason Lowe-Power <power.jg@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
This commit is contained in:
Srikant Bharadwaj
2019-03-27 21:23:02 -04:00
parent 6be16d84da
commit 615067c163
14 changed files with 124 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/mem/ruby/network/garnet2.0/GarnetNetwork.cc
View File

@@ -58,7 +58,7 @@ GarnetNetwork::GarnetNetwork(const Params *p)
{
m_num_rows = p->num_rows;
m_ni_flit_size = p->ni_flit_size;
m_vcs_per_vnet = p->vcs_per_vnet;
m_max_vcs_per_vnet = 0;
m_buffers_per_data_vc = p->buffers_per_data_vc;
m_buffers_per_ctrl_vc = p->buffers_per_ctrl_vc;
m_routing_algorithm = p->routing_algorithm;
@@ -166,6 +166,9 @@ GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
PortDirection dst_inport_dirn = "Local";
m_max_vcs_per_vnet = std::max(m_max_vcs_per_vnet,
m_routers[dest]->get_vc_per_vnet());
/*
* We check if a bridge was enabled at any end of the link.
* The bridge is enabled if either of clock domain
@@ -185,9 +188,10 @@ GarnetNetwork::makeExtInLink(NodeID global_src, SwitchID dest, BasicLink* link,
m_nis[local_src]->
addOutPort(garnet_link->extNetBridge[LinkDirection_In],
garnet_link->extCredBridge[LinkDirection_In],
dest);
dest, m_routers[dest]->get_vc_per_vnet());
} else {
m_nis[local_src]->addOutPort(net_link, credit_link, dest);
m_nis[local_src]->addOutPort(net_link, credit_link, dest,
m_routers[dest]->get_vc_per_vnet());
}
if (garnet_link->intBridgeEn) {
@@ -231,6 +235,9 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
PortDirection src_outport_dirn = "Local";
m_max_vcs_per_vnet = std::max(m_max_vcs_per_vnet,
m_routers[src]->get_vc_per_vnet());
/*
* We check if a bridge was enabled at any end of the link.
* The bridge is enabled if either of clock domain
@@ -261,12 +268,14 @@ GarnetNetwork::makeExtOutLink(SwitchID src, NodeID global_dest,
addOutPort(src_outport_dirn,
garnet_link->intNetBridge[LinkDirection_Out],
routing_table_entry, link->m_weight,
garnet_link->intCredBridge[LinkDirection_Out]);
garnet_link->intCredBridge[LinkDirection_Out],
m_routers[src]->get_vc_per_vnet());
} else {
m_routers[src]->
addOutPort(src_outport_dirn, net_link,
routing_table_entry,
link->m_weight, credit_link);
link->m_weight, credit_link,
m_routers[src]->get_vc_per_vnet());
}
}
@@ -291,6 +300,10 @@ GarnetNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
m_networklinks.push_back(net_link);
m_creditlinks.push_back(credit_link);
m_max_vcs_per_vnet = std::max(m_max_vcs_per_vnet,
std::max(m_routers[dest]->get_vc_per_vnet(),
m_routers[src]->get_vc_per_vnet()));
/*
* We check if a bridge was enabled at any end of the link.
* The bridge is enabled if either of clock domain
@@ -319,11 +332,13 @@ GarnetNetwork::makeInternalLink(SwitchID src, SwitchID dest, BasicLink* link,
m_routers[src]->
addOutPort(src_outport_dirn, garnet_link->srcNetBridge,
routing_table_entry,
link->m_weight, garnet_link->srcCredBridge);
link->m_weight, garnet_link->srcCredBridge,
m_routers[dest]->get_vc_per_vnet());
} else {
m_routers[src]->addOutPort(src_outport_dirn, net_link,
routing_table_entry,
link->m_weight, credit_link);
link->m_weight, credit_link,
m_routers[dest]->get_vc_per_vnet());
}
}
@@ -479,9 +494,8 @@ GarnetNetwork::regStats()
.name(name() + ".int_link_utilization");
m_average_link_utilization
.name(name() + ".avg_link_utilization");
m_average_vc_load
.init(m_virtual_networks * m_vcs_per_vnet)
.init(m_virtual_networks * m_max_vcs_per_vnet)
.name(name() + ".avg_vc_load")
.flags(Stats::pdf | Stats::total | Stats::nozero | Stats::oneline)
;

6
src/mem/ruby/network/garnet2.0/GarnetNetwork.hh
View File

@@ -64,7 +64,6 @@ class GarnetNetwork : public Network
// for network
uint32_t getNiFlitSize() const { return m_ni_flit_size; }
uint32_t getVCsPerVnet() const { return m_vcs_per_vnet; }
uint32_t getBuffersPerDataVC() { return m_buffers_per_data_vc; }
uint32_t getBuffersPerCtrlVC() { return m_buffers_per_ctrl_vc; }
int getRoutingAlgorithm() const { return m_routing_algorithm; }
@@ -76,9 +75,8 @@ class GarnetNetwork : public Network
// Internal configuration
bool isVNetOrdered(int vnet) const { return m_ordered[vnet]; }
VNET_type
get_vnet_type(int vc)
get_vnet_type(int vnet)
{
int vnet = vc/getVCsPerVnet();
return m_vnet_type[vnet];
}
int getNumRouters();
@@ -147,7 +145,7 @@ class GarnetNetwork : public Network
int m_num_rows;
int m_num_cols;
uint32_t m_ni_flit_size;
uint32_t m_vcs_per_vnet;
uint32_t m_max_vcs_per_vnet;
uint32_t m_buffers_per_ctrl_vc;
uint32_t m_buffers_per_data_vc;
int m_routing_algorithm;

16
src/mem/ruby/network/garnet2.0/NetworkBridge.cc
View File

@@ -63,11 +63,19 @@ NetworkBridge::NetworkBridge(const Params *p)
// CDC type must be set
panic("CDC type must be set");
}
}
lenBuffer.resize(p->vcs_per_vnet * p->virt_nets);
sizeSent.resize(p->vcs_per_vnet * p->virt_nets);
flitsSent.resize(p->vcs_per_vnet * p->virt_nets);
extraCredit.resize(p->vcs_per_vnet * p->virt_nets);
void
NetworkBridge::setVcsPerVnet(uint32_t consumerVcs)
{
DPRINTF(RubyNetwork, "VcsPerVnet VC: %d\n", consumerVcs);
NetworkLink::setVcsPerVnet(consumerVcs);
lenBuffer.resize(consumerVcs * m_virt_nets);
sizeSent.resize(consumerVcs * m_virt_nets);
flitsSent.resize(consumerVcs * m_virt_nets);
extraCredit.resize(consumerVcs * m_virt_nets);
nLink->setVcsPerVnet(consumerVcs);
}
void

1
src/mem/ruby/network/garnet2.0/NetworkBridge.hh
View File

@@ -64,6 +64,7 @@ class NetworkBridge: public CreditLink
void scheduleFlit(flit *t_flit, Cycles latency);
void flitisizeAndSend(flit *t_flit);
void setVcsPerVnet(uint32_t consumerVcs);
protected:
// Pointer to co-existing bridge

57
src/mem/ruby/network/garnet2.0/NetworkInterface.cc
View File

@@ -51,26 +51,8 @@ NetworkInterface::NetworkInterface(const Params *p)
m_deadlock_threshold(p->garnet_deadlock_threshold),
vc_busy_counter(m_virtual_networks, 0)
{
const int num_vcs = m_vc_per_vnet * m_virtual_networks;
niOutVcs.resize(num_vcs);
m_ni_out_vcs_enqueue_time.resize(num_vcs);
// instantiating the NI flit buffers
for (auto& time : m_ni_out_vcs_enqueue_time) {
time = Tick(INFINITE_);
}
m_stall_count.resize(m_virtual_networks);
}
void
NetworkInterface::init()
{
const int num_vcs = m_vc_per_vnet * m_virtual_networks;
outVcState.reserve(num_vcs);
for (int i = 0; i < num_vcs; i++) {
outVcState.emplace_back(i, m_net_ptr);
}
niOutVcs.resize(0);
}
void
@@ -84,22 +66,57 @@ NetworkInterface::addInPort(NetworkLink *in_link,
in_link->setLinkConsumer(this);
credit_link->setSourceQueue(newInPort->outCreditQueue(), this);
if (m_vc_per_vnet != 0) {
in_link->setVcsPerVnet(m_vc_per_vnet);
credit_link->setVcsPerVnet(m_vc_per_vnet);
}
}
void
NetworkInterface::addOutPort(NetworkLink *out_link,
CreditLink *credit_link,
SwitchID router_id)
SwitchID router_id, uint32_t consumerVcs)
{
OutputPort *newOutPort = new OutputPort(out_link, credit_link, router_id);
outPorts.push_back(newOutPort);
assert(consumerVcs > 0);
// We are not allowing different physical links to have different vcs
// If it is required that the Network Interface support different VCs
// for every physical link connected to it. Then they need to change
// the logic within outport and inport.
if (niOutVcs.size() == 0) {
m_vc_per_vnet = consumerVcs;
int m_num_vcs = consumerVcs * m_virtual_networks;
niOutVcs.resize(m_num_vcs);
outVcState.reserve(m_num_vcs);
m_ni_out_vcs_enqueue_time.resize(m_num_vcs);
// instantiating the NI flit buffers
for (int i = 0; i < m_num_vcs; i++) {
m_ni_out_vcs_enqueue_time[i] = Tick(INFINITE_);
outVcState.emplace_back(i, m_net_ptr, consumerVcs);
}
// Reset VC Per VNET for input links already instantiated
for (auto &iPort: inPorts) {
NetworkLink *inNetLink = iPort->inNetLink();
inNetLink->setVcsPerVnet(m_vc_per_vnet);
credit_link->setVcsPerVnet(m_vc_per_vnet);
}
} else {
fatal_if(consumerVcs != m_vc_per_vnet,
"%s: Connected Physical links have different vc requests: %d and %d\n",
name(), consumerVcs, m_vc_per_vnet);
}
DPRINTF(RubyNetwork, "OutputPort:%s Vnet: %s\n",
out_link->name(), newOutPort->printVnets());
out_link->setSourceQueue(newOutPort->outFlitQueue(), this);
out_link->setVcsPerVnet(m_vc_per_vnet);
credit_link->setLinkConsumer(this);
credit_link->setVcsPerVnet(m_vc_per_vnet);
}
void

8
src/mem/ruby/network/garnet2.0/NetworkInterface.hh
View File

@@ -56,11 +56,9 @@ class NetworkInterface : public ClockedObject, public Consumer
NetworkInterface(const Params *p);
~NetworkInterface() = default;
void init();
void addInPort(NetworkLink *in_link, CreditLink *credit_link);
void addOutPort(NetworkLink *out_link, CreditLink *credit_link,
SwitchID router_id);
SwitchID router_id, uint32_t consumerVcs);
void dequeueCallback();
void wakeup();
@@ -262,8 +260,8 @@ class NetworkInterface : public ClockedObject, public Consumer
private:
GarnetNetwork *m_net_ptr;
const NodeID m_id;
const int m_virtual_networks, m_vc_per_vnet;
int m_router_id; // id of my router
const int m_virtual_networks;
int m_vc_per_vnet;
std::vector<int> m_vc_allocator;
std::vector<OutputPort *> outPorts;
std::vector<InputPort *> inPorts;

12
src/mem/ruby/network/garnet2.0/NetworkLink.cc
View File

@@ -40,9 +40,8 @@ NetworkLink::NetworkLink(const Params *p)
: ClockedObject(p), Consumer(this), m_id(p->link_id),
m_type(NUM_LINK_TYPES_),
m_latency(p->link_latency), m_link_utilized(0),
m_vc_load(p->vcs_per_vnet * p->virt_nets),
linkBuffer(), link_consumer(nullptr),
link_srcQueue(nullptr)
m_virt_nets(p->virt_nets), linkBuffer(),
link_consumer(nullptr), link_srcQueue(nullptr)
{
int num_vnets = (p->supported_vnets).size();
assert(num_vnets > 0);
@@ -51,7 +50,6 @@ NetworkLink::NetworkLink(const Params *p)
for (int i = 0; i < num_vnets; i++) {
mVnets[i] = p->supported_vnets[i];
}
DPRINTF(RubyNetwork,"Created with bitwidth:%d\n", bitWidth);
}
void
@@ -60,6 +58,12 @@ NetworkLink::setLinkConsumer(Consumer *consumer)
link_consumer = consumer;
}
void
NetworkLink::setVcsPerVnet(uint32_t consumerVcs)
{
m_vc_load.resize(m_virt_nets * consumerVcs);
}
void
NetworkLink::setSourceQueue(flitBuffer *src_queue, ClockedObject *srcClockObj)
{

2
src/mem/ruby/network/garnet2.0/NetworkLink.hh
View File

@@ -53,6 +53,7 @@ class NetworkLink : public ClockedObject, public Consumer
void setLinkConsumer(Consumer *consumer);
void setSourceQueue(flitBuffer *src_queue, ClockedObject *srcClockObject);
virtual void setVcsPerVnet(uint32_t consumerVcs);
void setType(link_type type) { m_type = type; }
link_type getType() { return m_type; }
void print(std::ostream& out) const {}
@@ -89,6 +90,7 @@ class NetworkLink : public ClockedObject, public Consumer
std::vector<unsigned int> m_vc_load;
protected:
uint32_t m_virt_nets;
flitBuffer linkBuffer;
Consumer *link_consumer;
flitBuffer *link_srcQueue;

11
src/mem/ruby/network/garnet2.0/OutVcState.cc
View File

@@ -32,13 +32,20 @@
#include "mem/ruby/system/RubySystem.hh"
OutVcState::OutVcState(int id, GarnetNetwork *network_ptr)
OutVcState::OutVcState(int id, GarnetNetwork *network_ptr,
uint32_t consumerVcs)
: m_time(0)
{
m_id = id;
m_vc_state = IDLE_;
/*
* We find the virtual network using the number of
* vcs per vnet. This assumes that the same vcs per
* vnet is used throughout the given object.
*/
int vnet = floor(id/consumerVcs);
if (network_ptr->get_vnet_type(id) == DATA_VNET_)
if (network_ptr->get_vnet_type(vnet) == DATA_VNET_)
m_max_credit_count = network_ptr->getBuffersPerDataVC();
else
m_max_credit_count = network_ptr->getBuffersPerCtrlVC();

2
src/mem/ruby/network/garnet2.0/OutVcState.hh
View File

@@ -37,7 +37,7 @@
class OutVcState
{
public:
OutVcState(int id, GarnetNetwork *network_ptr);
OutVcState(int id, GarnetNetwork *network_ptr, uint32_t consumerVcs);
int get_credit_count() { return m_credit_count; }
inline bool has_credit() { return (m_credit_count > 0); }

9
src/mem/ruby/network/garnet2.0/OutputUnit.cc
View File

@@ -37,14 +37,15 @@
#include "mem/ruby/network/garnet2.0/Router.hh"
#include "mem/ruby/network/garnet2.0/flitBuffer.hh"
OutputUnit::OutputUnit(int id, PortDirection direction, Router *router)
OutputUnit::OutputUnit(int id, PortDirection direction, Router *router,
uint32_t consumerVcs)
: Consumer(router), m_router(router), m_id(id), m_direction(direction),
m_vc_per_vnet(m_router->get_vc_per_vnet())
m_vc_per_vnet(consumerVcs)
{
const int m_num_vcs = m_router->get_num_vcs();
const int m_num_vcs = consumerVcs * m_router->get_num_vnets();
outVcState.reserve(m_num_vcs);
for (int i = 0; i < m_num_vcs; i++) {
outVcState.emplace_back(i, m_router->get_net_ptr());
outVcState.emplace_back(i, m_router->get_net_ptr(), consumerVcs);
}
}

9
src/mem/ruby/network/garnet2.0/OutputUnit.hh
View File

@@ -47,7 +47,8 @@ class Router;
class OutputUnit : public Consumer
{
public:
OutputUnit(int id, PortDirection direction, Router *router);
OutputUnit(int id, PortDirection direction, Router *router,
uint32_t consumerVcs);
~OutputUnit() = default;
void set_out_link(NetworkLink *link);
void set_credit_link(CreditLink *credit_link);
@@ -88,6 +89,12 @@ class OutputUnit : public Consumer
void insert_flit(flit *t_flit);
inline int
getVcsPerVnet()
{
return m_vc_per_vnet;
}
uint32_t functionalWrite(Packet *pkt);
private:

9
src/mem/ruby/network/garnet2.0/Router.cc
View File

@@ -103,7 +103,9 @@ Router::addInPort(PortDirection inport_dirn,
input_unit->set_in_link(in_link);
input_unit->set_credit_link(credit_link);
in_link->setLinkConsumer(this);
in_link->setVcsPerVnet(get_vc_per_vnet());
credit_link->setSourceQueue(input_unit->getCreditQueue(), this);
credit_link->setVcsPerVnet(get_vc_per_vnet());
m_input_unit.push_back(std::shared_ptr<InputUnit>(input_unit));
@@ -114,18 +116,21 @@ void
Router::addOutPort(PortDirection outport_dirn,
NetworkLink *out_link,
std::vector<NetDest>& routing_table_entry, int link_weight,
CreditLink *credit_link)
CreditLink *credit_link, uint32_t consumerVcs)
{
fatal_if(out_link->bitWidth != m_bit_width, "Widths of units do not match."
" Consider inserting SerDes Units");
int port_num = m_output_unit.size();
OutputUnit *output_unit = new OutputUnit(port_num, outport_dirn, this);
OutputUnit *output_unit = new OutputUnit(port_num, outport_dirn, this,
consumerVcs);
output_unit->set_out_link(out_link);
output_unit->set_credit_link(credit_link);
credit_link->setLinkConsumer(this);
credit_link->setVcsPerVnet(consumerVcs);
out_link->setSourceQueue(output_unit->getOutQueue(), this);
out_link->setVcsPerVnet(consumerVcs);
m_output_unit.push_back(std::shared_ptr<OutputUnit>(output_unit));

11
src/mem/ruby/network/garnet2.0/Router.hh
View File

@@ -69,12 +69,13 @@ class Router : public BasicRouter, public Consumer
CreditLink *credit_link);
void addOutPort(PortDirection outport_dirn, NetworkLink *link,
std::vector<NetDest>& routing_table_entry,
int link_weight, CreditLink *credit_link);
int link_weight, CreditLink *credit_link,
uint32_t consumerVcs);
Cycles get_pipe_stages(){ return m_latency; }
int get_num_vcs() { return m_num_vcs; }
int get_num_vnets() { return m_virtual_networks; }
int get_vc_per_vnet() { return m_vc_per_vnet; }
uint32_t get_num_vcs() { return m_num_vcs; }
uint32_t get_num_vnets() { return m_virtual_networks; }
uint32_t get_vc_per_vnet() { return m_vc_per_vnet; }
int get_num_inports() { return m_input_unit.size(); }
int get_num_outports() { return m_output_unit.size(); }
int get_id() { return m_id; }
@@ -132,7 +133,7 @@ class Router : public BasicRouter, public Consumer
private:
Cycles m_latency;
int m_virtual_networks, m_vc_per_vnet, m_num_vcs;
uint32_t m_virtual_networks, m_vc_per_vnet, m_num_vcs;
uint32_t m_bit_width;
GarnetNetwork *m_network_ptr;