ruby: record fully busy cycle with in the controller

This patch does several things. First, the counter for fully busy cycles for a
controller is now kept with in the controller, instead of being part of the profiler.
Second, the topology class no longer keeps an array of controllers which was only
used for printing stats. Instead, ruby system will now ask each controller to print
the stats. Thirdly, the statistical variable for recording how many different types
were created is being moved in to the controller from the profiler. Note that for
printing, the profiler will collate results from different controllers.

src/mem/protocol/MOESI_CMP_directory.slicc

@@ -1,7 +1,7 @@
 protocol "MOESI_CMP_directory";
 include "RubySlicc_interfaces.slicc";
 include "MOESI_CMP_directory-msg.sm";
-include "MOESI_CMP_directory-L2cache.sm";
 include "MOESI_CMP_directory-L1cache.sm";
+include "MOESI_CMP_directory-L2cache.sm";
 include "MOESI_CMP_directory-dma.sm";
 include "MOESI_CMP_directory-dir.sm";

src/mem/ruby/common/Global.cc

@@ -28,4 +28,7 @@
 
 #include "mem/ruby/common/Global.hh"
 
+using namespace std;
+
 RubySystem* g_system_ptr = 0;
+vector<map<uint32_t, AbstractController *> > g_abs_controls;

src/mem/ruby/common/Global.hh

@@ -29,10 +29,16 @@
 #ifndef __MEM_RUBY_COMMON_GLOBAL_HH__
 #define __MEM_RUBY_COMMON_GLOBAL_HH__
 
+#include <map>
+#include <vector>
+
 #include "base/str.hh"
 
 class RubySystem;
 extern RubySystem* g_system_ptr;
 
+class AbstractController;
+extern std::vector<std::map<uint32_t, AbstractController *> > g_abs_controls;
+
 #endif // __MEM_RUBY_COMMON_GLOBAL_HH__
 

src/mem/ruby/network/Topology.cc

@@ -89,8 +89,7 @@ Topology::Topology(const Params *p)
         AbstractController *abs_cntrl = ext_link->params()->ext_node;
         BasicRouter *router = ext_link->params()->int_node;
 
-        // Store the controller and ExtLink pointers for later
-        m_controller_vector.push_back(abs_cntrl);
+        // Store the ExtLink pointers for later
         m_ext_link_vector.push_back(ext_link);
 
         int ext_idx1 = abs_cntrl->params()->cntrl_id;
@@ -265,22 +264,6 @@ Topology::makeLink(Network *net, SwitchID src, SwitchID dest,
     }
 }
 
-void
-Topology::printStats(std::ostream& out) const
-{
-    for (int cntrl = 0; cntrl < m_controller_vector.size(); cntrl++) {
-        m_controller_vector[cntrl]->printStats(out);
-    }
-}
-
-void
-Topology::clearStats()
-{
-    for (int cntrl = 0; cntrl < m_controller_vector.size(); cntrl++) {
-        m_controller_vector[cntrl]->clearStats();
-    }
-}
-
 // The following all-pairs shortest path algorithm is based on the
 // discussion from Cormen et al., Chapter 26.1.
 void

src/mem/ruby/network/Topology.hh

@@ -78,8 +78,6 @@ class Topology : public SimObject
     void initNetworkPtr(Network* net_ptr);
 
     const std::string getName() { return m_name; }
-    void printStats(std::ostream& out) const;
-    void clearStats();
     void print(std::ostream& out) const { out << "[Topology]"; }
 
   protected:
@@ -99,7 +97,6 @@ class Topology : public SimObject
     NodeID m_nodes;
     int m_number_of_switches;
 
-    std::vector<AbstractController*> m_controller_vector;
     std::vector<BasicExtLink*> m_ext_link_vector;
     std::vector<BasicIntLink*> m_int_link_vector;
 

src/mem/ruby/network/garnet/BaseGarnetNetwork.cc

@@ -146,7 +146,6 @@ BaseGarnetNetwork::printStats(ostream& out) const
     printPerformanceStats(out);
     printLinkStats(out);
     printPowerStats(out);
-    m_topology_ptr->printStats(out);
 }
 
 void

src/mem/ruby/network/simple/SimpleNetwork.cc

@@ -304,7 +304,6 @@ SimpleNetwork::printStats(ostream& out) const
     for (int i = 0; i < m_switch_ptr_vector.size(); i++) {
         m_switch_ptr_vector[i]->printStats(out);
     }
-    m_topology_ptr->printStats(out);
 }
 
 void
@@ -313,7 +312,6 @@ SimpleNetwork::clearStats()
     for (int i = 0; i < m_switch_ptr_vector.size(); i++) {
         m_switch_ptr_vector[i]->clearStats();
     }
-    m_topology_ptr->clearStats();
 }
 
 void

src/mem/ruby/profiler/Profiler.cc

@@ -170,6 +170,59 @@ Profiler::print(ostream& out) const
     out << "[Profiler]";
 }
 
+void
+Profiler::printRequestProfile(ostream &out)
+{
+    out << "Request vs. RubySystem State Profile" << endl;
+    out << "--------------------------------" << endl;
+    out << endl;
+
+    map<string, uint64_t> m_requestProfileMap;
+    uint64_t m_requests = 0;
+
+    for (uint32_t i = 0; i < MachineType_NUM; i++) {
+        for (map<uint32_t, AbstractController*>::iterator it =
+                  g_abs_controls[i].begin();
+             it != g_abs_controls[i].end(); ++it) {
+
+            AbstractController *ctr = (*it).second;
+            map<string, uint64_t> mp = ctr->getRequestProfileMap();
+
+            for (map<string, uint64_t>::iterator jt = mp.begin();
+                 jt != mp.end(); ++jt) {
+
+                map<string, uint64_t>::iterator kt =
+                    m_requestProfileMap.find((*jt).first);
+                if (kt != m_requestProfileMap.end()) {
+                    (*kt).second += (*jt).second;
+                } else {
+                    m_requestProfileMap[(*jt).first] = (*jt).second;
+                }
+            }
+
+            m_requests += ctr->getRequestCount();
+        }
+    }
+
+    map<string, uint64_t>::const_iterator i = m_requestProfileMap.begin();
+    map<string, uint64_t>::const_iterator end = m_requestProfileMap.end();
+    for (; i != end; ++i) {
+        const string &key = i->first;
+        uint64_t count = i->second;
+
+        double percent = (100.0 * double(count)) / double(m_requests);
+        vector<string> items;
+        tokenize(items, key, ':');
+        vector<string>::iterator j = items.begin();
+        vector<string>::iterator end = items.end();
+        for (; j != end; ++i)
+            out << setw(10) << *j;
+        out << setw(11) << count;
+        out << setw(14) << percent << endl;
+    }
+    out << endl;
+}
+
 void
 Profiler::printStats(ostream& out, bool short_stats)
 {
@@ -237,13 +290,17 @@ Profiler::printStats(ostream& out, bool short_stats)
 
     if (!short_stats) {
         out << "Busy Controller Counts:" << endl;
-        for (int i = 0; i < MachineType_NUM; i++) {
-            int size = MachineType_base_count((MachineType)i);
-            for (int j = 0; j < size; j++) {
+        for (uint32_t i = 0; i < MachineType_NUM; i++) {
+            uint32_t size = MachineType_base_count((MachineType)i);
+
+            for (uint32_t j = 0; j < size; j++) {
                 MachineID machID;
                 machID.type = (MachineType)i;
                 machID.num = j;
-                out << machID << ":" << m_busyControllerCount[i][j] << "  ";
+
+                AbstractController *ctr =
+                    (*(g_abs_controls[i].find(j))).second;
+                out << machID << ":" << ctr->getFullyBusyCycles() << "  ";
                 if ((j + 1) % 8 == 0) {
                     out << endl;
                 }
@@ -365,27 +422,7 @@ Profiler::printStats(ostream& out, bool short_stats)
     }
 
     if (!short_stats) {
-        out << "Request vs. RubySystem State Profile" << endl;
-        out << "--------------------------------" << endl;
-        out << endl;
-
-        map<string, int>::const_iterator i = m_requestProfileMap.begin();
-        map<string, int>::const_iterator end = m_requestProfileMap.end();
-        for (; i != end; ++i) {
-            const string &key = i->first;
-            int count = i->second;
-
-            double percent = (100.0 * double(count)) / double(m_requests);
-            vector<string> items;
-            tokenize(items, key, ':');
-            vector<string>::iterator j = items.begin();
-            vector<string>::iterator end = items.end();
-            for (; j != end; ++i)
-                out << setw(10) << *j;
-            out << setw(11) << count;
-            out << setw(14) << percent << endl;
-        }
-        out << endl;
+        printRequestProfile(out);
 
         out << "filter_action: " << m_filter_action_histogram << endl;
 
@@ -449,14 +486,6 @@ Profiler::clearStats()
         }
     }
 
-    m_busyControllerCount.resize(MachineType_NUM); // all machines
-    for (int i = 0; i < MachineType_NUM; i++) {
-        int size = MachineType_base_count((MachineType)i);
-        m_busyControllerCount[i].resize(size);
-        for (int j = 0; j < size; j++) {
-            m_busyControllerCount[i][j] = 0;
-        }
-    }
     m_busyBankCount = 0;
 
     m_delayedCyclesHistogram.clear();
@@ -511,12 +540,6 @@ Profiler::clearStats()
     m_cache_to_cache = 0;
     m_memory_to_cache = 0;
 
-    // clear HashMaps
-    m_requestProfileMap.clear();
-
-    // count requests profiled
-    m_requests = 0;
-
     m_outstanding_requests.clear();
     m_outstanding_persistent_requests.clear();
 
@@ -581,23 +604,6 @@ Profiler::profileMsgDelay(uint32_t virtualNetwork, Time delayCycles)
     }
 }
 
-// profiles original cache requests including PUTs
-void
-Profiler::profileRequest(const string& requestStr)
-{
-    m_requests++;
-
-    // if it doesn't exist, conveniently, it will be created with the
-    // default value which is 0
-    m_requestProfileMap[requestStr]++;
-}
-
-void
-Profiler::controllerBusy(MachineID machID)
-{
-    m_busyControllerCount[(int)machID.type][(int)machID.num]++;
-}
-
 void
 Profiler::profilePFWait(Time waitTime)
 {

src/mem/ruby/profiler/Profiler.hh

@@ -170,6 +170,9 @@ class Profiler : public SimObject
     bool getHotLines() { return m_hot_lines; }
     bool getAllInstructions() { return m_all_instructions; }
 
+  private:
+    void printRequestProfile(std::ostream &out);
+
   private:
     // Private copy constructor and assignment operator
     Profiler(const Profiler& obj);
@@ -187,7 +190,6 @@ class Profiler : public SimObject
     Time m_ruby_start;
     time_t m_real_time_start_time;
 
-    std::vector<std::vector<int64_t> > m_busyControllerCount;
     int64_t m_busyBankCount;
     Histogram m_multicast_retry_histogram;
 
@@ -234,9 +236,6 @@ class Profiler : public SimObject
     Histogram m_average_latency_estimate;
 
     m5::hash_set<Address> m_watch_address_set;
-    // counts all initiated cache request including PUTs
-    int m_requests;
-    std::map<std::string, int> m_requestProfileMap;
 
     //added by SS
     bool m_hot_lines;

src/mem/ruby/slicc_interface/AbstractController.cc

@@ -30,7 +30,8 @@
 #include "mem/ruby/system/System.hh"
 
 AbstractController::AbstractController(const Params *p)
-    : ClockedObject(p), Consumer(this)
+    : ClockedObject(p), Consumer(this), m_fully_busy_cycles(0),
+    m_request_count(0)
 {
     m_version = p->version;
     m_transitions_per_cycle = p->transitions_per_cycle;
@@ -38,5 +39,27 @@ AbstractController::AbstractController(const Params *p)
     m_recycle_latency = p->recycle_latency;
     m_number_of_TBEs = p->number_of_TBEs;
     m_is_blocking = false;
-    p->ruby_system->registerAbstractController(this);
+}
+
+void
+AbstractController::init()
+{
+    params()->ruby_system->registerAbstractController(this);
+}
+
+void
+AbstractController::clearStats()
+{
+    m_requestProfileMap.clear();
+    m_request_count = 0;
+}
+
+void
+AbstractController::profileRequest(const std::string &request)
+{
+    m_request_count++;
+
+    // if it doesn't exist, conveniently, it will be created with the
+    // default value which is 0
+    m_requestProfileMap[request]++;
 }

src/mem/ruby/slicc_interface/AbstractController.hh

@@ -51,6 +51,7 @@ class AbstractController : public ClockedObject, public Consumer
   public:
     typedef RubyControllerParams Params;
     AbstractController(const Params *p);
+    void init();
     const Params *params() const { return (const Params *)_params; }
     virtual MessageBuffer* getMandatoryQueue() const = 0;
     virtual const int & getVersion() const = 0;
@@ -84,12 +85,22 @@ class AbstractController : public ClockedObject, public Consumer
     virtual void enqueuePrefetch(const Address&, const RubyRequestType&)
     { fatal("Prefetches not implemented!");}
 
+  public:
+    MachineID getMachineID() const { return m_machineID; }
+    uint64_t getFullyBusyCycles() const { return m_fully_busy_cycles; }
+    uint64_t getRequestCount() const { return m_request_count; }
+    const std::map<std::string, uint64_t>& getRequestProfileMap() const
+    { return m_requestProfileMap; }
+
+  protected:
+    //! Profiles original cache requests including PUTs
+    void profileRequest(const std::string &request);
+
   protected:
     int m_transitions_per_cycle;
     int m_buffer_size;
     int m_recycle_latency;
     std::string m_name;
-    std::map<std::string, std::string> m_cfg;
     NodeID m_version;
     Network* m_net_ptr;
     MachineID m_machineID;
@@ -101,6 +112,15 @@ class AbstractController : public ClockedObject, public Consumer
     int m_max_in_port_rank;
     int m_cur_in_port_rank;
     int m_number_of_TBEs;
+
+    //! Counter for the number of cycles when the transitions carried out
+    //! were equal to the maximum allowed
+    uint64_t m_fully_busy_cycles;
+
+    //! Map for couting requests of different types. The controller should
+    //! call requisite function for updating the count.
+    std::map<std::string, uint64_t> m_requestProfileMap;
+    uint64_t m_request_count;
 };
 
 #endif // __MEM_RUBY_SLICC_INTERFACE_ABSTRACTCONTROLLER_HH__

src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.cc

@@ -35,16 +35,6 @@
 
 using namespace std;
 
-void
-profile_request(const string& L1CacheState, const string& L2CacheState,
-                const string& directoryState, const string& requestType)
-{
-    string requestStr = L1CacheState + ":" + L2CacheState + ":" +
-        directoryState + ":" + requestType;
-
-    g_system_ptr->getProfiler()->profileRequest(requestStr);
-}
-
 void
 profile_outstanding_request(int outstanding)
 {

src/mem/ruby/slicc_interface/RubySlicc_Profiler_interface.hh

@@ -46,10 +46,6 @@ void profile_outstanding_persistent_request(int outstanding);
 void profile_outstanding_request(int outstanding);
 void profile_sharing(const Address& addr, AccessType type, NodeID requestor,
                      const Set& sharers, const Set& owner);
-void profile_request(const std::string& L1CacheStateStr,
-                     const std::string& L2CacheStateStr,
-                     const std::string& directoryStateStr,
-                     const std::string& requestTypeStr);
 void profile_miss(const RubyRequest& msg, NodeID id);
 void profile_token_retry(const Address& addr, AccessType type, int count);
 void profile_filter_action(int action);

src/mem/ruby/system/MachineID.hh

@@ -38,7 +38,11 @@
 struct MachineID
 {
     MachineType type;
-    int num;  // range: 0 ... number of this machine's components in system - 1
+    //! range: 0 ... number of this machine's components in system - 1
+    uint32_t num;
+
+    MachineType getType() const { return type; }
+    uint32_t getNum() const { return num; }
 };
 
 inline std::string

src/mem/ruby/system/System.cc

@@ -72,7 +72,6 @@ RubySystem::RubySystem(const Params *p)
         m_memory_size_bits = ceilLog2(m_memory_size_bytes);
     }
 
-    g_system_ptr = this;
     if (p->no_mem_vec) {
         m_mem_vec_ptr = NULL;
     } else {
@@ -86,6 +85,12 @@ RubySystem::RubySystem(const Params *p)
 
     m_warmup_enabled = false;
     m_cooldown_enabled = false;
+
+    // Setup the global variables used in Ruby
+    g_system_ptr = this;
+
+    // Resize to the size of different machine types
+    g_abs_controls.resize(MachineType_NUM);
 }
 
 void
@@ -111,6 +116,9 @@ void
 RubySystem::registerAbstractController(AbstractController* cntrl)
 {
   m_abs_cntrl_vec.push_back(cntrl);
+
+  MachineID id = cntrl->getMachineID();
+  g_abs_controls[id.getType()][id.getNum()] = cntrl;
 }
 
 void
@@ -144,6 +152,15 @@ RubySystem::printStats(ostream& out)
 
     m_profiler_ptr->printStats(out);
     m_network_ptr->printStats(out);
+
+    for (uint32_t i = 0;i < g_abs_controls.size(); ++i) {
+        for (map<uint32_t, AbstractController *>::iterator it =
+                g_abs_controls[i].begin();
+             it != g_abs_controls[i].end(); ++it) {
+
+            ((*it).second)->printStats(out);
+        }
+    }
 }
 
 void
@@ -397,6 +414,9 @@ RubySystem::resetStats()
 {
     m_profiler_ptr->clearStats();
     m_network_ptr->clearStats();
+    for (uint32_t cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) {
+        m_abs_cntrl_vec[cntrl]->clearStats();
+    }
 }
 
 bool

src/mem/slicc/symbols/StateMachine.py

@@ -663,7 +663,11 @@ $vid->setDescription("[Version " + to_string(m_version) + ", ${ident}, name=${{v
                 code('m_profiler.possibleTransition($state, $event);')
 
         code.dedent()
-        code('}')
+        code('''
+    AbstractController::init();
+    clearStats();
+}
+''')
 
         has_mandatory_q = False
         for port in self.in_ports:
@@ -839,6 +843,7 @@ void $c_ident::clearStats() {
 
         code('''
     m_profiler.clearStats();
+    AbstractController::clearStats();
 }
 ''')
 
@@ -1047,7 +1052,7 @@ ${ident}_Controller::wakeup()
         assert(counter <= m_transitions_per_cycle);
         if (counter == m_transitions_per_cycle) {
             // Count how often we are fully utilized
-            g_system_ptr->getProfiler()->controllerBusy(m_machineID);
+            m_fully_busy_cycles++;
 
             // Wakeup in another cycle and try again
             scheduleEvent(1);