sim-ruby: checkpointing fixes and dependent eventq improvements

Fixes checkpointing with respect to lost events after swapping event queues.
Also adds DPRINTFs to better understand what's going on when Ruby serializes
and unserializes.

src/mem/ruby/system/CacheMemory.cc

@@ -28,6 +28,7 @@
 
 #include "base/intmath.hh"
 #include "debug/RubyCache.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/system/CacheMemory.hh"
 #include "mem/ruby/system/System.hh"
@@ -398,7 +399,7 @@ CacheMemory::recordCacheContents(int cntrl, CacheRecorder* tr) const
         }
     }
 
-    DPRINTF(RubyCache, "%s: %lli blocks of %lli total blocks"
+    DPRINTF(RubyCacheTrace, "%s: %lli blocks of %lli total blocks"
             "recorded %.2f%% \n", name().c_str(), warmedUpBlocks,
             (uint64)m_cache_num_sets * (uint64)m_cache_assoc,
             (float(warmedUpBlocks)/float(totalBlocks))*100.0);

src/mem/ruby/system/MemoryControl.cc

@@ -144,7 +144,7 @@ operator<<(ostream& out, const MemoryControl& obj)
 
 // CONSTRUCTOR
 MemoryControl::MemoryControl(const Params *p)
-    : SimObject(p)
+    : SimObject(p), m_event(this)
 {
     m_mem_bus_cycle_multiplier = p->mem_bus_cycle_multiplier;
     m_banks_per_rank = p->banks_per_rank;
@@ -204,7 +204,6 @@ MemoryControl::init()
     m_refresh_count = 1;
     m_need_refresh = 0;
     m_refresh_bank = 0;
-    m_awakened = 0;
     m_idleCount = 0;
     m_ageCounter = 0;
 
@@ -253,16 +252,17 @@ MemoryControl::enqueueMemRef(MemoryNode& memRef)
     physical_address_t addr = memRef.m_addr;
     int bank = getBank(addr);
 
-    DPRINTF(RubyMemory, "New memory request%7d: %#08x %c arrived at %10d bank = %3x\n",
+    DPRINTF(RubyMemory,
+            "New memory request%7d: %#08x %c arrived at %10d bank = %3x sched %c\n",
             m_msg_counter, addr, memRef.m_is_mem_read ? 'R':'W',
             memRef.m_time * g_eventQueue_ptr->getClock(),
-            bank);
+            bank, m_event.scheduled() ? 'Y':'N');
 
     m_profiler_ptr->profileMemReq(bank);
     m_input_queue.push_back(memRef);
-    if (!m_awakened) {
-        g_eventQueue_ptr->scheduleEvent(this, 1);
-        m_awakened = 1;
+
+    if (!m_event.scheduled()) {
+        schedule(m_event, curTick() + 1);
     }
 }
 
@@ -289,8 +289,9 @@ MemoryControl::peekNode()
 {
     assert(isReady());
     MemoryNode req = m_response_queue.front();
-    DPRINTF(RubyMemory, "Peek: memory request%7d: %#08x %c\n",
-            req.m_msg_counter, req.m_addr, req.m_is_mem_read ? 'R':'W');
+    DPRINTF(RubyMemory, "Peek: memory request%7d: %#08x %c sched %c\n",
+            req.m_msg_counter, req.m_addr, req.m_is_mem_read ? 'R':'W',
+            m_event.scheduled() ? 'Y':'N');
 
     return req;
 }
@@ -513,9 +514,9 @@ MemoryControl::issueRequest(int bank)
     m_bankQueues[bank].pop_front();
 
     DPRINTF(RubyMemory, "Mem issue request%7d: %#08x %c "
-            "bank=%3x\n", req.m_msg_counter, req.m_addr,
+            "bank=%3x sched %c\n", req.m_msg_counter, req.m_addr,
             req.m_is_mem_read? 'R':'W',
-            bank);
+            bank, m_event.scheduled() ? 'Y':'N');
 
     if (req.m_msgptr) {  // don't enqueue L3 writebacks
         enqueueToDirectory(req, m_mem_ctl_latency + m_mem_fixed_delay);
@@ -642,19 +643,28 @@ MemoryControl::executeCycle()
     }
 }
 
+unsigned int
+MemoryControl::drain(Event *de)
+{
+    DPRINTF(RubyMemory, "MemoryController drain\n");
+    if(m_event.scheduled()) {
+        deschedule(m_event);
+    }
+    return 0;
+}
+
 // wakeup:  This function is called once per memory controller clock cycle.
 void
 MemoryControl::wakeup()
 {
+    DPRINTF(RubyMemory, "MemoryController wakeup\n");
     // execute everything
     executeCycle();
 
     m_idleCount--;
-    if (m_idleCount <= 0) {
-        m_awakened = 0;
-    } else {
-        // Reschedule ourselves so that we run every memory cycle:
-        g_eventQueue_ptr->scheduleEvent(this, m_mem_bus_cycle_multiplier);
+    if (m_idleCount > 0) {
+        assert(!m_event.scheduled());
+        schedule(m_event, curTick() + m_mem_bus_cycle_multiplier);
     }
 }
 

src/mem/ruby/system/MemoryControl.hh

@@ -54,12 +54,15 @@ class MemoryControl :
     public SimObject, public Consumer, public AbstractMemOrCache
 {
   public:
+
     typedef RubyMemoryControlParams Params;
     MemoryControl(const Params *p);
     void init();
 
     ~MemoryControl();
 
+    unsigned int drain(Event *de);
+
     void wakeup();
 
     void setConsumer(Consumer* consumer_ptr);
@@ -90,6 +93,19 @@ class MemoryControl :
     int getDimmsPerChannel() { return m_dimms_per_channel; }
 
   private:
+    class MemCntrlEvent : public Event
+    {
+      public:
+        MemCntrlEvent(MemoryControl* _mem_cntrl)
+        {
+            mem_cntrl = _mem_cntrl;
+        }
+      private:
+        void process() { mem_cntrl->wakeup(); }
+
+        MemoryControl* mem_cntrl;
+    };
+
     void enqueueToDirectory(MemoryNode req, int latency);
     int getBank(physical_address_t addr);
     int getRank(int bank);
@@ -107,7 +123,6 @@ class MemoryControl :
     Consumer* m_consumer_ptr;  // Consumer to signal a wakeup()
     std::string m_description;
     int m_msg_counter;
-    int m_awakened;
 
     int m_mem_bus_cycle_multiplier;
     int m_banks_per_rank;
@@ -159,6 +174,8 @@ class MemoryControl :
     int m_idleCount;          // watchdog timer for shutting down
 
     MemCntrlProfiler* m_profiler_ptr;
+
+    MemCntrlEvent m_event;
 };
 
 #endif // __MEM_RUBY_SYSTEM_MEMORY_CONTROL_HH__

src/mem/ruby/system/MemoryVector.hh

@@ -30,6 +30,7 @@
 #define __MEM_RUBY_SYSTEM_MEMORYVECTOR_HH__
 
 #include "base/trace.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/ruby/common/Address.hh"
 
 class DirectoryMemory;
@@ -184,6 +185,8 @@ MemoryVector::collatePages(uint8* &raw_data)
     memcpy(raw_data, &m_num_pages, sizeof(uint32));
     data_size = sizeof(uint32);
 
+    DPRINTF(RubyCacheTrace, "collating %d pages\n", m_num_pages);
+
     for (uint32 i = 0;i < m_num_pages; ++i)
     {
         if (m_pages[i] == 0) {
@@ -217,6 +220,8 @@ MemoryVector::populatePages(uint8* raw_data)
     data_size = sizeof(uint32);
     assert(num_pages == m_num_pages);
 
+    DPRINTF(RubyCacheTrace, "Populating %d pages\n", num_pages);
+
     for (uint32 i = 0;i < m_num_pages; ++i)
     {
         assert(m_pages[i] == 0);

src/mem/ruby/system/System.cc

@@ -33,11 +33,12 @@
 
 #include "base/intmath.hh"
 #include "base/output.hh"
-#include "debug/RubySystem.hh"
+#include "debug/RubyCacheTrace.hh"
 #include "mem/ruby/common/Address.hh"
 #include "mem/ruby/network/Network.hh"
 #include "mem/ruby/profiler/Profiler.hh"
 #include "mem/ruby/system/System.hh"
+#include "sim/eventq.hh"
 #include "sim/simulate.hh"
 
 using namespace std;
@@ -224,6 +225,7 @@ RubySystem::serialize(std::ostream &os)
         }
     }
 
+    DPRINTF(RubyCacheTrace, "Recording Cache Trace\n");
     // Create the CacheRecorder and record the cache trace
     m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map);
 
@@ -231,15 +233,19 @@ RubySystem::serialize(std::ostream &os)
         m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder);
     }
 
+    DPRINTF(RubyCacheTrace, "Cache Trace Complete\n");
     // save the current tick value
     Tick curtick_original = curTick();
     // save the event queue head
     Event* eventq_head = eventq->replaceHead(NULL);
+    DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n",
+            curtick_original);
 
     // Schedule an event to start cache cooldown
-    RubyEvent* e = new RubyEvent(this);
-    schedule(e,curTick());
+    DPRINTF(RubyCacheTrace, "Starting cache flush\n");
+    enqueueRubyEvent(curTick());
     simulate();
+    DPRINTF(RubyCacheTrace, "Cache flush complete\n");
 
     // Restore eventq head
     eventq_head = eventq->replaceHead(eventq_head);
@@ -378,8 +384,7 @@ RubySystem::startup()
         curTick(0);
 
         // Schedule an event to start cache warmup
-        RubyEvent* e = new RubyEvent(this);
-        schedule(e,curTick());
+        enqueueRubyEvent(curTick());
         simulate();
 
         delete m_cache_recorder;

src/mem/ruby/system/System.hh

@@ -129,6 +129,13 @@ class RubySystem : public SimObject
     void registerAbstractController(AbstractController*);
     void registerSparseMemory(SparseMemory*);
 
+    bool eventQueueEmpty() { return eventq->empty(); }
+    void enqueueRubyEvent(Tick tick)
+    {
+        RubyEvent* e = new RubyEvent(this);
+        schedule(e, tick);
+    }
+
   private:
     // Private copy constructor and assignment operator
     RubySystem(const RubySystem& obj);