cpu-kvm: Reinitialize threads on drainResume

Event queue service threads may have been re-created while the
simulator was drained. We therefore need to initialize the new thread
by setting correct signal masks and re-attaching performance counters.

Change-Id: Ic0dab80543928327021cade037770c917e73a47f
Signed-off-by: Andreas Sandberg <andreas.sandberg@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/50409
Reviewed-by: Austin Harris <mail@austin-harris.com>
Tested-by: kokoro <noreply+kokoro@google.com>

src/cpu/kvm/base.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2015, 2017 ARM Limited
+ * Copyright (c) 2012, 2015, 2017, 2021 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -154,9 +154,9 @@ BaseKvmCPU::startup()
         inform("KVM: Coalesced not supported by host OS\n");
     }
 
-    Event *startupEvent(
-        new EventFunctionWrapper([this]{ startupThread(); }, name(), true));
-    schedule(startupEvent, curTick());
+    schedule(new EventFunctionWrapper([this]{
+                restartEqThread();
+            }, name(), true), curTick());
 }
 
 BaseKvmCPU::Status
@@ -228,7 +228,7 @@ BaseKvmCPU::finishMMIOPending()
 }
 
 void
-BaseKvmCPU::startupThread()
+BaseKvmCPU::restartEqThread()
 {
     // Do thread-specific initialization. We need to setup signal
     // delivery for counters and timers from within the thread that
@@ -389,6 +389,13 @@ BaseKvmCPU::drainResume()
     DPRINTF(Kvm, "drainResume\n");
     verifyMemoryMode();
 
+    /* The simulator may have terminated the threads servicing event
+     * queues. In that case, we need to re-initialize the new
+     * threads. */
+    schedule(new EventFunctionWrapper([this]{
+                restartEqThread();
+            }, name(), true), curTick());
+
     // The tick event is de-scheduled as a part of the draining
     // process. Re-schedule it if the thread context is active.
     if (tc->status() == ThreadContext::Active) {
@@ -1275,6 +1282,11 @@ BaseKvmCPU::setupCounters()
             .samplePeriod(42);
     }
 
+    // We might be re-attaching counters due threads being
+    // re-initialised after fork.
+    if (hwCycles.attached())
+        hwCycles.detach();
+
     hwCycles.attach(cfgCycles,
                     0); // TID (0 => currentThread)
 

src/cpu/kvm/base.hh

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012 ARM Limited
+ * Copyright (c) 2012, 2021 ARM Limited
  * All rights reserved
  *
  * The license below extends only to copyright in the software and shall
@@ -682,11 +682,12 @@ class BaseKvmCPU : public BaseCPU
      * example, when setting up timers, we need to know the TID of the
      * thread executing in KVM in order to deliver the timer signal to
      * that thread. This method is called as the first event in this
-     * SimObject's event queue.
+     * SimObject's event queue and after drainResume to handle changes
+     * to event queue service threads.
      *
      * @see startup
      */
-    void startupThread();
+    void restartEqThread();
 
     /** Try to drain the CPU if a drain is pending */
     bool tryDrain();