From e44cbe724b7c1746eea17e2e3b71b3806270ec7d Mon Sep 17 00:00:00 2001
From: Earl Ou <shunhsingou@google.com>
Date: Wed, 1 Feb 2023 21:55:51 -0800
Subject: [PATCH] sim: handle async events in main thread only

In the current implementation pollqueue is not thread safe. The design
of multi threads handle async events is thus causing issue in parallel
environment. Given the low rate of async events, it should be OK to only
handle them in the main thread to avoid unexpected racing issues.

Change-Id: Iddd512235e84e9d77f60985bb1771aa4cc693004
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/67533
Reviewed-by: Gabe Black <gabe.black@gmail.com>
Reviewed-by: Yu-hsin Wang <yuhsingw@google.com>
Maintainer: Gabe Black <gabe.black@gmail.com>
Tested-by: kokoro <noreply+kokoro@google.com>
---
 src/sim/simulate.cc | 28 ++++------------------------
 1 file changed, 4 insertions(+), 24 deletions(-)

diff --git a/src/sim/simulate.cc b/src/sim/simulate.cc
index 86d516d39a..abd2b1d391 100644
--- a/src/sim/simulate.cc
+++ b/src/sim/simulate.cc
@@ -43,7 +43,6 @@
 #include "sim/simulate.hh"
 
 #include <atomic>
-#include <mutex>
 #include <thread>
 
 #include "base/logging.hh"
@@ -273,28 +272,6 @@ terminateEventQueueThreads()
 }
 
 
-/**
- * Test and clear the global async_event flag, such that each time the
- * flag is cleared, only one thread returns true (and thus is assigned
- * to handle the corresponding async event(s)).
- */
-static bool
-testAndClearAsyncEvent()
-{
-    static std::mutex mutex;
-
-    bool was_set = false;
-    mutex.lock();
-
-    if (async_event) {
-        was_set = true;
-        async_event = false;
-    }
-
-    mutex.unlock();
-    return was_set;
-}
-
 /**
  * The main per-thread simulation loop. This loop is executed by all
  * simulation threads (the main thread and the subordinate threads) in
@@ -307,6 +284,8 @@ doSimLoop(EventQueue *eventq)
     curEventQueue(eventq);
     eventq->handleAsyncInsertions();
 
+    bool mainQueue = eventq == getEventQueue(0);
+
     while (1) {
         // there should always be at least one event (the SimLoopExitEvent
         // we just scheduled) in the queue
@@ -314,7 +293,8 @@ doSimLoop(EventQueue *eventq)
         assert(curTick() <= eventq->nextTick() &&
                "event scheduled in the past");
 
-        if (async_event && testAndClearAsyncEvent()) {
+        if (mainQueue && async_event) {
+            async_event = false;
             // Take the event queue lock in case any of the service
             // routines want to schedule new events.
             std::lock_guard<EventQueue> lock(*eventq);