diff --git a/src/mem/ruby/system/RubyPort.hh b/src/mem/ruby/system/RubyPort.hh
index 2fb31ca092..e68af6dab3 100644
--- a/src/mem/ruby/system/RubyPort.hh
+++ b/src/mem/ruby/system/RubyPort.hh
@@ -188,8 +188,8 @@ class RubyPort : public MemObject
   private:
     void addToRetryList(MemSlavePort * port)
     {
-        assert(std::find(retryList.begin(), retryList.end(), port) ==
-               retryList.end());
+        if (std::find(retryList.begin(), retryList.end(), port) !=
+               retryList.end()) return;
         retryList.push_back(port);
     }