diff --git a/src/mem/ruby/system/RubyPort.cc b/src/mem/ruby/system/RubyPort.cc
index 800046e101..a6cee054a6 100644
--- a/src/mem/ruby/system/RubyPort.cc
+++ b/src/mem/ruby/system/RubyPort.cc
@@ -339,7 +339,10 @@ RubyPort::MemSlavePort::recvAtomic(PacketPtr pkt)
     RubySystem *rs = ruby_port->m_ruby_system;
     AbstractController *directory =
         rs->m_abstract_controls[id.getType()][id.getNum()];
-    return directory->recvAtomic(pkt);
+    Tick latency = directory->recvAtomic(pkt);
+    if (access_backing_store)
+        rs->getPhysMem()->access(pkt);
+    return latency;
 }
 
 void