diff --git a/configs/common/cores/arm/O3_ARM_v7a.py b/configs/common/cores/arm/O3_ARM_v7a.py index de258324be..5413839747 100644 --- a/configs/common/cores/arm/O3_ARM_v7a.py +++ b/configs/common/cores/arm/O3_ARM_v7a.py @@ -207,9 +207,8 @@ class O3_ARM_v7aL2(Cache): size = "1MB" assoc = 16 write_buffers = 8 - prefetch_on_access = True clusivity = "mostly_excl" # Simple stride prefetcher - prefetcher = StridePrefetcher(degree=8, latency=1) + prefetcher = StridePrefetcher(degree=8, latency=1, prefetch_on_access=True) tags = BaseSetAssoc() replacement_policy = RandomRP() diff --git a/configs/common/cores/arm/ex5_LITTLE.py b/configs/common/cores/arm/ex5_LITTLE.py index a89881436d..372e5c97b3 100644 --- a/configs/common/cores/arm/ex5_LITTLE.py +++ b/configs/common/cores/arm/ex5_LITTLE.py @@ -147,9 +147,8 @@ class L2(Cache): size = "512kB" assoc = 8 write_buffers = 16 - prefetch_on_access = True clusivity = "mostly_excl" # Simple stride prefetcher - prefetcher = StridePrefetcher(degree=1, latency=1) + prefetcher = StridePrefetcher(degree=1, latency=1, prefetch_on_access=True) tags = BaseSetAssoc() replacement_policy = RandomRP() diff --git a/configs/common/cores/arm/ex5_big.py b/configs/common/cores/arm/ex5_big.py index 7803c1e0cc..53677ce3b3 100644 --- a/configs/common/cores/arm/ex5_big.py +++ b/configs/common/cores/arm/ex5_big.py @@ -200,9 +200,8 @@ class L2(Cache): size = "2MB" assoc = 16 write_buffers = 8 - prefetch_on_access = True clusivity = "mostly_excl" # Simple stride prefetcher - prefetcher = StridePrefetcher(degree=8, latency=1) + prefetcher = StridePrefetcher(degree=8, latency=1, prefetch_on_access=True) tags = BaseSetAssoc() replacement_policy = RandomRP() diff --git a/configs/ruby/CHI_config.py b/configs/ruby/CHI_config.py index 9c7391c0ec..80ca38ad5a 100644 --- a/configs/ruby/CHI_config.py +++ b/configs/ruby/CHI_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021,2022 ARM Limited +# Copyright (c) 2021-2023 ARM Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -232,7 +232,8 @@ class CHI_L1Controller(CHI_Cache_Controller): super().__init__(ruby_system) self.sequencer = sequencer self.cache = cache - self.use_prefetcher = False + self.prefetcher = prefetcher + self.use_prefetcher = prefetcher != NULL self.send_evictions = True self.is_HN = False self.enable_DMT = False @@ -269,7 +270,8 @@ class CHI_L2Controller(CHI_Cache_Controller): super().__init__(ruby_system) self.sequencer = NULL self.cache = cache - self.use_prefetcher = False + self.prefetcher = prefetcher + self.use_prefetcher = prefetcher != NULL self.allow_SD = True self.is_HN = False self.enable_DMT = False @@ -305,7 +307,8 @@ class CHI_HNFController(CHI_Cache_Controller): super().__init__(ruby_system) self.sequencer = NULL self.cache = cache - self.use_prefetcher = False + self.prefetcher = prefetcher + self.use_prefetcher = prefetcher != NULL self.addr_ranges = addr_ranges self.allow_SD = True self.is_HN = True @@ -381,6 +384,7 @@ class CHI_DMAController(CHI_Cache_Controller): size = "128" assoc = 1 + self.prefetcher = NULL self.use_prefetcher = False self.cache = DummyCache() self.sequencer.dcache = NULL @@ -500,11 +504,16 @@ class CHI_RNF(CHI_Node): start_index_bit=self._block_size_bits, is_icache=False ) - # Placeholders for future prefetcher support - if l1Iprefetcher_type != None or l1Dprefetcher_type != None: - m5.fatal("Prefetching not supported yet") - l1i_pf = NULL - l1d_pf = NULL + # prefetcher wrappers + if l1Iprefetcher_type != None: + l1i_pf = l1Iprefetcher_type() + else: + l1i_pf = NULL + + if l1Dprefetcher_type != None: + l1d_pf = l1Dprefetcher_type() + else: + l1d_pf = NULL # cache controllers cpu.l1i = CHI_L1Controller( @@ -549,9 +558,11 @@ class CHI_RNF(CHI_Node): l2_cache = cache_type( start_index_bit=self._block_size_bits, is_icache=False ) + if pf_type != None: - m5.fatal("Prefetching not supported yet") - l2_pf = NULL + l2_pf = pf_type() + else: + l2_pf = NULL cpu.l2 = CHI_L2Controller(self._ruby_system, l2_cache, l2_pf) diff --git a/src/mem/cache/Cache.py b/src/mem/cache/Cache.py index ed4851e69a..c1b1b95648 100644 --- a/src/mem/cache/Cache.py +++ b/src/mem/cache/Cache.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012-2013, 2015, 2018 ARM Limited +# Copyright (c) 2012-2013, 2015, 2018, 2023 ARM Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -106,13 +106,6 @@ class BaseCache(ClockedObject): is_read_only = Param.Bool(False, "Is this cache read only (e.g. inst)") prefetcher = Param.BasePrefetcher(NULL, "Prefetcher attached to cache") - prefetch_on_access = Param.Bool( - False, - "Notify the hardware prefetcher on every access (not just misses)", - ) - prefetch_on_pf_hit = Param.Bool( - False, "Notify the hardware prefetcher on hit on prefetched lines" - ) tags = Param.BaseTags(BaseSetAssoc(), "Tag store") replacement_policy = Param.BaseReplacementPolicy( diff --git a/src/mem/cache/base.cc b/src/mem/cache/base.cc index 87c44cefb7..e738167066 100644 --- a/src/mem/cache/base.cc +++ b/src/mem/cache/base.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013, 2018-2019 ARM Limited + * Copyright (c) 2012-2013, 2018-2019, 2023 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -81,6 +81,7 @@ BaseCache::BaseCache(const BaseCacheParams &p, unsigned blk_size) : ClockedObject(p), cpuSidePort (p.name + ".cpu_side_port", *this, "CpuSidePort"), memSidePort(p.name + ".mem_side_port", this, "MemSidePort"), + accessor(*this), mshrQueue("MSHRs", p.mshrs, 0, p.demand_mshr_reserve, p.name), writeBuffer("write buffer", p.write_buffers, p.mshrs, p.name), tags(p.tags), @@ -126,7 +127,7 @@ BaseCache::BaseCache(const BaseCacheParams &p, unsigned blk_size) tags->tagsInit(); if (prefetcher) - prefetcher->setCache(this); + prefetcher->setParentInfo(system, getProbeManager(), getBlockSize()); fatal_if(compressor && !dynamic_cast(tags), "The tags of compressed cache %s must derive from CompressedTags", @@ -448,7 +449,7 @@ BaseCache::recvTimingReq(PacketPtr pkt) if (satisfied) { // notify before anything else as later handleTimingReqHit might turn // the packet in a response - ppHit->notify(pkt); + ppHit->notify(CacheAccessProbeArg(pkt,accessor)); if (prefetcher && blk && blk->wasPrefetched()) { DPRINTF(Cache, "Hit on prefetch for addr %#x (%s)\n", @@ -460,7 +461,7 @@ BaseCache::recvTimingReq(PacketPtr pkt) } else { handleTimingReqMiss(pkt, blk, forward_time, request_time); - ppMiss->notify(pkt); + ppMiss->notify(CacheAccessProbeArg(pkt,accessor)); } if (prefetcher) { @@ -557,7 +558,7 @@ BaseCache::recvTimingResp(PacketPtr pkt) writeAllocator->allocate() : mshr->allocOnFill(); blk = handleFill(pkt, blk, writebacks, allocate); assert(blk != nullptr); - ppFill->notify(pkt); + ppFill->notify(CacheAccessProbeArg(pkt, accessor)); } // Don't want to promote the Locked RMW Read until @@ -771,7 +772,9 @@ void BaseCache::updateBlockData(CacheBlk *blk, const PacketPtr cpkt, bool has_old_data) { - DataUpdate data_update(regenerateBlkAddr(blk), blk->isSecure()); + CacheDataUpdateProbeArg data_update( + regenerateBlkAddr(blk), blk->isSecure(), + blk->getSrcRequestorId(), accessor); if (ppDataUpdate->hasListeners()) { if (has_old_data) { data_update.oldData = std::vector(blk->data, @@ -788,6 +791,7 @@ BaseCache::updateBlockData(CacheBlk *blk, const PacketPtr cpkt, if (cpkt) { data_update.newData = std::vector(blk->data, blk->data + (blkSize / sizeof(uint64_t))); + data_update.hwPrefetched = blk->wasPrefetched(); } ppDataUpdate->notify(data_update); } @@ -809,7 +813,9 @@ BaseCache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt) assert(sizeof(uint64_t) >= pkt->getSize()); // Get a copy of the old block's contents for the probe before the update - DataUpdate data_update(regenerateBlkAddr(blk), blk->isSecure()); + CacheDataUpdateProbeArg data_update( + regenerateBlkAddr(blk), blk->isSecure(), blk->getSrcRequestorId(), + accessor); if (ppDataUpdate->hasListeners()) { data_update.oldData = std::vector(blk->data, blk->data + (blkSize / sizeof(uint64_t))); @@ -1106,7 +1112,9 @@ BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) if (pkt->isAtomicOp()) { // Get a copy of the old block's contents for the probe before // the update - DataUpdate data_update(regenerateBlkAddr(blk), blk->isSecure()); + CacheDataUpdateProbeArg data_update( + regenerateBlkAddr(blk), blk->isSecure(), + blk->getSrcRequestorId(), accessor); if (ppDataUpdate->hasListeners()) { data_update.oldData = std::vector(blk->data, blk->data + (blkSize / sizeof(uint64_t))); @@ -1125,6 +1133,7 @@ BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) if (ppDataUpdate->hasListeners()) { data_update.newData = std::vector(blk->data, blk->data + (blkSize / sizeof(uint64_t))); + data_update.hwPrefetched = blk->wasPrefetched(); ppDataUpdate->notify(data_update); } @@ -2507,11 +2516,15 @@ BaseCache::CacheStats::regStats() void BaseCache::regProbePoints() { - ppHit = new ProbePointArg(this->getProbeManager(), "Hit"); - ppMiss = new ProbePointArg(this->getProbeManager(), "Miss"); - ppFill = new ProbePointArg(this->getProbeManager(), "Fill"); + ppHit = new ProbePointArg( + this->getProbeManager(), "Hit"); + ppMiss = new ProbePointArg( + this->getProbeManager(), "Miss"); + ppFill = new ProbePointArg( + this->getProbeManager(), "Fill"); ppDataUpdate = - new ProbePointArg(this->getProbeManager(), "Data Update"); + new ProbePointArg( + this->getProbeManager(), "Data Update"); } /////////////// diff --git a/src/mem/cache/base.hh b/src/mem/cache/base.hh index 8a06ec2c42..c2d9bc8a7b 100644 --- a/src/mem/cache/base.hh +++ b/src/mem/cache/base.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012-2013, 2015-2016, 2018-2019 ARM Limited + * Copyright (c) 2012-2013, 2015-2016, 2018-2019, 2023 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -59,6 +59,7 @@ #include "debug/CachePort.hh" #include "enums/Clusivity.hh" #include "mem/cache/cache_blk.hh" +#include "mem/cache/cache_probe_arg.hh" #include "mem/cache/compressors/base.hh" #include "mem/cache/mshr_queue.hh" #include "mem/cache/tags/base.hh" @@ -115,28 +116,6 @@ class BaseCache : public ClockedObject NUM_BLOCKED_CAUSES }; - /** - * A data contents update is composed of the updated block's address, - * the old contents, and the new contents. - * @sa ppDataUpdate - */ - struct DataUpdate - { - /** The updated block's address. */ - Addr addr; - /** Whether the block belongs to the secure address space. */ - bool isSecure; - /** The stale data contents. If zero-sized this update is a fill. */ - std::vector oldData; - /** The new data contents. If zero-sized this is an invalidation. */ - std::vector newData; - - DataUpdate(Addr _addr, bool is_secure) - : addr(_addr), isSecure(is_secure), oldData(), newData() - { - } - }; - protected: /** @@ -336,6 +315,30 @@ class BaseCache : public ClockedObject protected: + struct CacheAccessorImpl : CacheAccessor + { + BaseCache &cache; + + CacheAccessorImpl(BaseCache &_cache) :cache(_cache) {} + + bool inCache(Addr addr, bool is_secure) const override + { return cache.inCache(addr, is_secure); } + + bool hasBeenPrefetched(Addr addr, bool is_secure) const override + { return cache.hasBeenPrefetched(addr, is_secure); } + + bool hasBeenPrefetched(Addr addr, bool is_secure, + RequestorID requestor) const override + { return cache.hasBeenPrefetched(addr, is_secure, requestor); } + + bool inMissQueue(Addr addr, bool is_secure) const override + { return cache.inMissQueue(addr, is_secure); } + + bool coalesce() const override + { return cache.coalesce(); } + + } accessor; + /** Miss status registers */ MSHRQueue mshrQueue; @@ -352,20 +355,20 @@ class BaseCache : public ClockedObject prefetch::Base *prefetcher; /** To probe when a cache hit occurs */ - ProbePointArg *ppHit; + ProbePointArg *ppHit; /** To probe when a cache miss occurs */ - ProbePointArg *ppMiss; + ProbePointArg *ppMiss; /** To probe when a cache fill occurs */ - ProbePointArg *ppFill; + ProbePointArg *ppFill; /** * To probe when the contents of a block are updated. Content updates * include data fills, overwrites, and invalidations, which means that * this probe partially overlaps with other probes. */ - ProbePointArg *ppDataUpdate; + ProbePointArg *ppDataUpdate; /** * The writeAllocator drive optimizations for streaming writes. @@ -1278,11 +1281,14 @@ class BaseCache : public ClockedObject bool hasBeenPrefetched(Addr addr, bool is_secure) const { CacheBlk *block = tags->findBlock(addr, is_secure); - if (block) { - return block->wasPrefetched(); - } else { - return false; - } + return block && block->wasPrefetched(); + } + + bool hasBeenPrefetched(Addr addr, bool is_secure, + RequestorID requestor) const { + CacheBlk *block = tags->findBlock(addr, is_secure); + return block && block->wasPrefetched() && + (block->getSrcRequestorId() == requestor); } bool inMissQueue(Addr addr, bool is_secure) const { diff --git a/src/mem/cache/cache_probe_arg.hh b/src/mem/cache/cache_probe_arg.hh new file mode 100644 index 0000000000..27c7bc4041 --- /dev/null +++ b/src/mem/cache/cache_probe_arg.hh @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023 ARM Limited + * All rights reserved. + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef __MEM_CACHE_PROBE_ARG_HH__ +#define __MEM_CACHE_PROBE_ARG_HH__ + +#include "mem/packet.hh" + +namespace gem5 +{ + +/** + * Provides generic cache lookup functions. A cache may provide + * a CacheAccessor object to other components that need to perform + * a lookup outside the normal cache control flow. Currently this + * is used by prefetchers that perform lookups when notified by + * cache events. + */ +struct CacheAccessor +{ + /** Determine if address is in cache */ + virtual bool inCache(Addr addr, bool is_secure) const = 0; + + /** Determine if address has been prefetched */ + virtual bool hasBeenPrefetched(Addr addr, bool is_secure) const = 0; + + /** Determine if address has been prefetched by the requestor */ + virtual bool hasBeenPrefetched(Addr addr, bool is_secure, + RequestorID requestor) const = 0; + + /** Determine if address is in cache miss queue */ + virtual bool inMissQueue(Addr addr, bool is_secure) const = 0; + + /** Determine if cache is coalescing writes */ + virtual bool coalesce() const = 0; +}; + +/** + * Information provided to probes on a cache event. + * @sa ppHit, ppMiss, ppFill in gem5::BaseCache (src/mem/cache/base.hh) + */ +class CacheAccessProbeArg +{ + public: + /** Packet that triggered the cache access*/ + PacketPtr pkt; + /** Accessor for the cache */ + CacheAccessor &cache; + + CacheAccessProbeArg(PacketPtr _pkt, CacheAccessor &_cache) + :pkt(_pkt), cache(_cache) + { + } +}; + +/** + * A data contents update is composed of the updated block's address, + * the old contents, and the new contents. + * @sa ppDataUpdate in gem5::BaseCache (src/mem/cache/base.hh) + */ +struct CacheDataUpdateProbeArg +{ + /** The updated block's address. */ + Addr addr; + /** Whether the block belongs to the secure address space. */ + bool isSecure; + /** Block original requestor */ + const RequestorID requestorID; + /** The stale data contents. If zero-sized this update is a fill. */ + std::vector oldData; + /** The new data contents. If zero-sized this is an invalidation. */ + std::vector newData; + /** Set if the update is from a prefetch or evicting a prefetched + * block that was never used. */ + bool hwPrefetched; + /** Accessor for the cache */ + CacheAccessor &accessor; + + CacheDataUpdateProbeArg(Addr _addr, bool is_secure, + RequestorID _requestorID, + CacheAccessor &_accessor) + : addr(_addr), isSecure(is_secure), requestorID(_requestorID), + oldData(), newData(), accessor(_accessor) + { + } +}; + +} // namespace gem5 + +#endif //__MEM_CACHE_PROBE_ARG_HH__ diff --git a/src/mem/cache/compressors/frequent_values.hh b/src/mem/cache/compressors/frequent_values.hh index e7eac2644f..41103ce594 100644 --- a/src/mem/cache/compressors/frequent_values.hh +++ b/src/mem/cache/compressors/frequent_values.hh @@ -37,6 +37,7 @@ #include "base/sat_counter.hh" #include "base/types.hh" #include "mem/cache/base.hh" +#include "mem/cache/cache_probe_arg.hh" #include "mem/cache/compressors/base.hh" #include "mem/cache/compressors/encoders/huffman.hh" #include "mem/cache/prefetch/associative_set.hh" @@ -63,7 +64,7 @@ class FrequentValues : public Base private: class CompData; - using DataUpdate = BaseCache::DataUpdate; + using DataUpdate = CacheDataUpdateProbeArg; class FrequentValuesListener : public ProbeListenerArgBase { diff --git a/src/mem/cache/prefetch/Prefetcher.py b/src/mem/cache/prefetch/Prefetcher.py index 0a8c91d538..335d6f6942 100644 --- a/src/mem/cache/prefetch/Prefetcher.py +++ b/src/mem/cache/prefetch/Prefetcher.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012, 2014, 2019 ARM Limited +# Copyright (c) 2012, 2014, 2019, 2023 ARM Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -75,11 +75,11 @@ class BasePrefetcher(ClockedObject): on_data = Param.Bool(True, "Notify prefetcher on data accesses") on_inst = Param.Bool(True, "Notify prefetcher on instruction accesses") prefetch_on_access = Param.Bool( - Parent.prefetch_on_access, + False, "Notify the hardware prefetcher on every access (not just misses)", ) prefetch_on_pf_hit = Param.Bool( - Parent.prefetch_on_pf_hit, + False, "Notify the hardware prefetcher on hit on prefetched lines", ) use_virtual_addresses = Param.Bool( diff --git a/src/mem/cache/prefetch/access_map_pattern_matching.cc b/src/mem/cache/prefetch/access_map_pattern_matching.cc index 989f3c6be1..f53b77868f 100644 --- a/src/mem/cache/prefetch/access_map_pattern_matching.cc +++ b/src/mem/cache/prefetch/access_map_pattern_matching.cc @@ -156,7 +156,8 @@ AccessMapPatternMatching::setEntryState(AccessMapEntry &entry, void AccessMapPatternMatching::calculatePrefetch(const Base::PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { assert(addresses.empty()); @@ -262,9 +263,10 @@ AMPM::AMPM(const AMPMPrefetcherParams &p) void AMPM::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { - ampm.calculatePrefetch(pfi, addresses); + ampm.calculatePrefetch(pfi, addresses, cache); } } // namespace prefetch diff --git a/src/mem/cache/prefetch/access_map_pattern_matching.hh b/src/mem/cache/prefetch/access_map_pattern_matching.hh index 893d30dec2..b98a241b02 100644 --- a/src/mem/cache/prefetch/access_map_pattern_matching.hh +++ b/src/mem/cache/prefetch/access_map_pattern_matching.hh @@ -190,7 +190,8 @@ class AccessMapPatternMatching : public ClockedObject void startup() override; void calculatePrefetch(const Base::PrefetchInfo &pfi, - std::vector &addresses); + std::vector &addresses, + const CacheAccessor &cache); }; class AMPM : public Queued @@ -201,7 +202,8 @@ class AMPM : public Queued ~AMPM() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/base.cc b/src/mem/cache/prefetch/base.cc index e3e4b24cf2..f9d2624e7a 100644 --- a/src/mem/cache/prefetch/base.cc +++ b/src/mem/cache/prefetch/base.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 ARM Limited + * Copyright (c) 2013-2014, 2023 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -83,18 +83,26 @@ Base::PrefetchInfo::PrefetchInfo(PrefetchInfo const &pfi, Addr addr) } void -Base::PrefetchListener::notify(const PacketPtr &pkt) +Base::PrefetchListener::notify(const CacheAccessProbeArg &arg) { if (isFill) { - parent.notifyFill(pkt); + parent.notifyFill(arg); } else { - parent.probeNotify(pkt, miss); + parent.probeNotify(arg, miss); } } +void +Base::PrefetchEvictListener::notify(const EvictionInfo &info) +{ + if (info.newData.empty()) + parent.notifyEvict(info); +} + Base::Base(const BasePrefetcherParams &p) - : ClockedObject(p), listeners(), cache(nullptr), blkSize(p.block_size), - lBlkSize(floorLog2(blkSize)), onMiss(p.on_miss), onRead(p.on_read), + : ClockedObject(p), listeners(), system(nullptr), probeManager(nullptr), + blkSize(p.block_size), lBlkSize(floorLog2(blkSize)), + onMiss(p.on_miss), onRead(p.on_read), onWrite(p.on_write), onData(p.on_data), onInst(p.on_inst), requestorId(p.sys->getRequestorId(this)), pageBytes(p.page_bytes), @@ -107,13 +115,13 @@ Base::Base(const BasePrefetcherParams &p) } void -Base::setCache(BaseCache *_cache) +Base::setParentInfo(System *sys, ProbeManager *pm, unsigned blk_size) { - assert(!cache); - cache = _cache; - + assert(!system && !probeManager); + system = sys; + probeManager = pm; // If the cache has a different block size from the system's, save it - blkSize = cache->getBlockSize(); + blkSize = blk_size; lBlkSize = floorLog2(blkSize); } @@ -157,7 +165,7 @@ Base::StatGroup::StatGroup(statistics::Group *parent) } bool -Base::observeAccess(const PacketPtr &pkt, bool miss) const +Base::observeAccess(const PacketPtr &pkt, bool miss, bool prefetched) const { bool fetch = pkt->req->isInstFetch(); bool read = pkt->isRead(); @@ -165,7 +173,7 @@ Base::observeAccess(const PacketPtr &pkt, bool miss) const if (!miss) { if (prefetchOnPfHit) - return hasBeenPrefetched(pkt->getAddr(), pkt->isSecure()); + return prefetched; if (!prefetchOnAccess) return false; } @@ -184,24 +192,6 @@ Base::observeAccess(const PacketPtr &pkt, bool miss) const return true; } -bool -Base::inCache(Addr addr, bool is_secure) const -{ - return cache->inCache(addr, is_secure); -} - -bool -Base::inMissQueue(Addr addr, bool is_secure) const -{ - return cache->inMissQueue(addr, is_secure); -} - -bool -Base::hasBeenPrefetched(Addr addr, bool is_secure) const -{ - return cache->hasBeenPrefetched(addr, is_secure); -} - bool Base::samePage(Addr a, Addr b) const { @@ -239,18 +229,24 @@ Base::pageIthBlockAddress(Addr page, uint32_t blockIndex) const } void -Base::probeNotify(const PacketPtr &pkt, bool miss) +Base::probeNotify(const CacheAccessProbeArg &acc, bool miss) { + const PacketPtr pkt = acc.pkt; + const CacheAccessor &cache = acc.cache; + // Don't notify prefetcher on SWPrefetch, cache maintenance // operations or for writes that we are coaslescing. if (pkt->cmd.isSWPrefetch()) return; if (pkt->req->isCacheMaintenance()) return; - if (pkt->isWrite() && cache != nullptr && cache->coalesce()) return; + if (pkt->isWrite() && cache.coalesce()) return; if (!pkt->req->hasPaddr()) { panic("Request must have a physical address"); } - if (hasBeenPrefetched(pkt->getAddr(), pkt->isSecure())) { + bool has_been_prefetched = + acc.cache.hasBeenPrefetched(pkt->getAddr(), pkt->isSecure(), + requestorId); + if (has_been_prefetched) { usefulPrefetches += 1; prefetchStats.pfUseful++; if (miss) @@ -260,13 +256,13 @@ Base::probeNotify(const PacketPtr &pkt, bool miss) } // Verify this access type is observed by prefetcher - if (observeAccess(pkt, miss)) { + if (observeAccess(pkt, miss, has_been_prefetched)) { if (useVirtualAddresses && pkt->req->hasVaddr()) { PrefetchInfo pfi(pkt, pkt->req->getVaddr(), miss); - notify(pkt, pfi); + notify(acc, pfi); } else if (!useVirtualAddresses) { PrefetchInfo pfi(pkt, pkt->req->getPaddr(), miss); - notify(pkt, pfi); + notify(acc, pfi); } } } @@ -279,14 +275,15 @@ Base::regProbeListeners() * parent cache using the probe "Miss". Also connect to "Hit", if the * cache is configured to prefetch on accesses. */ - if (listeners.empty() && cache != nullptr) { - ProbeManager *pm(cache->getProbeManager()); - listeners.push_back(new PrefetchListener(*this, pm, "Miss", false, - true)); - listeners.push_back(new PrefetchListener(*this, pm, "Fill", true, - false)); - listeners.push_back(new PrefetchListener(*this, pm, "Hit", false, - false)); + if (listeners.empty() && probeManager != nullptr) { + listeners.push_back(new PrefetchListener(*this, probeManager, + "Miss", false, true)); + listeners.push_back(new PrefetchListener(*this, probeManager, + "Fill", true, false)); + listeners.push_back(new PrefetchListener(*this, probeManager, + "Hit", false, false)); + listeners.push_back(new PrefetchEvictListener(*this, probeManager, + "Data Update")); } } diff --git a/src/mem/cache/prefetch/base.hh b/src/mem/cache/prefetch/base.hh index 6bae73519c..f4d5cb051b 100644 --- a/src/mem/cache/prefetch/base.hh +++ b/src/mem/cache/prefetch/base.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2014 ARM Limited + * Copyright (c) 2013-2014, 2023 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -52,7 +52,7 @@ #include "base/compiler.hh" #include "base/statistics.hh" #include "base/types.hh" -#include "mem/cache/cache_blk.hh" +#include "mem/cache/cache_probe_arg.hh" #include "mem/packet.hh" #include "mem/request.hh" #include "sim/byteswap.hh" @@ -62,15 +62,16 @@ namespace gem5 { -class BaseCache; struct BasePrefetcherParams; +class ProbeManager; +class System; namespace prefetch { class Base : public ClockedObject { - class PrefetchListener : public ProbeListenerArgBase + class PrefetchListener : public ProbeListenerArgBase { public: PrefetchListener(Base &_parent, ProbeManager *pm, @@ -78,14 +79,27 @@ class Base : public ClockedObject bool _miss = false) : ProbeListenerArgBase(pm, name), parent(_parent), isFill(_isFill), miss(_miss) {} - void notify(const PacketPtr &pkt) override; + void notify(const CacheAccessProbeArg &arg) override; protected: Base &parent; const bool isFill; const bool miss; }; - std::vector listeners; + using EvictionInfo = CacheDataUpdateProbeArg; + + class PrefetchEvictListener : public ProbeListenerArgBase + { + public: + PrefetchEvictListener(Base &_parent, ProbeManager *pm, + const std::string &name) + : ProbeListenerArgBase(pm, name), parent(_parent) {} + void notify(const EvictionInfo &info) override; + protected: + Base &parent; + }; + + std::vector listeners; public: @@ -262,8 +276,11 @@ class Base : public ClockedObject // PARAMETERS - /** Pointr to the parent cache. */ - BaseCache* cache; + /** Pointer to the parent system. */ + System* system; + + /** Pointer to the parent cache's probe manager. */ + ProbeManager *probeManager; /** The block size of the parent cache. */ unsigned blkSize; @@ -304,16 +321,9 @@ class Base : public ClockedObject * Determine if this access should be observed * @param pkt The memory request causing the event * @param miss whether this event comes from a cache miss + * @param prefetched on a hit, this indicates the block was prefetched */ - bool observeAccess(const PacketPtr &pkt, bool miss) const; - - /** Determine if address is in cache */ - bool inCache(Addr addr, bool is_secure) const; - - /** Determine if address is in cache miss queue */ - bool inMissQueue(Addr addr, bool is_secure) const; - - bool hasBeenPrefetched(Addr addr, bool is_secure) const; + bool observeAccess(const PacketPtr &pkt, bool miss, bool prefetched) const; /** Determine if addresses are on the same page */ bool samePage(Addr a, Addr b) const; @@ -370,16 +380,22 @@ class Base : public ClockedObject Base(const BasePrefetcherParams &p); virtual ~Base() = default; - virtual void setCache(BaseCache *_cache); + virtual void + setParentInfo(System *sys, ProbeManager *pm, unsigned blk_size); /** * Notify prefetcher of cache access (may be any access or just * misses, depending on cache parameters.) */ - virtual void notify(const PacketPtr &pkt, const PrefetchInfo &pfi) = 0; + virtual void + notify(const CacheAccessProbeArg &acc, const PrefetchInfo &pfi) = 0; /** Notify prefetcher of cache fill */ - virtual void notifyFill(const PacketPtr &pkt) + virtual void notifyFill(const CacheAccessProbeArg &acc) + {} + + /** Notify prefetcher of cache eviction */ + virtual void notifyEvict(const EvictionInfo &info) {} virtual PacketPtr getPacket() = 0; @@ -423,10 +439,10 @@ class Base : public ClockedObject /** * Process a notification event from the ProbeListener. - * @param pkt The memory request causing the event + * @param acc probe arg encapsulating the memory request causing the event * @param miss whether this event comes from a cache miss */ - void probeNotify(const PacketPtr &pkt, bool miss); + void probeNotify(const CacheAccessProbeArg &acc, bool miss); /** * Add a SimObject and a probe name to listen events from diff --git a/src/mem/cache/prefetch/bop.cc b/src/mem/cache/prefetch/bop.cc index ce2502bee6..d50e9460d4 100644 --- a/src/mem/cache/prefetch/bop.cc +++ b/src/mem/cache/prefetch/bop.cc @@ -227,7 +227,8 @@ BOP::bestOffsetLearning(Addr x) void BOP::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { Addr addr = pfi.getAddr(); Addr tag_x = tag(addr); @@ -252,8 +253,10 @@ BOP::calculatePrefetch(const PrefetchInfo &pfi, } void -BOP::notifyFill(const PacketPtr& pkt) +BOP::notifyFill(const CacheAccessProbeArg &arg) { + const PacketPtr& pkt = arg.pkt; + // Only insert into the RR right way if it's the pkt is a HWP if (!pkt->cmd.isHWPrefetch()) return; diff --git a/src/mem/cache/prefetch/bop.hh b/src/mem/cache/prefetch/bop.hh index bb1b05dfa9..09ae4ea72d 100644 --- a/src/mem/cache/prefetch/bop.hh +++ b/src/mem/cache/prefetch/bop.hh @@ -148,7 +148,7 @@ class BOP : public Queued void bestOffsetLearning(Addr); /** Update the RR right table after a prefetch fill */ - void notifyFill(const PacketPtr& pkt) override; + void notifyFill(const CacheAccessProbeArg &arg) override; public: @@ -156,7 +156,8 @@ class BOP : public Queued ~BOP() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/delta_correlating_prediction_tables.cc b/src/mem/cache/prefetch/delta_correlating_prediction_tables.cc index b59394ce25..ea59bea3c0 100644 --- a/src/mem/cache/prefetch/delta_correlating_prediction_tables.cc +++ b/src/mem/cache/prefetch/delta_correlating_prediction_tables.cc @@ -125,7 +125,8 @@ DeltaCorrelatingPredictionTables::DCPTEntry::getCandidates( void DeltaCorrelatingPredictionTables::calculatePrefetch( const Base::PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { if (!pfi.hasPC()) { DPRINTF(HWPrefetch, "Ignoring request with no PC.\n"); @@ -156,9 +157,10 @@ DCPT::DCPT(const DCPTPrefetcherParams &p) void DCPT::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { - dcpt.calculatePrefetch(pfi, addresses); + dcpt.calculatePrefetch(pfi, addresses, cache); } } // namespace prefetch diff --git a/src/mem/cache/prefetch/delta_correlating_prediction_tables.hh b/src/mem/cache/prefetch/delta_correlating_prediction_tables.hh index 0218e9138a..7280c96733 100644 --- a/src/mem/cache/prefetch/delta_correlating_prediction_tables.hh +++ b/src/mem/cache/prefetch/delta_correlating_prediction_tables.hh @@ -114,9 +114,11 @@ class DeltaCorrelatingPredictionTables : public SimObject * Computes the prefetch candidates given a prefetch event. * @param pfi The prefetch event information * @param addresses prefetch candidates generated + * @param cache accessor for cache lookups */ void calculatePrefetch(const Base::PrefetchInfo &pfi, - std::vector &addresses); + std::vector &addresses, + const CacheAccessor &cache); }; @@ -130,7 +132,8 @@ class DCPT : public Queued ~DCPT() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/indirect_memory.cc b/src/mem/cache/prefetch/indirect_memory.cc index ab84ce25a2..780879bc6a 100644 --- a/src/mem/cache/prefetch/indirect_memory.cc +++ b/src/mem/cache/prefetch/indirect_memory.cc @@ -56,7 +56,8 @@ IndirectMemory::IndirectMemory(const IndirectMemoryPrefetcherParams &p) void IndirectMemory::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { // This prefetcher requires a PC if (!pfi.hasPC()) { diff --git a/src/mem/cache/prefetch/indirect_memory.hh b/src/mem/cache/prefetch/indirect_memory.hh index da3e894cfa..877e55d63d 100644 --- a/src/mem/cache/prefetch/indirect_memory.hh +++ b/src/mem/cache/prefetch/indirect_memory.hh @@ -201,7 +201,8 @@ class IndirectMemory : public Queued ~IndirectMemory() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/irregular_stream_buffer.cc b/src/mem/cache/prefetch/irregular_stream_buffer.cc index ce30b41aa6..bf81ebedc5 100644 --- a/src/mem/cache/prefetch/irregular_stream_buffer.cc +++ b/src/mem/cache/prefetch/irregular_stream_buffer.cc @@ -66,7 +66,8 @@ IrregularStreamBuffer::IrregularStreamBuffer( void IrregularStreamBuffer::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { // This prefetcher requires a PC if (!pfi.hasPC()) { diff --git a/src/mem/cache/prefetch/irregular_stream_buffer.hh b/src/mem/cache/prefetch/irregular_stream_buffer.hh index 39373010bb..399268a5a4 100644 --- a/src/mem/cache/prefetch/irregular_stream_buffer.hh +++ b/src/mem/cache/prefetch/irregular_stream_buffer.hh @@ -137,7 +137,8 @@ class IrregularStreamBuffer : public Queued ~IrregularStreamBuffer() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/multi.cc b/src/mem/cache/prefetch/multi.cc index 1f7298f354..9cd40163e2 100644 --- a/src/mem/cache/prefetch/multi.cc +++ b/src/mem/cache/prefetch/multi.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019 ARM Limited + * Copyright (c) 2014, 2019, 2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -53,10 +53,10 @@ Multi::Multi(const MultiPrefetcherParams &p) } void -Multi::setCache(BaseCache *_cache) +Multi::setParentInfo(System *sys, ProbeManager *pm, unsigned blk_size) { for (auto pf : prefetchers) - pf->setCache(_cache); + pf->setParentInfo(sys, pm, blk_size); } Tick diff --git a/src/mem/cache/prefetch/multi.hh b/src/mem/cache/prefetch/multi.hh index 7890f090b5..74cdaec984 100644 --- a/src/mem/cache/prefetch/multi.hh +++ b/src/mem/cache/prefetch/multi.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2019 ARM Limited + * Copyright (c) 2014, 2019, 2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -56,7 +56,8 @@ class Multi : public Base Multi(const MultiPrefetcherParams &p); public: - void setCache(BaseCache *_cache) override; + void + setParentInfo(System *sys, ProbeManager *pm, unsigned blk_size) override; PacketPtr getPacket() override; Tick nextPrefetchReadyTime() const override; @@ -65,8 +66,11 @@ class Multi : public Base * Ignore notifications since each sub-prefetcher already gets a * notification through their probes-based interface. */ - void notify(const PacketPtr &pkt, const PrefetchInfo &pfi) override {}; - void notifyFill(const PacketPtr &pkt) override {}; + void + notify(const CacheAccessProbeArg &arg, const PrefetchInfo &pfi) override + {}; + + void notifyFill(const CacheAccessProbeArg &arg) override {}; /** @} */ protected: diff --git a/src/mem/cache/prefetch/pif.cc b/src/mem/cache/prefetch/pif.cc index 79e8e6d747..581831491f 100644 --- a/src/mem/cache/prefetch/pif.cc +++ b/src/mem/cache/prefetch/pif.cc @@ -198,7 +198,8 @@ PIF::notifyRetiredInst(const Addr pc) void PIF::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { if (!pfi.hasPC()) { return; diff --git a/src/mem/cache/prefetch/pif.hh b/src/mem/cache/prefetch/pif.hh index 296087e8e0..ecb97db78d 100644 --- a/src/mem/cache/prefetch/pif.hh +++ b/src/mem/cache/prefetch/pif.hh @@ -182,7 +182,8 @@ class PIF : public Queued ~PIF() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses); + std::vector &addresses, + const CacheAccessor &cache); /** * Add a SimObject and a probe name to monitor the retired instructions diff --git a/src/mem/cache/prefetch/queued.cc b/src/mem/cache/prefetch/queued.cc index 1ab34d2e9b..e22d6f5eb8 100644 --- a/src/mem/cache/prefetch/queued.cc +++ b/src/mem/cache/prefetch/queued.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2015 ARM Limited + * Copyright (c) 2014-2015, 2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -94,7 +94,7 @@ Queued::DeferredPacket::finish(const Fault &fault, assert(ongoingTranslation); ongoingTranslation = false; bool failed = (fault != NoFault); - owner->translationComplete(this, failed); + owner->translationComplete(this, failed, *cache); } Queued::Queued(const QueuedPrefetcherParams &p) @@ -169,10 +169,12 @@ Queued::getMaxPermittedPrefetches(size_t total) const } void -Queued::notify(const PacketPtr &pkt, const PrefetchInfo &pfi) +Queued::notify(const CacheAccessProbeArg &acc, const PrefetchInfo &pfi) { Addr blk_addr = blockAddress(pfi.getAddr()); bool is_secure = pfi.isSecure(); + const PacketPtr pkt = acc.pkt; + const CacheAccessor &cache = acc.cache; // Squash queued prefetches if demand miss to same line if (queueSquash) { @@ -195,7 +197,7 @@ Queued::notify(const PacketPtr &pkt, const PrefetchInfo &pfi) // Calculate prefetches given this access std::vector addresses; - calculatePrefetch(pfi, addresses); + calculatePrefetch(pfi, addresses, cache); // Get the maximu number of prefetches that we are allowed to generate size_t max_pfs = getMaxPermittedPrefetches(addresses.size()); @@ -210,7 +212,7 @@ Queued::notify(const PacketPtr &pkt, const PrefetchInfo &pfi) if (!samePage(addr_prio.first, pfi.getAddr())) { statsQueued.pfSpanPage += 1; - if (hasBeenPrefetched(pkt->getAddr(), pkt->isSecure())) { + if (cache.hasBeenPrefetched(pkt->getAddr(), pkt->isSecure())) { statsQueued.pfUsefulSpanPage += 1; } } @@ -222,7 +224,7 @@ Queued::notify(const PacketPtr &pkt, const PrefetchInfo &pfi) DPRINTF(HWPrefetch, "Found a pf candidate addr: %#x, " "inserting into prefetch queue.\n", new_pfi.getAddr()); // Create and insert the request - insert(pkt, new_pfi, addr_prio.second); + insert(pkt, new_pfi, addr_prio.second, cache); num_pfs += 1; if (num_pfs == max_pfs) { break; @@ -298,7 +300,8 @@ Queued::processMissingTranslations(unsigned max) } void -Queued::translationComplete(DeferredPacket *dp, bool failed) +Queued::translationComplete(DeferredPacket *dp, bool failed, + const CacheAccessor &cache) { auto it = pfqMissingTranslation.begin(); while (it != pfqMissingTranslation.end()) { @@ -315,8 +318,9 @@ Queued::translationComplete(DeferredPacket *dp, bool failed) it->translationRequest->getPaddr()); Addr target_paddr = it->translationRequest->getPaddr(); // check if this prefetch is already redundant - if (cacheSnoop && (inCache(target_paddr, it->pfInfo.isSecure()) || - inMissQueue(target_paddr, it->pfInfo.isSecure()))) { + if (cacheSnoop && + (cache.inCache(target_paddr, it->pfInfo.isSecure()) || + cache.inMissQueue(target_paddr, it->pfInfo.isSecure()))) { statsQueued.pfInCache++; DPRINTF(HWPrefetch, "Dropping redundant in " "cache/MSHR prefetch addr:%#x\n", target_paddr); @@ -382,7 +386,7 @@ Queued::createPrefetchRequest(Addr addr, PrefetchInfo const &pfi, void Queued::insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, - int32_t priority) + int32_t priority, const CacheAccessor &cache) { if (queueFilter) { if (alreadyInQueue(pfq, new_pfi, priority)) { @@ -451,8 +455,8 @@ Queued::insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, } } if (has_target_pa && cacheSnoop && - (inCache(target_paddr, new_pfi.isSecure()) || - inMissQueue(target_paddr, new_pfi.isSecure()))) { + (cache.inCache(target_paddr, new_pfi.isSecure()) || + cache.inMissQueue(target_paddr, new_pfi.isSecure()))) { statsQueued.pfInCache++; DPRINTF(HWPrefetch, "Dropping redundant in " "cache/MSHR prefetch addr:%#x\n", target_paddr); @@ -460,7 +464,7 @@ Queued::insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, } /* Create the packet and find the spot to insert it */ - DeferredPacket dpp(this, new_pfi, 0, priority); + DeferredPacket dpp(this, new_pfi, 0, priority, cache); if (has_target_pa) { Tick pf_time = curTick() + clockPeriod() * latency; dpp.createPkt(target_paddr, blkSize, requestorId, tagPrefetch, @@ -472,7 +476,7 @@ Queued::insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, } else { // Add the translation request and try to resolve it later dpp.setTranslationRequest(translation_req); - dpp.tc = cache->system->threads[translation_req->contextId()]; + dpp.tc = system->threads[translation_req->contextId()]; DPRINTF(HWPrefetch, "Prefetch queued with no translation. " "addr:%#x priority: %3d\n", new_pfi.getAddr(), priority); addToQueue(pfqMissingTranslation, dpp); diff --git a/src/mem/cache/prefetch/queued.hh b/src/mem/cache/prefetch/queued.hh index 1d1a3faef4..f3620bd826 100644 --- a/src/mem/cache/prefetch/queued.hh +++ b/src/mem/cache/prefetch/queued.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2015 ARM Limited + * Copyright (c) 2014-2015, 2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -75,6 +75,7 @@ class Queued : public Base RequestPtr translationRequest; ThreadContext *tc; bool ongoingTranslation; + const CacheAccessor *cache; /** * Constructor @@ -85,9 +86,10 @@ class Queued : public Base * @param prio This prefetch priority */ DeferredPacket(Queued *o, PrefetchInfo const &pfi, Tick t, - int32_t prio) : owner(o), pfInfo(pfi), tick(t), pkt(nullptr), + int32_t prio, const CacheAccessor &_cache) + : owner(o), pfInfo(pfi), tick(t), pkt(nullptr), priority(prio), translationRequest(), tc(nullptr), - ongoingTranslation(false) { + ongoingTranslation(false), cache(&_cache) { } bool operator>(const DeferredPacket& that) const @@ -192,12 +194,15 @@ class Queued : public Base Queued(const QueuedPrefetcherParams &p); virtual ~Queued(); - void notify(const PacketPtr &pkt, const PrefetchInfo &pfi) override; + void + notify(const CacheAccessProbeArg &acc, const PrefetchInfo &pfi) override; - void insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, int32_t priority); + void insert(const PacketPtr &pkt, PrefetchInfo &new_pfi, int32_t priority, + const CacheAccessor &cache); virtual void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) = 0; + std::vector &addresses, + const CacheAccessor &cache) = 0; PacketPtr getPacket() override; Tick nextPrefetchReadyTime() const override @@ -231,8 +236,10 @@ class Queued : public Base * new prefetch request. * @param dp the deferred packet that has completed the translation request * @param failed whether the translation was successful + * @param cache accessor for lookups on the cache that originated this pkt */ - void translationComplete(DeferredPacket *dp, bool failed); + void translationComplete(DeferredPacket *dp, bool failed, + const CacheAccessor &cache); /** * Checks whether the specified prefetch request is already in the diff --git a/src/mem/cache/prefetch/sbooe.cc b/src/mem/cache/prefetch/sbooe.cc index 44a10c232d..7f4c5ee04d 100644 --- a/src/mem/cache/prefetch/sbooe.cc +++ b/src/mem/cache/prefetch/sbooe.cc @@ -91,8 +91,10 @@ SBOOE::access(Addr access_line) } void -SBOOE::notifyFill(const PacketPtr& pkt) +SBOOE::notifyFill(const CacheAccessProbeArg &arg) { + const PacketPtr& pkt = arg.pkt; + // (1) Look for the address in the demands list // (2) Calculate the elapsed cycles until it was filled (curTick) // (3) Insert the latency into the latency buffer (FIFO) @@ -117,7 +119,8 @@ SBOOE::notifyFill(const PacketPtr& pkt) void SBOOE::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { const Addr pfi_addr = pfi.getAddr(); const Addr pfi_line = pfi_addr >> lBlkSize; diff --git a/src/mem/cache/prefetch/sbooe.hh b/src/mem/cache/prefetch/sbooe.hh index 7914b88f45..26384923cf 100644 --- a/src/mem/cache/prefetch/sbooe.hh +++ b/src/mem/cache/prefetch/sbooe.hh @@ -153,13 +153,14 @@ class SBOOE : public Queued bool access(Addr line); /** Update the latency buffer after a prefetch fill */ - void notifyFill(const PacketPtr& pkt) override; + void notifyFill(const CacheAccessProbeArg &arg) override; public: SBOOE(const SBOOEPrefetcherParams &p); void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/signature_path.cc b/src/mem/cache/prefetch/signature_path.cc index a36ef809ce..6d3f05bf17 100644 --- a/src/mem/cache/prefetch/signature_path.cc +++ b/src/mem/cache/prefetch/signature_path.cc @@ -227,7 +227,8 @@ SignaturePath::calculateLookaheadConfidence(PatternEntry const &sig, void SignaturePath::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { Addr request_addr = pfi.getAddr(); Addr ppn = request_addr / pageBytes; diff --git a/src/mem/cache/prefetch/signature_path.hh b/src/mem/cache/prefetch/signature_path.hh index 9613fe0886..a561cda063 100644 --- a/src/mem/cache/prefetch/signature_path.hh +++ b/src/mem/cache/prefetch/signature_path.hh @@ -287,7 +287,8 @@ class SignaturePath : public Queued ~SignaturePath() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/slim_ampm.cc b/src/mem/cache/prefetch/slim_ampm.cc index 950994a4bd..8717d0314b 100644 --- a/src/mem/cache/prefetch/slim_ampm.cc +++ b/src/mem/cache/prefetch/slim_ampm.cc @@ -43,11 +43,12 @@ SlimAMPM::SlimAMPM(const SlimAMPMPrefetcherParams &p) void SlimAMPM::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { - dcpt.calculatePrefetch(pfi, addresses); + dcpt.calculatePrefetch(pfi, addresses, cache); if (addresses.empty()) { - ampm.calculatePrefetch(pfi, addresses); + ampm.calculatePrefetch(pfi, addresses, cache); } } diff --git a/src/mem/cache/prefetch/slim_ampm.hh b/src/mem/cache/prefetch/slim_ampm.hh index 54f38d4885..04ef6ca921 100644 --- a/src/mem/cache/prefetch/slim_ampm.hh +++ b/src/mem/cache/prefetch/slim_ampm.hh @@ -62,7 +62,8 @@ class SlimAMPM : public Queued ~SlimAMPM() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/spatio_temporal_memory_streaming.cc b/src/mem/cache/prefetch/spatio_temporal_memory_streaming.cc index 0e3211579c..406e444be0 100644 --- a/src/mem/cache/prefetch/spatio_temporal_memory_streaming.cc +++ b/src/mem/cache/prefetch/spatio_temporal_memory_streaming.cc @@ -63,7 +63,7 @@ STeMS::STeMS(const STeMSPrefetcherParams &p) } void -STeMS::checkForActiveGenerationsEnd() +STeMS::checkForActiveGenerationsEnd(const CacheAccessor &cache) { // This prefetcher operates attached to the L1 and it observes all // accesses, this guarantees that no evictions are missed @@ -79,8 +79,8 @@ STeMS::checkForActiveGenerationsEnd() if (seq_entry.counter > 0) { Addr cache_addr = agt_entry.paddress + seq_entry.offset * blkSize; - if (!inCache(cache_addr, sr_is_secure) && - !inMissQueue(cache_addr, sr_is_secure)) { + if (!cache.inCache(cache_addr, sr_is_secure) && + !cache.inMissQueue(cache_addr, sr_is_secure)) { generation_ended = true; pst_addr = (agt_entry.pc << spatialRegionSizeBits) + seq_entry.offset; @@ -135,7 +135,8 @@ STeMS::addToRMOB(Addr sr_addr, Addr pst_addr, unsigned int delta) void STeMS::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { if (!pfi.hasPC()) { DPRINTF(HWPrefetch, "Ignoring request with no PC.\n"); @@ -152,7 +153,7 @@ STeMS::calculatePrefetch(const PrefetchInfo &pfi, Addr sr_offset = (pfi.getAddr() % spatialRegionSize) / blkSize; // Check if any active generation has ended - checkForActiveGenerationsEnd(); + checkForActiveGenerationsEnd(cache); ActiveGenerationTableEntry *agt_entry = activeGenerationTable.findEntry(sr_addr, is_secure); diff --git a/src/mem/cache/prefetch/spatio_temporal_memory_streaming.hh b/src/mem/cache/prefetch/spatio_temporal_memory_streaming.hh index cdd2788104..c6cd2f72d1 100644 --- a/src/mem/cache/prefetch/spatio_temporal_memory_streaming.hh +++ b/src/mem/cache/prefetch/spatio_temporal_memory_streaming.hh @@ -181,7 +181,7 @@ class STeMS : public Queued unsigned int lastTriggerCounter; /** Checks if the active generations have ended */ - void checkForActiveGenerationsEnd(); + void checkForActiveGenerationsEnd(const CacheAccessor &cache); /** * Adds an entry to the RMOB * @param sr_addr Spatial region address @@ -206,7 +206,8 @@ class STeMS : public Queued ~STeMS() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/stride.cc b/src/mem/cache/prefetch/stride.cc index 0a77b28a1c..b2e4702f20 100644 --- a/src/mem/cache/prefetch/stride.cc +++ b/src/mem/cache/prefetch/stride.cc @@ -118,7 +118,8 @@ Stride::allocateNewContext(int context) void Stride::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { if (!pfi.hasPC()) { DPRINTF(HWPrefetch, "Ignoring request with no PC.\n"); diff --git a/src/mem/cache/prefetch/stride.hh b/src/mem/cache/prefetch/stride.hh index 7e55abea21..41cadbe7d0 100644 --- a/src/mem/cache/prefetch/stride.hh +++ b/src/mem/cache/prefetch/stride.hh @@ -160,7 +160,8 @@ class Stride : public Queued Stride(const StridePrefetcherParams &p); void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/cache/prefetch/tagged.cc b/src/mem/cache/prefetch/tagged.cc index 0d4d79b006..e34e1a692d 100644 --- a/src/mem/cache/prefetch/tagged.cc +++ b/src/mem/cache/prefetch/tagged.cc @@ -49,7 +49,8 @@ Tagged::Tagged(const TaggedPrefetcherParams &p) void Tagged::calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) + std::vector &addresses, + const CacheAccessor &cache) { Addr blkAddr = blockAddress(pfi.getAddr()); diff --git a/src/mem/cache/prefetch/tagged.hh b/src/mem/cache/prefetch/tagged.hh index 5c91f654b1..1ed11e9355 100644 --- a/src/mem/cache/prefetch/tagged.hh +++ b/src/mem/cache/prefetch/tagged.hh @@ -55,7 +55,8 @@ class Tagged : public Queued ~Tagged() = default; void calculatePrefetch(const PrefetchInfo &pfi, - std::vector &addresses) override; + std::vector &addresses, + const CacheAccessor &cache) override; }; } // namespace prefetch diff --git a/src/mem/ruby/SConscript b/src/mem/ruby/SConscript index 2eb46b0f4d..ae2e20c4a3 100644 --- a/src/mem/ruby/SConscript +++ b/src/mem/ruby/SConscript @@ -1,6 +1,6 @@ # -*- mode:python -*- -# Copyright (c) 2021 Arm Limited +# Copyright (c) 2021,2023 Arm Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -111,6 +111,7 @@ MakeInclude('structures/DirectoryMemory.hh') MakeInclude('structures/PerfectCacheMemory.hh') MakeInclude('structures/PersistentTable.hh') MakeInclude('structures/RubyPrefetcher.hh') +MakeInclude('structures/RubyPrefetcherProxy.hh') MakeInclude('structures/TBEStorage.hh') if env['CONF']['PROTOCOL'] == 'CHI': MakeInclude('structures/MN_TBEStorage.hh') diff --git a/src/mem/ruby/protocol/RubySlicc_Exports.sm b/src/mem/ruby/protocol/RubySlicc_Exports.sm index 0d2bc742f9..ca44fd3780 100644 --- a/src/mem/ruby/protocol/RubySlicc_Exports.sm +++ b/src/mem/ruby/protocol/RubySlicc_Exports.sm @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 ARM Limited + * Copyright (c) 2020-2021,2023 ARM Limited * All rights reserved. * * The license below extends only to copyright in the software and shall @@ -51,6 +51,8 @@ external_type(Addr, primitive="yes"); external_type(Cycles, primitive="yes", default="Cycles(0)"); external_type(Tick, primitive="yes", default="0"); external_type(RequestPtr, primitive="yes", default="nullptr"); +external_type(RequestorID, primitive="yes"); +external_type(prefetch::Base, primitive="yes"); structure(WriteMask, external="yes", desc="...") { void clear(); diff --git a/src/mem/ruby/protocol/RubySlicc_Types.sm b/src/mem/ruby/protocol/RubySlicc_Types.sm index 293c731c37..2206effa29 100644 --- a/src/mem/ruby/protocol/RubySlicc_Types.sm +++ b/src/mem/ruby/protocol/RubySlicc_Types.sm @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 ARM Limited + * Copyright (c) 2020-2021,2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -259,3 +259,14 @@ structure (RubyPrefetcher, external = "yes") { void observePfHit(Addr); void observePfMiss(Addr); } + +structure(RubyPrefetcherProxy, external = "yes") { + void notifyPfHit(RequestPtr, bool, DataBlock); + void notifyPfMiss(RequestPtr, bool, DataBlock); + void notifyPfFill(RequestPtr, DataBlock, bool); + void notifyPfEvict(Addr, bool, RequestorID); + void completePrefetch(Addr); + // SLICC controller must define its own regProbePoints and call + // this for every RubyPrefetcherProxy object present + void regProbePoints(); +} diff --git a/src/mem/ruby/protocol/RubySlicc_Util.sm b/src/mem/ruby/protocol/RubySlicc_Util.sm index 3079f20f23..104c7c034c 100644 --- a/src/mem/ruby/protocol/RubySlicc_Util.sm +++ b/src/mem/ruby/protocol/RubySlicc_Util.sm @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 ARM Limited + * Copyright (c) 2021,2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -60,3 +60,4 @@ Addr makeNextStrideAddress(Addr addr, int stride); structure(BoolVec, external="yes") { } int countBoolVec(BoolVec bVec); +RequestorID getRequestorID(RequestPtr req); diff --git a/src/mem/ruby/protocol/chi/CHI-cache-actions.sm b/src/mem/ruby/protocol/chi/CHI-cache-actions.sm index ca7b3e203a..07344718df 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-actions.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-actions.sm @@ -3441,7 +3441,7 @@ action(Callback_ExpressPrefetchHit, desc="") { cache.profilePrefetchHit(); peek(reqRdyPort, CHIRequestMsg) { assert(in_msg.is_local_pf); - notifyPfComplete(in_msg.addr); + pfProxy.completePrefetch(in_msg.addr); } } @@ -3452,7 +3452,7 @@ action(Callback_Miss, desc="") { assert(is_valid(tbe)); if (tbe.dataValid && tbe.is_local_pf) { assert(use_prefetcher); - notifyPfComplete(tbe.addr); + pfProxy.completePrefetch(tbe.addr); } else if (tbe.dataValid && (tbe.reqType == CHIRequestType:Load)) { DPRINTF(RubySlicc, "Read data %s\n", tbe.dataBlk); @@ -3564,7 +3564,7 @@ action(Profile_Miss, desc="") { // FIXME: this dataBlk is likely to have stale data. This should be fixed // if our prefetcher uses cached data to make prefetch decisions. - notifyPfMiss(tbe.seqReq, is_read, tbe.dataBlk); + pfProxy.notifyPfMiss(tbe.seqReq, is_read, tbe.dataBlk); } } @@ -3588,7 +3588,7 @@ action(Profile_Hit, desc="") { } else { assert(isWriteReqType(tbe.reqType)); } - notifyPfHit(tbe.seqReq, is_read, tbe.dataBlk); + pfProxy.notifyPfHit(tbe.seqReq, is_read, tbe.dataBlk); cache_entry.HWPrefetched := false; } @@ -3602,10 +3602,11 @@ action(Profile_Fill, desc="") { cache_entry.HWPrefetched := tbe.is_local_pf || (tbe.is_remote_pf && (upstream_prefetch_trains_prefetcher == false)); + cache_entry.requestor := getRequestorID(tbe.seqReq); // Prefetchers that use this info require notifications from both // demand and pf fills (unlike notifyPfHit/notifyPfMiss) - notifyPfFill(tbe.seqReq, tbe.dataBlk, tbe.is_local_pf); + pfProxy.notifyPfFill(tbe.seqReq, tbe.dataBlk, tbe.is_local_pf); } } @@ -3619,7 +3620,7 @@ action(Profile_Eviction, desc="") { sequencer.evictionCallback(address); } if (use_prefetcher && is_valid(cache_entry)) { - notifyPfEvict(address, cache_entry.HWPrefetched); + pfProxy.notifyPfEvict(address, cache_entry.HWPrefetched, cache_entry.requestor); } } diff --git a/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm b/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm index ed8358fea4..113f4c4871 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-funcs.sm @@ -68,12 +68,6 @@ void outgoingTransactionEnd(Addr, bool, bool); Event curTransitionEvent(); State curTransitionNextState(); -// Placeholders for future prefetch support -void notifyPfHit(RequestPtr req, bool is_read, DataBlock blk) { } -void notifyPfMiss(RequestPtr req, bool is_read, DataBlock blk) { } -void notifyPfFill(RequestPtr req, DataBlock blk, bool from_pf) { } -void notifyPfEvict(Addr blkAddr, bool hwPrefetched) { } -void notifyPfComplete(Addr addr) { } //////////////////////////////////////////////////////////////////////////// // Interface functions required by SLICC @@ -307,7 +301,11 @@ bool fromSequencer(CHIRequestType reqType) { reqType == CHIRequestType:AtomicStore; } -bool inCache(Addr addr) { +void regProbePoints() { + pfProxy.regProbePoints(); +} + +bool inCache(Addr addr, bool is_secure) { CacheEntry entry := getCacheEntry(makeLineAddress(addr)); // NOTE: we consider data for the addr to be in cache if it exists in local, // upstream, or both caches. @@ -318,7 +316,16 @@ bool inCache(Addr addr) { } } -bool hasBeenPrefetched(Addr addr) { +bool hasBeenPrefetched(Addr addr, bool is_secure, RequestorID requestor) { + CacheEntry entry := getCacheEntry(makeLineAddress(addr)); + if (is_valid(entry)) { + return entry.HWPrefetched && (entry.requestor == requestor); + } else { + return false; + } +} + +bool hasBeenPrefetched(Addr addr, bool is_secure) { CacheEntry entry := getCacheEntry(makeLineAddress(addr)); if (is_valid(entry)) { return entry.HWPrefetched; @@ -327,12 +334,16 @@ bool hasBeenPrefetched(Addr addr) { } } -bool inMissQueue(Addr addr) { +bool inMissQueue(Addr addr, bool is_secure) { Addr line_addr := makeLineAddress(addr); TBE tbe := getCurrentActiveTBE(line_addr); return is_valid(tbe); } +bool coalesce() { + return false; +} + void notifyCoalesced(Addr addr, RubyRequestType type, RequestPtr req, DataBlock data_blk, bool was_miss) { DPRINTF(RubySlicc, "notifyCoalesced(addr=%#x, type=%s, was_miss=%d)\n", @@ -347,9 +358,9 @@ void notifyCoalesced(Addr addr, RubyRequestType type, RequestPtr req, (type == RubyRequestType:Load_Linked) || (type == RubyRequestType:IFETCH); if (was_miss) { - notifyPfMiss(req, is_read, data_blk); + pfProxy.notifyPfMiss(req, is_read, data_blk); } else { - notifyPfHit(req, is_read, data_blk); + pfProxy.notifyPfHit(req, is_read, data_blk); } } } diff --git a/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm b/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm index 0e8c6ec0e3..1654b9bf02 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache-transitions.sm @@ -515,7 +515,7 @@ transition({UD,UD_T,SD,UC,SC}, Load, BUSY_BLKD) { // the local cache entry at the end since our data is stale. If the cache is // inclusive for unique data we need to keep the block, so just bypass the // normal path. -transition({UD,UD_T,SD,UC,SC,RU,RSC,RSD,RUSD,SC_RSC,SD_RSC,SD_RSD,UC_RSC,UC_RU,UD_RU,UD_RSD,UD_RSC}, Prefetch) { +transition({UD,UD_T,SD,UC,SC,RU,RSC,RSD,RUSC,RUSD,SC_RSC,SD_RSC,SD_RSD,UC_RSC,UC_RU,UD_RU,UD_RSD,UD_RSC}, Prefetch) { Callback_ExpressPrefetchHit; Pop_ReqRdyQueue; } diff --git a/src/mem/ruby/protocol/chi/CHI-cache.sm b/src/mem/ruby/protocol/chi/CHI-cache.sm index f806488b45..62fb2231ff 100644 --- a/src/mem/ruby/protocol/chi/CHI-cache.sm +++ b/src/mem/ruby/protocol/chi/CHI-cache.sm @@ -46,6 +46,10 @@ machine(MachineType:Cache, "Cache coherency protocol") : // happen in parallel. The cache tag latency is used for both cases. CacheMemory * cache; + // Prefetcher to insert prefetch requests + // null if the cache does not support prefetching + prefetch::Base * prefetcher; + // Additional pipeline latency modeling for the different request types // When defined, these are applied after the initial tag array read and // sending necessary snoops. @@ -568,6 +572,7 @@ machine(MachineType:Cache, "Cache coherency protocol") : State state, desc="SLICC line state"; DataBlock DataBlk, desc="data for the block"; bool HWPrefetched, default="false", desc="Set if this cache entry was prefetched"; + RequestorID requestor, desc="First requestor to fill this block"; } // Directory entry @@ -812,6 +817,10 @@ machine(MachineType:Cache, "Cache coherency protocol") : TBETable dvmSnpTBEs, template="", constructor="m_number_of_DVM_snoop_TBEs"; TBEStorage storDvmSnpTBEs, constructor="this, m_number_of_DVM_snoop_TBEs"; + // Interface to prefetchers + RubyPrefetcherProxy pfProxy, constructor="this, m_prefetcher_ptr, m_prefetchQueue_ptr"; + + // DVM data // Queue of non-sync operations that haven't been Comp-d yet. // Before a Sync operation can start, this queue must be emptied diff --git a/src/mem/ruby/slicc_interface/AbstractController.hh b/src/mem/ruby/slicc_interface/AbstractController.hh index 72b679d6cf..ce6a6972af 100644 --- a/src/mem/ruby/slicc_interface/AbstractController.hh +++ b/src/mem/ruby/slicc_interface/AbstractController.hh @@ -367,6 +367,28 @@ class AbstractController : public ClockedObject, public Consumer void wakeUpAllBuffers(); bool serviceMemoryQueue(); + /** + * Functions needed by CacheAccessor. These are implemented in SLICC, + * thus the const& for all args to match the generated code. + */ + virtual bool inCache(const Addr &addr, const bool &is_secure) + { fatal("inCache: prefetching not supported"); return false; } + + virtual bool hasBeenPrefetched(const Addr &addr, const bool &is_secure) + { fatal("hasBeenPrefetched: prefetching not supported"); return false; } + + virtual bool hasBeenPrefetched(const Addr &addr, const bool &is_secure, + const RequestorID &requestor) + { fatal("hasBeenPrefetched: prefetching not supported"); return false; } + + virtual bool inMissQueue(const Addr &addr, const bool &is_secure) + { fatal("inMissQueue: prefetching not supported"); return false; } + + virtual bool coalesce() + { fatal("coalesce: prefetching not supported"); return false; } + + friend class RubyPrefetcherProxy; + protected: const NodeID m_version; MachineID m_machineID; diff --git a/src/mem/ruby/slicc_interface/RubySlicc_Util.hh b/src/mem/ruby/slicc_interface/RubySlicc_Util.hh index edfbe4eea5..8df56c7013 100644 --- a/src/mem/ruby/slicc_interface/RubySlicc_Util.hh +++ b/src/mem/ruby/slicc_interface/RubySlicc_Util.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 ARM Limited + * Copyright (c) 2020-2021,2023 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -316,6 +316,12 @@ countBoolVec(BoolVec bVec) return count; } +inline RequestorID +getRequestorID(RequestPtr req) +{ + return req->requestorId(); +} + } // namespace ruby } // namespace gem5 diff --git a/src/mem/ruby/structures/RubyPrefetcherProxy.cc b/src/mem/ruby/structures/RubyPrefetcherProxy.cc new file mode 100644 index 0000000000..2a29fbc88e --- /dev/null +++ b/src/mem/ruby/structures/RubyPrefetcherProxy.cc @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2023 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "mem/ruby/structures/RubyPrefetcherProxy.hh" + +#include "debug/HWPrefetch.hh" +#include "mem/ruby/system/RubySystem.hh" + +namespace gem5 +{ + +namespace ruby +{ + +RubyPrefetcherProxy::RubyPrefetcherProxy(AbstractController* _parent, + prefetch::Base* _prefetcher, + MessageBuffer *_pf_queue) + :Named(_parent->name()), + prefetcher(_prefetcher), + cacheCntrl(_parent), + pfQueue(_pf_queue), + pfEvent([this]{ issuePrefetch(); }, name()), + ppHit(nullptr), ppMiss(nullptr), + ppFill(nullptr), ppDataUpdate(nullptr) +{ + fatal_if(!cacheCntrl, + "initializing a RubyPrefetcherProxy without a parent"); + if (prefetcher) { + fatal_if(!pfQueue, + "%s initializing a RubyPrefetcherProxy without a prefetch queue", + name()); + prefetcher->setParentInfo( + cacheCntrl->params().system, + cacheCntrl->getProbeManager(), + RubySystem::getBlockSizeBytes()); + } +} + +void +RubyPrefetcherProxy::scheduleNextPrefetch() +{ + if (pfEvent.scheduled()) + return; + + Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), + cacheCntrl->clockEdge(Cycles(1))); + if (next_pf_time != MaxTick) { + DPRINTF(HWPrefetch, "Next prefetch ready at %d\n", next_pf_time); + cacheCntrl->schedule(&pfEvent, next_pf_time); + } +} + +void +RubyPrefetcherProxy::deschedulePrefetch() +{ + if (pfEvent.scheduled()) + cacheCntrl->deschedule(&pfEvent); +} + +void +RubyPrefetcherProxy::completePrefetch(Addr addr) +{ + assert(makeLineAddress(addr) == addr); + assert(issuedPfPkts.count(addr) == 1); + DPRINTF(HWPrefetch, "Prefetch request for addr %#x completed\n", addr); + delete issuedPfPkts[addr]; + issuedPfPkts.erase(addr); +} + +void +RubyPrefetcherProxy::issuePrefetch() +{ + assert(prefetcher); + assert(pfQueue); + + if (pfQueue->areNSlotsAvailable(1, curTick())) { + PacketPtr pkt = prefetcher->getPacket(); + + if (pkt) { + DPRINTF(HWPrefetch, "Next prefetch ready %s\n", pkt->print()); + unsigned blk_size = RubySystem::getBlockSizeBytes(); + Addr line_addr = pkt->getBlockAddr(blk_size); + + if (issuedPfPkts.count(line_addr) == 0) { + DPRINTF(HWPrefetch, "Issued PF request for paddr=%#x, " + "line_addr=%#x, is_write=%d\n", + pkt->getAddr(), line_addr, + pkt->needsWritable()); + + RubyRequestType req_type = pkt->needsWritable() ? + RubyRequestType_ST : RubyRequestType_LD; + + std::shared_ptr msg = + std::make_shared(cacheCntrl->clockEdge(), + pkt->getAddr(), + blk_size, + 0, // pc + req_type, + RubyAccessMode_Supervisor, + pkt, + PrefetchBit_Yes); + + // enqueue request into prefetch queue to the cache + pfQueue->enqueue(msg, cacheCntrl->clockEdge(), + cacheCntrl->cyclesToTicks(Cycles(1))); + + // track all pending PF requests + issuedPfPkts[line_addr] = pkt; + } else { + DPRINTF(HWPrefetch, "Aborted PF request for address being " + "prefetched\n"); + delete pkt; + } + } + } else { + DPRINTF(HWPrefetch, "No prefetch slots are available\n"); + } + + scheduleNextPrefetch(); +} + +void +RubyPrefetcherProxy::notifyPfHit(const RequestPtr& req, bool is_read, + const DataBlock& data_blk) +{ + assert(ppHit); + assert(req); + Packet pkt(req, is_read ? Packet::makeReadCmd(req) : + Packet::makeWriteCmd(req)); + // NOTE: for now we only communicate physical address with prefetchers + pkt.dataStaticConst(data_blk.getData(getOffset(req->getPaddr()), + pkt.getSize())); + DPRINTF(HWPrefetch, "notify hit: %s\n", pkt.print()); + ppHit->notify(CacheAccessProbeArg(&pkt, *this)); + scheduleNextPrefetch(); +} + +void +RubyPrefetcherProxy::notifyPfMiss(const RequestPtr& req, bool is_read, + const DataBlock& data_blk) +{ + assert(ppMiss); + assert(req); + Packet pkt(req, is_read ? Packet::makeReadCmd(req) : + Packet::makeWriteCmd(req)); + // NOTE: for now we only communicate physical address with prefetchers + pkt.dataStaticConst(data_blk.getData(getOffset(req->getPaddr()), + pkt.getSize())); + DPRINTF(HWPrefetch, "notify miss: %s\n", pkt.print()); + ppMiss->notify(CacheAccessProbeArg(&pkt, *this)); + scheduleNextPrefetch(); +} + +void +RubyPrefetcherProxy::notifyPfFill(const RequestPtr& req, + const DataBlock& data_blk, + bool from_pf) +{ + assert(ppFill); + assert(req); + Packet pkt(req, Packet::makeReadCmd(req)); + if (from_pf) + pkt.cmd = Packet::Command::HardPFReq; + // NOTE: for now we only communicate physical address with prefetchers + pkt.dataStaticConst(data_blk.getData(getOffset(req->getPaddr()), + pkt.getSize())); + DPRINTF(HWPrefetch, "notify fill: %s\n", pkt.print()); + ppFill->notify(CacheAccessProbeArg(&pkt, *this)); + scheduleNextPrefetch(); +} + +void +RubyPrefetcherProxy::notifyPfEvict(Addr blkAddr, bool hwPrefetched, + RequestorID requestorID) +{ + DPRINTF(HWPrefetch, "notify evict: %#x hw_pf=%d\n", blkAddr, hwPrefetched); + CacheDataUpdateProbeArg data_update( + blkAddr, false, requestorID, *this); + data_update.hwPrefetched = hwPrefetched; + ppDataUpdate->notify(data_update); + scheduleNextPrefetch(); +} + +void +RubyPrefetcherProxy::regProbePoints() +{ + assert(cacheCntrl); + ppHit = new ProbePointArg( + cacheCntrl->getProbeManager(), "Hit"); + ppMiss = new ProbePointArg( + cacheCntrl->getProbeManager(), "Miss"); + ppFill = new ProbePointArg( + cacheCntrl->getProbeManager(), "Fill"); + ppDataUpdate = + new ProbePointArg( + cacheCntrl->getProbeManager(), "Data Update"); +} + +} // namespace ruby +} // namespace gem5 diff --git a/src/mem/ruby/structures/RubyPrefetcherProxy.hh b/src/mem/ruby/structures/RubyPrefetcherProxy.hh new file mode 100644 index 0000000000..34c40154b6 --- /dev/null +++ b/src/mem/ruby/structures/RubyPrefetcherProxy.hh @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023 ARM Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __MEM_RUBY_STRUCTURES_RUBY_PREFETCHER_WRAPPER_HH__ +#define __MEM_RUBY_STRUCTURES_RUBY_PREFETCHER_WRAPPER_HH__ + +#include + +#include "mem/cache/cache_probe_arg.hh" +#include "mem/cache/prefetch/base.hh" +#include "mem/ruby/slicc_interface/AbstractController.hh" +#include "mem/ruby/slicc_interface/RubyRequest.hh" + +namespace gem5 +{ + +namespace ruby +{ + +/** + * This is a proxy for prefetcher class in classic memory. This wrapper + * enables a SLICC machine to interact with classic prefetchers. + * + * The expected use case for this class is to instantiate it in the SLICC + * state machine definition and provide a pointer to the prefetcher object + * (typically defined in SLICC as one of the SM's configuration parameters) + * and the prefetch queue where prefetch requests will be inserted. + * + * The SLICC SM can them use the notifyPF* functions to notify the prefetcher. + * + * Notes: + * + * This object's regProbePoints() must be called explicitly. The SLICC SM may + * defined it's own regProbePoints() to call it. + * + * completePrefetch(Addr) must be called when a request injected into the + * prefetch queue is completed. + * + * A nullptr prefetcher can be provided, in which case the notifyPf* are + * no-ops. + * + */ +class RubyPrefetcherProxy : public CacheAccessor, public Named +{ + public: + + RubyPrefetcherProxy(AbstractController* parent, + prefetch::Base* prefetcher, + MessageBuffer *pf_queue); + + /** Deschedled the ready prefetch event */ + void deschedulePrefetch(); + + /** Notifies a completed prefetch request */ + void completePrefetch(Addr addr); + + /** + * Notify PF probes hit/miss/fill + */ + void notifyPfHit(const RequestPtr& req, bool is_read, + const DataBlock& data_blk); + void notifyPfMiss(const RequestPtr& req, bool is_read, + const DataBlock& data_blk); + void notifyPfFill(const RequestPtr& req, const DataBlock& data_blk, + bool from_pf); + void notifyPfEvict(Addr blkAddr, bool hwPrefetched, + RequestorID requestorID); + + /** Registers probes. */ + void regProbePoints(); + + private: + + /** Schedule the next ready prefetch */ + void scheduleNextPrefetch(); + + /** Issue prefetch to the contoller prefetch queue */ + void issuePrefetch(); + + /** Prefetcher from classic memory */ + prefetch::Base* prefetcher; + + /** Ruby cache controller */ + AbstractController* cacheCntrl; + + /** Prefetch queue to the cache controller */ + MessageBuffer* pfQueue; + + /** List of issued prefetch request packets */ + std::unordered_map issuedPfPkts; + + /** Prefetch event */ + EventFunctionWrapper pfEvent; + + /** To probe when a cache hit occurs */ + ProbePointArg *ppHit; + + /** To probe when a cache miss occurs */ + ProbePointArg *ppMiss; + + /** To probe when a cache fill occurs */ + ProbePointArg *ppFill; + + /** + * To probe when the contents of a block are updated. Content updates + * include data fills, overwrites, and invalidations, which means that + * this probe partially overlaps with other probes. + */ + ProbePointArg *ppDataUpdate; + + public: + + /** Accessor functions */ + + bool inCache(Addr addr, bool is_secure) const override + { + return cacheCntrl->inCache(addr, is_secure); + } + + bool hasBeenPrefetched(Addr addr, bool is_secure) const override + { + return cacheCntrl->hasBeenPrefetched(addr, is_secure); + } + + bool hasBeenPrefetched(Addr addr, bool is_secure, + RequestorID requestor) const override + { + return cacheCntrl->hasBeenPrefetched(addr, is_secure, requestor); + } + + bool inMissQueue(Addr addr, bool is_secure) const override + { + return cacheCntrl->inMissQueue(addr, is_secure); + } + + bool coalesce() const override + { return cacheCntrl->coalesce(); } + +}; + +} // namespace ruby +} // namespace gem5 + +#endif // __MEM_RUBY_STRUCTURES_RUBY_PREFETCHER_WRAPPER_HH__ diff --git a/src/mem/ruby/structures/SConscript b/src/mem/ruby/structures/SConscript index 242a72e919..fb7b99f265 100644 --- a/src/mem/ruby/structures/SConscript +++ b/src/mem/ruby/structures/SConscript @@ -1,6 +1,6 @@ # -*- mode:python -*- -# Copyright (c) 2021 ARM Limited +# Copyright (c) 2021,2023 ARM Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -53,6 +53,7 @@ Source('CacheMemory.cc') Source('WireBuffer.cc') Source('PersistentTable.cc') Source('RubyPrefetcher.cc') +Source('RubyPrefetcherProxy.cc') Source('TimerTable.cc') Source('BankedArray.cc') Source('ALUFreeListArray.cc') diff --git a/src/mem/slicc/symbols/StateMachine.py b/src/mem/slicc/symbols/StateMachine.py index 078e666d26..b523522501 100644 --- a/src/mem/slicc/symbols/StateMachine.py +++ b/src/mem/slicc/symbols/StateMachine.py @@ -63,6 +63,7 @@ python_class_map = { "MessageBuffer": "MessageBuffer", "DMASequencer": "DMASequencer", "RubyPrefetcher": "RubyPrefetcher", + "prefetch::Base": "BasePrefetcher", "Cycles": "Cycles", "Addr": "Addr", }