diff --git a/src/arch/arm/tlb.cc b/src/arch/arm/tlb.cc index f22401e74f..e2979f5c7c 100644 --- a/src/arch/arm/tlb.cc +++ b/src/arch/arm/tlb.cc @@ -315,19 +315,13 @@ TLB::flushAll() } void -TLB::flush(const TLBIALL& tlbi_op) +TLB::flush(const TLBIOp& tlbi_op) { - DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n", - (tlbi_op.secureLookup ? "secure" : "non-secure")); int x = 0; TlbEntry *te; while (x < size) { te = &table[x]; - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - if (te->valid && tlbi_op.secureLookup == !te->nstid && - (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) { - + if (tlbi_op.match(te, vmid)) { DPRINTF(TLB, " - %s\n", te->print()); te->valid = false; stats.flushedEntries++; @@ -338,295 +332,6 @@ TLB::flush(const TLBIALL& tlbi_op) stats.flushTlb++; } -void -TLB::flush(const ITLBIALL& tlbi_op) -{ - DPRINTF(TLB, "Flushing all ITLB entries (%s lookup)\n", - (tlbi_op.secureLookup ? "secure" : "non-secure")); - int x = 0; - TlbEntry *te; - while (x < size) { - te = &table[x]; - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - if (te->type & TypeTLB::instruction && te->valid && - tlbi_op.secureLookup == !te->nstid && - (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) { - - DPRINTF(TLB, " - %s\n", te->print()); - te->valid = false; - stats.flushedEntries++; - } - ++x; - } - - stats.flushTlb++; -} - -void -TLB::flush(const DTLBIALL& tlbi_op) -{ - DPRINTF(TLB, "Flushing all DTLB entries (%s lookup)\n", - (tlbi_op.secureLookup ? "secure" : "non-secure")); - int x = 0; - TlbEntry *te; - while (x < size) { - te = &table[x]; - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - if (te->type & TypeTLB::data && te->valid && - tlbi_op.secureLookup == !te->nstid && - (te->vmid == vmid || tlbi_op.el2Enabled) && el_match) { - - DPRINTF(TLB, " - %s\n", te->print()); - te->valid = false; - stats.flushedEntries++; - } - ++x; - } - - stats.flushTlb++; -} - -void -TLB::flush(const TLBIALLEL &tlbi_op) -{ - DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n", - (tlbi_op.secureLookup ? "secure" : "non-secure")); - int x = 0; - TlbEntry *te; - while (x < size) { - te = &table[x]; - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - if (te->valid && tlbi_op.secureLookup == !te->nstid && el_match) { - - DPRINTF(TLB, " - %s\n", te->print()); - te->valid = false; - stats.flushedEntries++; - } - ++x; - } - - stats.flushTlb++; -} - -void -TLB::flush(const TLBIVMALL &tlbi_op) -{ - DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n", - (tlbi_op.secureLookup ? "secure" : "non-secure")); - int x = 0; - TlbEntry *te; - while (x < size) { - te = &table[x]; - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - - const bool vmid_match = - te->vmid == vmid || - !tlbi_op.el2Enabled || - (!tlbi_op.stage2Flush() && tlbi_op.inHost); - - if (te->valid && tlbi_op.secureLookup == !te->nstid && - el_match && vmid_match) { - - DPRINTF(TLB, " - %s\n", te->print()); - te->valid = false; - stats.flushedEntries++; - } - ++x; - } - - stats.flushTlb++; -} - -void -TLB::flush(const TLBIALLN &tlbi_op) -{ - bool hyp = tlbi_op.targetEL == EL2; - - DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n", - (hyp ? "hyp" : "non-hyp")); - int x = 0; - TlbEntry *te; - while (x < size) { - te = &table[x]; - const bool el_match = te->checkELMatch(tlbi_op.targetEL, false); - - if (te->valid && te->nstid && te->isHyp == hyp && el_match) { - - DPRINTF(TLB, " - %s\n", te->print()); - stats.flushedEntries++; - te->valid = false; - } - ++x; - } - - stats.flushTlb++; -} - -void -TLB::flush(const TLBIMVA &tlbi_op) -{ - DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x " - "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid, - (tlbi_op.secureLookup ? "secure" : "non-secure")); - _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false, - tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified); - stats.flushTlbMvaAsid++; -} - -void -TLB::flush(const ITLBIMVA &tlbi_op) -{ - DPRINTF(TLB, "Flushing ITLB entries with mva: %#x, asid: %#x " - "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid, - (tlbi_op.secureLookup ? "secure" : "non-secure")); - _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false, - tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::instruction); - stats.flushTlbMvaAsid++; -} - -void -TLB::flush(const DTLBIMVA &tlbi_op) -{ - DPRINTF(TLB, "Flushing DTLB entries with mva: %#x, asid: %#x " - "(%s lookup)\n", tlbi_op.addr, tlbi_op.asid, - (tlbi_op.secureLookup ? "secure" : "non-secure")); - _flushMva(tlbi_op.addr, tlbi_op.asid, tlbi_op.secureLookup, false, - tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::data); - stats.flushTlbMvaAsid++; -} - -void -TLB::flush(const TLBIASID &tlbi_op) -{ - DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n", - tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure")); - - int x = 0 ; - TlbEntry *te; - - while (x < size) { - te = &table[x]; - - const bool el_match = te->checkELMatch( - tlbi_op.targetEL, tlbi_op.inHost); - - const bool vmid_match = - te->vmid == vmid || !tlbi_op.el2Enabled || tlbi_op.inHost; - - if (te->valid && te->asid == tlbi_op.asid && - tlbi_op.secureLookup == !te->nstid && - vmid_match && el_match) { - - te->valid = false; - DPRINTF(TLB, " - %s\n", te->print()); - stats.flushedEntries++; - } - ++x; - } - stats.flushTlbAsid++; -} - -void -TLB::flush(const ITLBIASID &tlbi_op) -{ - DPRINTF(TLB, "Flushing ITLB entries with asid: %#x (%s lookup)\n", - tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure")); - - int x = 0 ; - TlbEntry *te; - - while (x < size) { - te = &table[x]; - if (te->type & TypeTLB::instruction && - te->valid && te->asid == tlbi_op.asid && - tlbi_op.secureLookup == !te->nstid && - (te->vmid == vmid || tlbi_op.el2Enabled) && - te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) { - - te->valid = false; - DPRINTF(TLB, " - %s\n", te->print()); - stats.flushedEntries++; - } - ++x; - } - stats.flushTlbAsid++; -} - -void -TLB::flush(const DTLBIASID &tlbi_op) -{ - DPRINTF(TLB, "Flushing DTLB entries with asid: %#x (%s lookup)\n", - tlbi_op.asid, (tlbi_op.secureLookup ? "secure" : "non-secure")); - - int x = 0 ; - TlbEntry *te; - - while (x < size) { - te = &table[x]; - if (te->type & TypeTLB::data && - te->valid && te->asid == tlbi_op.asid && - tlbi_op.secureLookup == !te->nstid && - (te->vmid == vmid || tlbi_op.el2Enabled) && - te->checkELMatch(tlbi_op.targetEL, tlbi_op.inHost)) { - - te->valid = false; - DPRINTF(TLB, " - %s\n", te->print()); - stats.flushedEntries++; - } - ++x; - } - stats.flushTlbAsid++; -} - -void -TLB::flush(const TLBIMVAA &tlbi_op) { - - DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n", - tlbi_op.addr, - (tlbi_op.secureLookup ? "secure" : "non-secure")); - _flushMva(tlbi_op.addr, 0xbeef, tlbi_op.secureLookup, true, - tlbi_op.targetEL, tlbi_op.inHost, TypeTLB::unified); - stats.flushTlbMva++; -} - -void -TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup, - bool ignore_asn, ExceptionLevel target_el, bool in_host, - TypeTLB entry_type) -{ - int x = 0; - TlbEntry *te; - Lookup lookup_data; - - lookup_data.va = sext<56>(mva); - lookup_data.asn = asn; - lookup_data.ignoreAsn = ignore_asn; - lookup_data.vmid = vmid; - lookup_data.hyp = target_el == EL2; - lookup_data.secure = secure_lookup; - lookup_data.functional = true; - lookup_data.targetEL = target_el; - lookup_data.inHost = in_host; - lookup_data.mode = BaseMMU::Read; - - while (x < size) { - if (table[x].match(lookup_data)) { - te = &table[x]; - bool matching_type = (te->type & entry_type); - if (matching_type && secure_lookup == !te->nstid) { - DPRINTF(TLB, " - %s\n", te->print()); - te->valid = false; - stats.flushedEntries++; - } - } - ++x; - } -} - void TLB::takeOverFrom(BaseTLB *_otlb) { @@ -645,13 +350,7 @@ TLB::TlbStats::TlbStats(TLB &parent) ADD_STAT(inserts, statistics::units::Count::get(), "Number of times an entry is inserted into the TLB"), ADD_STAT(flushTlb, statistics::units::Count::get(), - "Number of times complete TLB was flushed"), - ADD_STAT(flushTlbMva, statistics::units::Count::get(), - "Number of times TLB was flushed by MVA"), - ADD_STAT(flushTlbMvaAsid, statistics::units::Count::get(), - "Number of times TLB was flushed by MVA & ASID"), - ADD_STAT(flushTlbAsid, statistics::units::Count::get(), - "Number of times TLB was flushed by ASID"), + "Number of times a TLB invalidation was requested"), ADD_STAT(flushedEntries, statistics::units::Count::get(), "Number of entries that have been flushed from TLB"), ADD_STAT(readAccesses, statistics::units::Count::get(), "Read accesses", diff --git a/src/arch/arm/tlb.hh b/src/arch/arm/tlb.hh index 40ad76ca76..fc9b68f0be 100644 --- a/src/arch/arm/tlb.hh +++ b/src/arch/arm/tlb.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2013, 2016, 2019-2021 Arm Limited + * Copyright (c) 2010-2013, 2016, 2019-2022 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -61,20 +61,7 @@ namespace ArmISA { class TableWalker; class TLB; - -class TLBIALL; -class ITLBIALL; -class DTLBIALL; -class TLBIALLEL; -class TLBIVMALL; -class TLBIALLN; -class TLBIMVA; -class ITLBIMVA; -class DTLBIMVA; -class TLBIASID; -class ITLBIASID; -class DTLBIASID; -class TLBIMVAA; +class TLBIOp; class TlbTestInterface { @@ -153,9 +140,6 @@ class TLB : public BaseTLB mutable statistics::Scalar writeMisses; mutable statistics::Scalar inserts; mutable statistics::Scalar flushTlb; - mutable statistics::Scalar flushTlbMva; - mutable statistics::Scalar flushTlbMvaAsid; - mutable statistics::Scalar flushTlbAsid; mutable statistics::Scalar flushedEntries; statistics::Formula readAccesses; @@ -219,42 +203,9 @@ class TLB : public BaseTLB void flushAll() override; - /** Reset the entire TLB + /** Flush TLB entries */ - void flush(const TLBIALL &tlbi_op); - void flush(const ITLBIALL &tlbi_op); - void flush(const DTLBIALL &tlbi_op); - - /** Implementaton of AArch64 TLBI ALLE1(IS), ALLE2(IS), ALLE3(IS) - * instructions - */ - void flush(const TLBIALLEL &tlbi_op); - - /** Implementaton of AArch64 TLBI VMALLE1(IS)/VMALLS112E1(IS) - * instructions - */ - void flush(const TLBIVMALL &tlbi_op); - - /** Remove all entries in the non secure world, depending on whether they - * were allocated in hyp mode or not - */ - void flush(const TLBIALLN &tlbi_op); - - /** Remove any entries that match both a va and asn - */ - void flush(const TLBIMVA &tlbi_op); - void flush(const ITLBIMVA &tlbi_op); - void flush(const DTLBIMVA &tlbi_op); - - /** Remove any entries that match the asn - */ - void flush(const TLBIASID &tlbi_op); - void flush(const ITLBIASID &tlbi_op); - void flush(const DTLBIASID &tlbi_op); - - /** Remove all entries that match the va regardless of asn - */ - void flush(const TLBIMVAA &tlbi_op); + void flush(const TLBIOp &tlbi_op); Fault trickBoxCheck(const RequestPtr &req, BaseMMU::Mode mode, TlbEntry::DomainType domain); diff --git a/src/arch/arm/tlbi_op.cc b/src/arch/arm/tlbi_op.cc index 5c595a8ebb..b718ff554e 100644 --- a/src/arch/arm/tlbi_op.cc +++ b/src/arch/arm/tlbi_op.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited + * Copyright (c) 2018-2022 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -62,6 +62,14 @@ TLBIALL::operator()(ThreadContext* tc) } } +bool +TLBIALL::match(TlbEntry* te, vmid_t vmid) const +{ + return te->valid && secureLookup == !te->nstid && + (te->vmid == vmid || el2Enabled) && + te->checkELMatch(targetEL, inHost); +} + void ITLBIALL::operator()(ThreadContext* tc) { @@ -69,6 +77,12 @@ ITLBIALL::operator()(ThreadContext* tc) getMMUPtr(tc)->iflush(*this); } +bool +ITLBIALL::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIALL::match(te, vmid) && (te->type & TypeTLB::instruction); +} + void DTLBIALL::operator()(ThreadContext* tc) { @@ -76,6 +90,12 @@ DTLBIALL::operator()(ThreadContext* tc) getMMUPtr(tc)->dflush(*this); } +bool +DTLBIALL::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIALL::match(te, vmid) && (te->type & TypeTLB::data); +} + void TLBIALLEL::operator()(ThreadContext* tc) { @@ -90,6 +110,13 @@ TLBIALLEL::operator()(ThreadContext* tc) } } +bool +TLBIALLEL::match(TlbEntry* te, vmid_t vmid) const +{ + return te->valid && secureLookup == !te->nstid && + te->checkELMatch(targetEL, inHost); +} + void TLBIVMALL::operator()(ThreadContext* tc) { @@ -106,6 +133,14 @@ TLBIVMALL::operator()(ThreadContext* tc) } } +bool +TLBIVMALL::match(TlbEntry* te, vmid_t vmid) const +{ + return te->valid && secureLookup == !te->nstid && + te->checkELMatch(targetEL, inHost) && + (te->vmid == vmid || !el2Enabled || (!stage2Flush() && inHost)); +} + void TLBIASID::operator()(ThreadContext* tc) { @@ -120,6 +155,15 @@ TLBIASID::operator()(ThreadContext* tc) } } +bool +TLBIASID::match(TlbEntry* te, vmid_t vmid) const +{ + return te->valid && te->asid == asid && + secureLookup == !te->nstid && + te->checkELMatch(targetEL, inHost) && + (te->vmid == vmid || !el2Enabled || inHost); +} + void ITLBIASID::operator()(ThreadContext* tc) { @@ -127,6 +171,12 @@ ITLBIASID::operator()(ThreadContext* tc) getMMUPtr(tc)->iflush(*this); } +bool +ITLBIASID::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIASID::match(te, vmid) && (te->type & TypeTLB::instruction); +} + void DTLBIASID::operator()(ThreadContext* tc) { @@ -134,6 +184,12 @@ DTLBIASID::operator()(ThreadContext* tc) getMMUPtr(tc)->dflush(*this); } +bool +DTLBIASID::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIASID::match(te, vmid) && (te->type & TypeTLB::data); +} + void TLBIALLN::operator()(ThreadContext* tc) { @@ -145,6 +201,14 @@ TLBIALLN::operator()(ThreadContext* tc) } } +bool +TLBIALLN::match(TlbEntry* te, vmid_t vmid) const +{ + return te->valid && te->nstid && + te->isHyp == (targetEL == EL2) && + te->checkELMatch(targetEL, false); +} + void TLBIMVAA::operator()(ThreadContext* tc) { @@ -158,6 +222,23 @@ TLBIMVAA::operator()(ThreadContext* tc) } } +bool +TLBIMVAA::match(TlbEntry* te, vmid_t vmid) const +{ + TlbEntry::Lookup lookup_data; + lookup_data.va = sext<56>(addr); + lookup_data.ignoreAsn = true; + lookup_data.vmid = vmid; + lookup_data.hyp = targetEL == EL2; + lookup_data.secure = secureLookup; + lookup_data.functional = true; + lookup_data.targetEL = targetEL; + lookup_data.inHost = inHost; + lookup_data.mode = BaseMMU::Read; + + return te->match(lookup_data); +} + void TLBIMVA::operator()(ThreadContext* tc) { @@ -171,18 +252,48 @@ TLBIMVA::operator()(ThreadContext* tc) } } +bool +TLBIMVA::match(TlbEntry* te, vmid_t vmid) const +{ + TlbEntry::Lookup lookup_data; + lookup_data.va = sext<56>(addr); + lookup_data.asn = asid; + lookup_data.ignoreAsn = false; + lookup_data.vmid = vmid; + lookup_data.hyp = targetEL == EL2; + lookup_data.secure = secureLookup; + lookup_data.functional = true; + lookup_data.targetEL = targetEL; + lookup_data.inHost = inHost; + lookup_data.mode = BaseMMU::Read; + + return te->match(lookup_data); +} + void ITLBIMVA::operator()(ThreadContext* tc) { getMMUPtr(tc)->iflush(*this); } +bool +ITLBIMVA::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIMVA::match(te, vmid) && (te->type & TypeTLB::instruction); +} + void DTLBIMVA::operator()(ThreadContext* tc) { getMMUPtr(tc)->dflush(*this); } +bool +DTLBIMVA::match(TlbEntry* te, vmid_t vmid) const +{ + return TLBIMVA::match(te, vmid) && (te->type & TypeTLB::data); +} + void TLBIIPA::operator()(ThreadContext* tc) { diff --git a/src/arch/arm/tlbi_op.hh b/src/arch/arm/tlbi_op.hh index 0151ff92f7..7fcf8e3e74 100644 --- a/src/arch/arm/tlbi_op.hh +++ b/src/arch/arm/tlbi_op.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020 ARM Limited + * Copyright (c) 2018-2020, 2022 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -76,6 +76,8 @@ class TLBIOp (*this)(oc); } + virtual bool match(TlbEntry *entry, vmid_t curr_vmid) const = 0; + /** * Return true if the TLBI op needs to flush stage1 * entries, Defaulting to true in the TLBIOp abstract @@ -113,6 +115,8 @@ class TLBIALL : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool stage2Flush() const override { @@ -143,6 +147,8 @@ class ITLBIALL : public TLBIALL void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate All */ @@ -156,6 +162,8 @@ class DTLBIALL : public TLBIALL void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Implementaton of AArch64 TLBI ALLE(1,2,3)(IS) instructions */ @@ -168,6 +176,8 @@ class TLBIALLEL : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool stage2Flush() const override { @@ -195,6 +205,8 @@ class TLBIVMALL : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool stage2Flush() const override { @@ -223,6 +235,8 @@ class TLBIASID : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + uint16_t asid; bool inHost; bool el2Enabled; @@ -239,6 +253,8 @@ class ITLBIASID : public TLBIASID void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate by ASID match */ @@ -252,6 +268,8 @@ class DTLBIASID : public TLBIASID void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** TLB Invalidate All, Non-Secure */ @@ -264,6 +282,8 @@ class TLBIALLN : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool stage2Flush() const override { @@ -288,6 +308,8 @@ class TLBIMVAA : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + Addr addr; bool inHost; }; @@ -304,6 +326,8 @@ class TLBIMVA : public TLBIOp void operator()(ThreadContext* tc) override; + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + Addr addr; uint16_t asid; bool inHost; @@ -321,6 +345,8 @@ class ITLBIMVA : public TLBIMVA void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate by VA */ @@ -335,6 +361,8 @@ class DTLBIMVA : public TLBIMVA void broadcast(ThreadContext *tc) = delete; void operator()(ThreadContext* tc) override; + + bool match(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** TLB Invalidate by Intermediate Physical Address */ @@ -347,6 +375,12 @@ class TLBIIPA : public TLBIOp void operator()(ThreadContext* tc) override; + bool + match(TlbEntry *entry, vmid_t curr_vmid) const override + { + panic("This shouldn't be called\n"); + } + bool stage1Flush() const override {