From c4ed23a10b51f2f4f3c40fac060e8b34f5b18848 Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Wed, 29 May 2024 11:54:24 +0100 Subject: [PATCH] arch-arm: Implement HCR_EL2 force broadcast for EL1&0 TLBIs (#1175) According to the Arm architecture reference manual, it is possible to force the broadcast of the following TLBIs: AArch64: TLBI VMALLE1, TLBI VAE1, TLBI ASIDE1, TLBI VAAE1, TLBI VALE1, TLBI VAALE1, IC IALLU, TLBI RVAE1, TLBI RVAAE1, TLBI RVALE1, and TLBI RVAALE1. AArch32: BPIALL, TLBIALL, TLBIMVA, TLBIASID, DTLBIALL, DTLBIMVA, DTLBIASID, ITLBIALL, ITLBIMVA, ITLBIASID, TLBIMVAA, ICIALLU, TLBIMVAL, and TLBIMVAAL. Via the HCR_EL2.FB bit Change-Id: Ib11aa05cd202fadfbd9221db7a2043051196ecbd Signed-off-by: Giacomo Travaglini --- src/arch/arm/insts/misc.cc | 132 ++++++++++++++++++++++++--- src/arch/arm/insts/misc64.cc | 63 +++++++++++-- src/arch/arm/isa/formats/aarch64.isa | 4 +- src/arch/arm/isa/insts/data64.isa | 36 +++++--- src/arch/arm/tlbi_op.hh | 14 +-- 5 files changed, 201 insertions(+), 48 deletions(-) diff --git a/src/arch/arm/insts/misc.cc b/src/arch/arm/insts/misc.cc index 9475a6c1c3..78d489ec12 100644 --- a/src/arch/arm/insts/misc.cc +++ b/src/arch/arm/insts/misc.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2012-2013, 2017-2018, 2021, 2023 Arm Limited + * Copyright (c) 2010, 2012-2013, 2017-2018, 2021, 2023-2024 Arm Limited * Copyright (c) 2013 Advanced Micro Devices, Inc. * All rights reserved * @@ -408,11 +408,20 @@ TlbiOp::performTlbi(ExecContext *xc, MiscRegIndex dest_idx, RegVal value) const switch (dest_idx) { case MISCREG_TLBIALL: // TLBI all entries, EL0&1, { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TLBIALL tlbiOp(TranslationRegime::EL10, secure); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate All, Inner Shareable @@ -428,51 +437,87 @@ TlbiOp::performTlbi(ExecContext *xc, MiscRegIndex dest_idx, RegVal value) const // Instruction TLB Invalidate All case MISCREG_ITLBIALL: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + ITLBIALL tlbiOp(TranslationRegime::EL10, secure); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // Data TLB Invalidate All case MISCREG_DTLBIALL: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + DTLBIALL tlbiOp(TranslationRegime::EL10, secure); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by VA case MISCREG_TLBIMVA: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TLBIMVA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), bits(value, 7, 0), false); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by VA, Last Level case MISCREG_TLBIMVAL: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TLBIMVA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), bits(value, 7, 0), true); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by VA, Inner Shareable @@ -508,14 +553,23 @@ TlbiOp::performTlbi(ExecContext *xc, MiscRegIndex dest_idx, RegVal value) const // TLB Invalidate by ASID match case MISCREG_TLBIASID: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TLBIASID tlbiOp(TranslationRegime::EL10, secure, bits(value, 7, 0)); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by ASID match, Inner Shareable @@ -534,25 +588,42 @@ TlbiOp::performTlbi(ExecContext *xc, MiscRegIndex dest_idx, RegVal value) const // TLB Invalidate by VA, All ASID case MISCREG_TLBIMVAA: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; TLBIMVAA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), false); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by VA, Last Level, All ASID case MISCREG_TLBIMVAAL: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TLBIMVAA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), true); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate by VA, All ASID, Inner Shareable @@ -689,55 +760,90 @@ TlbiOp::performTlbi(ExecContext *xc, MiscRegIndex dest_idx, RegVal value) const // Instruction TLB Invalidate by VA case MISCREG_ITLBIMVA: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; ITLBIMVA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), bits(value, 7, 0)); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // Data TLB Invalidate by VA case MISCREG_DTLBIMVA: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + DTLBIMVA tlbiOp(TranslationRegime::EL10, secure, mbits(value, 31, 12), bits(value, 7, 0)); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // Instruction TLB Invalidate by ASID match case MISCREG_ITLBIASID: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + ITLBIASID tlbiOp(TranslationRegime::EL10, secure, bits(value, 7, 0)); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // Data TLB Invalidate by ASID match case MISCREG_DTLBIASID: { + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); SCR scr = tc->readMiscReg(MISCREG_SCR_EL3); bool secure = release->has(ArmExtension::SECURITY) && !scr.ns; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + DTLBIASID tlbiOp(TranslationRegime::EL10, secure, bits(value, 7, 0)); - tlbiOp(tc); + if (shareable) { + tlbiOp.broadcast(tc); + } else { + tlbiOp(tc); + } return; } // TLB Invalidate All, Non-Secure Non-Hyp diff --git a/src/arch/arm/insts/misc64.cc b/src/arch/arm/insts/misc64.cc index e9ab2e2771..a5ca423ea6 100644 --- a/src/arch/arm/insts/misc64.cc +++ b/src/arch/arm/insts/misc64.cc @@ -500,10 +500,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiVmall(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false); // shareable + shareable); // shareable } }, @@ -698,10 +703,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable false); // last level only } }, @@ -776,10 +786,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiAsid(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false); // shareable + shareable); // shareable } }, @@ -812,10 +827,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiVaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable false); // last level } }, @@ -851,10 +871,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiVaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable true); // last level } }, @@ -950,10 +975,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable false); // last level only } }, @@ -989,10 +1019,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiRvaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable false); // last level only } }, @@ -1028,10 +1063,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable true); // last level only } }, @@ -1067,10 +1107,15 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + TlbiOp64::tlbiRvaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - false, // shareable + shareable, // shareable true); // last level only } }, diff --git a/src/arch/arm/isa/formats/aarch64.isa b/src/arch/arm/isa/formats/aarch64.isa index eef6003410..246d1a7836 100644 --- a/src/arch/arm/isa/formats/aarch64.isa +++ b/src/arch/arm/isa/formats/aarch64.isa @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2023 Arm Limited +// Copyright (c) 2011-2024 Arm Limited // All rights reserved // // The license below extends only to copyright in the software and shall @@ -553,7 +553,7 @@ namespace Aarch64 case MISCREG_TLBI_RVAE3: case MISCREG_TLBI_RVALE3: return new Tlbi64LocalHub( - machInst, miscReg, rt); + machInst, miscReg, rt, dec.dvmEnabled); case MISCREG_TLBI_ALLE3IS: case MISCREG_TLBI_ALLE3OS: case MISCREG_TLBI_ALLE2IS: diff --git a/src/arch/arm/isa/insts/data64.isa b/src/arch/arm/isa/insts/data64.isa index 87f87130ce..02ea53881d 100644 --- a/src/arch/arm/isa/insts/data64.isa +++ b/src/arch/arm/isa/insts/data64.isa @@ -1,6 +1,6 @@ // -*- mode:c++ -*- -// Copyright (c) 2011-2013, 2016-2023 Arm Limited +// Copyright (c) 2011-2013, 2016-2024 Arm Limited // All rights reserved // // The license below extends only to copyright in the software and shall @@ -361,15 +361,8 @@ let {{ tlbiCode = msr_check_code + ''' performTlbi(xc, flat_idx, XOp1); ''' - msrTlbiIop = ArmInstObjParams("msr", "Tlbi64LocalHub", "TlbiOp64", - tlbiCode, - ["IsSerializeAfter", "IsNonSpeculative"]) - header_output += MiscRegRegOp64Declare.subst(msrTlbiIop) - decoder_output += MiscRegRegOp64Constructor.subst(msrTlbiIop) - exec_output += BasicExecute.subst(msrTlbiIop) - dvmCode = ''' - if (dvmEnabled) { + if (%(dvmCheck)s) { Request::Flags memAccessFlags = Request::STRICT_ORDER | Request::TLBI; @@ -378,9 +371,30 @@ let {{ PendingDvm = true; } ''' + msrTlbiIop = ArmInstObjParams("msr", "Tlbi64LocalHub", "TlbiOp64", + { + "code" : tlbiCode, + "dvm_code" : dvmCode % + { + "dvmCheck" : "HCR hcr = Hcr64; hcr.fb && dvmEnabled" + } + }, + ["IsSerializeAfter", "IsNonSpeculative"]) + header_output += DvmTlbiDeclare.subst(msrTlbiIop) + decoder_output += DvmTlbiConstructor.subst(msrTlbiIop) + exec_output += BasicExecute.subst(msrTlbiIop) + exec_output += DvmInitiateAcc.subst(msrTlbiIop) + exec_output += DvmCompleteAcc.subst(msrTlbiIop) + msrTlbiSIop = ArmInstObjParams("msr", "Tlbi64ShareableHub", "TlbiOp64", - { "code" : tlbiCode, "dvm_code" : dvmCode }, - ["IsSerializeAfter", "IsNonSpeculative"]) + { + "code" : tlbiCode, + "dvm_code" : dvmCode % + { + "dvmCheck" : "dvmEnabled" + } + }, + ["IsSerializeAfter", "IsNonSpeculative"]) header_output += DvmTlbiDeclare.subst(msrTlbiSIop) decoder_output += DvmTlbiConstructor.subst(msrTlbiSIop) exec_output += BasicExecute.subst(msrTlbiSIop) diff --git a/src/arch/arm/tlbi_op.hh b/src/arch/arm/tlbi_op.hh index c74a998e48..26105abfaf 100644 --- a/src/arch/arm/tlbi_op.hh +++ b/src/arch/arm/tlbi_op.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, 2022-2023 Arm Limited + * Copyright (c) 2018-2020, 2022-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -143,8 +143,6 @@ class ITLBIALL : public TLBIALL : TLBIALL(_target_regime, _secure) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override; @@ -158,8 +156,6 @@ class DTLBIALL : public TLBIALL : TLBIALL(_target_regime, _secure) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override; @@ -247,8 +243,6 @@ class ITLBIASID : public TLBIASID : TLBIASID(_target_regime, _secure, _asid) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override; @@ -262,8 +256,6 @@ class DTLBIASID : public TLBIASID : TLBIASID(_target_regime, _secure, _asid) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override; @@ -345,8 +337,6 @@ class ITLBIMVA : public TLBIMVA : TLBIMVA(_target_regime, _secure, _addr, _asid, false) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override; @@ -361,8 +351,6 @@ class DTLBIMVA : public TLBIMVA : TLBIMVA(_target_regime, _secure, _addr, _asid, false) {} - void broadcast(ThreadContext *tc) = delete; - void operator()(ThreadContext* tc) override; bool match(TlbEntry *entry, vmid_t curr_vmid) const override;