From b28659d4f9f7c1cf67aedf81649d6a4745401e30 Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Tue, 2 Jul 2024 08:52:59 +0100 Subject: [PATCH] arch-arm: Implement FEAT_XS (#1303) This patch is adding a functional implementation of FEAT_XS. Unless we operate with DVM enabled, TLBIs broadcasting is accomplished in 0 time; so there is no timing benefit introduced by enabling FEAT_XS other than the way it affects TLB management (invalidation) Change-Id: I067cb8b7702c59c40c9bbb8da536a0b7f3337b5d Signed-off-by: Giacomo Travaglini --- src/arch/arm/ArmSystem.py | 3 + src/arch/arm/insts/misc64.cc | 1353 ++++++++++++++++++++++++-- src/arch/arm/insts/misc64.hh | 24 +- src/arch/arm/isa/formats/aarch64.isa | 172 ++-- src/arch/arm/pagetable.hh | 14 +- src/arch/arm/regs/misc.cc | 354 +++++++ src/arch/arm/regs/misc.hh | 326 ++++--- src/arch/arm/regs/misc_types.hh | 3 + src/arch/arm/stage2_lookup.cc | 10 +- src/arch/arm/table_walker.cc | 56 +- src/arch/arm/table_walker.hh | 9 + src/arch/arm/tlb.cc | 15 +- src/arch/arm/tlbi_op.cc | 50 +- src/arch/arm/tlbi_op.hh | 106 +- 14 files changed, 2077 insertions(+), 418 deletions(-) diff --git a/src/arch/arm/ArmSystem.py b/src/arch/arm/ArmSystem.py index 94cbb496aa..582f397a61 100644 --- a/src/arch/arm/ArmSystem.py +++ b/src/arch/arm/ArmSystem.py @@ -105,6 +105,7 @@ class ArmExtension(ScopedEnum): "FEAT_FGT", # Armv8.7 "FEAT_HCX", + "FEAT_XS", # Armv8.9 "FEAT_SCTLR2", "FEAT_TCR2", @@ -210,6 +211,7 @@ class ArmDefaultRelease(Armv8): "FEAT_FGT", # Armv8.7 "FEAT_HCX", + "FEAT_XS", # Armv9.2 "FEAT_SME", ] @@ -271,6 +273,7 @@ class Armv86(Armv85): class Armv87(Armv86): extensions = Armv86.extensions + [ "FEAT_HCX", + "FEAT_XS", ] diff --git a/src/arch/arm/insts/misc64.cc b/src/arch/arm/insts/misc64.cc index a5ca423ea6..b5b8ce7415 100644 --- a/src/arch/arm/insts/misc64.cc +++ b/src/arch/arm/insts/misc64.cc @@ -247,9 +247,10 @@ RegNone::generateDisassembly( void TlbiOp64::tlbiAll(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable) + bool secure, TranslationRegime regime, bool shareable, + TlbiAttr attrs) { - TLBIALLEL tlbi_op(regime, secure); + TLBIALLEL tlbi_op(regime, secure, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { @@ -259,9 +260,10 @@ TlbiOp64::tlbiAll(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiVmall(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool stage2) + bool secure, TranslationRegime regime, bool shareable, bool stage2, + TlbiAttr attrs) { - TLBIVMALL tlbi_op(regime, secure, stage2); + TLBIVMALL tlbi_op(regime, secure, stage2, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { @@ -271,7 +273,8 @@ TlbiOp64::tlbiVmall(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiVa(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { if (MMU::hasUnprivRegime(regime)) { // The asid will only be used when e2h == 1 @@ -280,14 +283,16 @@ TlbiOp64::tlbiVa(ThreadContext *tc, RegVal value, bits(value, 55, 48); TLBIMVA tlbi_op(regime, secure, static_cast(bits(value, 43, 0)) << 12, - asid, last_level); + asid, last_level, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { tlbi_op(tc); } } else { - TLBIMVAA tlbi_op(regime, secure, static_cast(bits(value, 43, 0)) << 12, last_level); + TLBIMVAA tlbi_op(regime, secure, + static_cast(bits(value, 43, 0)) << 12, last_level, + attrs); if (shareable) { tlbi_op.broadcast(tc); } else { @@ -298,9 +303,13 @@ TlbiOp64::tlbiVa(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiVaa(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { - TLBIMVAA tlbi_op(regime, secure, static_cast(bits(value, 43, 0)) << 12, last_level); + TLBIMVAA tlbi_op(regime, secure, + static_cast(bits(value, 43, 0)) << 12, last_level, + attrs); + if (shareable) { tlbi_op.broadcast(tc); } else { @@ -310,13 +319,14 @@ TlbiOp64::tlbiVaa(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiAsid(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable) + bool secure, TranslationRegime regime, bool shareable, + TlbiAttr attrs) { bool asid_16bits = ArmSystem::haveLargeAsid64(tc); auto asid = asid_16bits ? bits(value, 63, 48) : bits(value, 55, 48); - TLBIASID tlbi_op(regime, secure, asid); + TLBIASID tlbi_op(regime, secure, asid, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { @@ -326,7 +336,8 @@ TlbiOp64::tlbiAsid(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiIpaS2(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { if (EL2Enabled(tc)) { auto isa = static_cast(tc->getIsaPtr()); @@ -340,7 +351,7 @@ TlbiOp64::tlbiIpaS2(ThreadContext *tc, RegVal value, 39 : 35; TLBIIPA tlbi_op(TranslationRegime::EL10, secure, static_cast(bits(value, top_bit, 0)) << 12, - last_level); + last_level, attrs); if (shareable) { tlbi_op.broadcast(tc); @@ -352,9 +363,10 @@ TlbiOp64::tlbiIpaS2(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiRvaa(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { - TLBIRMVAA tlbi_op(regime, secure, value, last_level); + TLBIRMVAA tlbi_op(regime, secure, value, last_level, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { @@ -364,7 +376,8 @@ TlbiOp64::tlbiRvaa(ThreadContext *tc, RegVal value, void TlbiOp64::tlbiRva(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { if (MMU::hasUnprivRegime(regime)) { // The asid will only be used when e2h == 1 @@ -372,20 +385,21 @@ TlbiOp64::tlbiRva(ThreadContext *tc, RegVal value, auto asid = asid_16bits ? bits(value, 63, 48) : bits(value, 55, 48); - TLBIRMVA tlbi_op(regime, secure, value, asid, last_level); + TLBIRMVA tlbi_op(regime, secure, value, asid, last_level, attrs); if (shareable) { tlbi_op.broadcast(tc); } else { tlbi_op(tc); } } else { - tlbiRvaa(tc, value, secure, regime, shareable, last_level); + tlbiRvaa(tc, value, secure, regime, shareable, last_level, attrs); } } void TlbiOp64::tlbiRipaS2(ThreadContext *tc, RegVal value, - bool secure, TranslationRegime regime, bool shareable, bool last_level) + bool secure, TranslationRegime regime, bool shareable, bool last_level, + TlbiAttr attrs) { if (EL2Enabled(tc)) { auto isa = static_cast(tc->getIsaPtr()); @@ -394,7 +408,8 @@ TlbiOp64::tlbiRipaS2(ThreadContext *tc, RegVal value, bool secure = release->has(ArmExtension::SECURITY) && !scr.ns && !bits(value, 63); - TLBIRIPA tlbi_op(TranslationRegime::EL10, secure, value, last_level); + TLBIRIPA tlbi_op(TranslationRegime::EL10, secure, value, + last_level, attrs); if (shareable) { tlbi_op.broadcast(tc); @@ -404,6 +419,16 @@ TlbiOp64::tlbiRipaS2(ThreadContext *tc, RegVal value, } } +bool +TlbiOp64::fnxsAttrs(ThreadContext *tc) +{ + HCRX hcrx = tc->readMiscRegNoEffect(MISCREG_HCRX_EL2); + return currEL(tc) == EL1 && + HaveExt(tc, ArmExtension::FEAT_XS) && + HaveExt(tc, ArmExtension::FEAT_HCX) && + isHcrxEL2Enabled(tc) && hcrx.fnxs; +} + std::unordered_map TlbiOp64::tlbiOps = { { MISCREG_TLBI_ALLE3, [](ThreadContext *tc, RegVal value) { @@ -414,6 +439,16 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE3NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + true, // secure + TranslationRegime::EL3, // regime + false, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE3IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiAll(tc, value, @@ -423,6 +458,16 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE3ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE3OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiAll(tc, value, @@ -432,6 +477,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE3OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_ALLE2, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -444,6 +500,19 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE2NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + false, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE2IS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -456,6 +525,19 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE2ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE2OS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -468,6 +550,19 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE2OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE1, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiAll(tc, value, @@ -477,6 +572,16 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE1NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiAll(tc, value, @@ -486,6 +591,16 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE1ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_ALLE1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiAll(tc, value, @@ -495,7 +610,40 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_ALLE1OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiAll(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VMALLE1, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + false, // stage2 + attrs); // attrs + } + }, + + { MISCREG_TLBI_VMALLE1NXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -508,7 +656,9 @@ std::unordered_map TlbiOp64::tlbiOps = { TlbiOp64::tlbiVmall(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - shareable); // shareable + shareable, // shareable + false, // stage2 + TlbiAttr::ExcludeXS); // attrs } }, @@ -517,10 +667,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVmall(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - true); // shareable + true, // shareable + false, // stage2 + attrs); // attrs + } + }, + + { MISCREG_TLBI_VMALLE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // stage2 + TlbiAttr::ExcludeXS); // attrs } }, @@ -529,10 +698,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVmall(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - true); // shareable + true, // shareable + false, // stage2 + attrs); // attrs + } + }, + + { MISCREG_TLBI_VMALLE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // stage2 + TlbiAttr::ExcludeXS); // attrs } }, @@ -546,6 +734,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VMALLS12E1NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + true, // stage2 + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VMALLS12E1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVmall(tc, value, @@ -556,6 +755,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VMALLS12E1ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // stage2 + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VMALLS12E1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVmall(tc, value, @@ -566,6 +776,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VMALLS12E1OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVmall(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // stage2 + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VAE3, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -576,6 +798,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE3NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + false, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VAE3IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -586,6 +820,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE3ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VAE3OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -596,6 +841,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE3OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VALE3, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -606,6 +863,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE3NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + false, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VALE3IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -616,6 +885,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE3ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VALE3OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -626,6 +907,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE3OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + true, // secure + TranslationRegime::EL3, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VAE2, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -639,6 +931,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE2NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + false, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VAE2IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -649,6 +955,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE2ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, EL2), // secure + TranslationRegime::EL2, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VAE2OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiVa(tc, value, @@ -659,6 +977,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VAE2OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc,EL2), // secure + TranslationRegime::EL2, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VALE2, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -672,6 +1002,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE2NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + false, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VALE2IS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -685,6 +1029,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE2ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VALE2OS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? @@ -698,7 +1056,43 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_VALE2OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_VAE1, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAE1NXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -712,7 +1106,8 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - false); // last level only + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -721,11 +1116,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -734,11 +1147,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -747,11 +1178,34 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime false, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VALE1NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -760,11 +1214,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VALE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -773,11 +1245,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VALE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -786,6 +1276,9 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && @@ -794,7 +1287,26 @@ std::unordered_map TlbiOp64::tlbiOps = { TlbiOp64::tlbiAsid(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - shareable); // shareable + shareable, // shareable + attrs); // attrs + } + }, + + { MISCREG_TLBI_ASIDE1NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiAsid(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + TlbiAttr::ExcludeXS); // attrs } }, @@ -803,10 +1315,27 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiAsid(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - true); // shareable + true, // shareable + attrs); // attrs + } + }, + + { MISCREG_TLBI_ASIDE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiAsid(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs } }, @@ -815,10 +1344,27 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiAsid(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime - true); // shareable + true, // shareable + attrs); // attrs + } + }, + + { MISCREG_TLBI_ASIDE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiAsid(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + TlbiAttr::ExcludeXS); // attrs } }, @@ -827,6 +1373,9 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && @@ -836,7 +1385,27 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - false); // last level + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAAE1NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + false, // last level + TlbiAttr::ExcludeXS); // attrs } }, @@ -845,15 +1414,19 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level + false, // last level only + attrs); // attrs } }, - { MISCREG_TLBI_VAAE1OS, [](ThreadContext *tc, RegVal value) + { MISCREG_TLBI_VAAE1ISNXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -862,11 +1435,65 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level + false, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VAAE1OS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAAE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level + TlbiAttr::ExcludeXS); // attrs } }, { MISCREG_TLBI_VAALE1, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAALE1NXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -880,7 +1507,8 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - true); // last level + true, // last level + TlbiAttr::ExcludeXS); // attrs } }, @@ -889,15 +1517,19 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiVaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level + true, // last level only + attrs); // attrs } }, - { MISCREG_TLBI_VAALE1OS, [](ThreadContext *tc, RegVal value) + { MISCREG_TLBI_VAALE1ISNXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -906,7 +1538,39 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level + true, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_VAALE1OS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_VAALE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiVaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level + TlbiAttr::ExcludeXS); // attrs } }, @@ -920,6 +1584,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2E1NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + false, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_IPAS2E1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiIpaS2(tc, value, @@ -930,6 +1605,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2E1ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_IPAS2E1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiIpaS2(tc, value, @@ -940,6 +1627,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2E1OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_IPAS2LE1, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiIpaS2(tc, value, @@ -950,6 +1648,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2LE1NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + true, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_IPAS2LE1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiIpaS2(tc, value, @@ -960,6 +1670,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2LE1ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_IPAS2LE1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiIpaS2(tc, value, @@ -970,7 +1692,41 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_IPAS2LE1OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiIpaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // last level + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_RVAE1, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAE1NXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -980,11 +1736,12 @@ std::unordered_map TlbiOp64::tlbiOps = { bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && hcr.fb && !hcr.tge; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - false); // last level only + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -993,11 +1750,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; - tlbiRva(tc, value, + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1006,11 +1781,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; - tlbiRva(tc, value, + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1019,6 +1812,9 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && @@ -1028,7 +1824,27 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - false); // last level only + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAAE1NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1037,15 +1853,19 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiRvaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + attrs); // attrs } }, - { MISCREG_TLBI_RVAAE1OS, [](ThreadContext *tc, RegVal value) + { MISCREG_TLBI_RVAAE1ISNXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -1054,7 +1874,40 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - false); // last level only + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + + { MISCREG_TLBI_RVAAE1OS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAAE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1063,16 +1916,39 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && hcr.fb && !hcr.tge; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVALE1NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1081,11 +1957,29 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; - tlbiRva(tc, value, + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVALE1ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1094,15 +1988,55 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; - tlbiRva(tc, value, + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVALE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, { MISCREG_TLBI_RVAALE1, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + // Check for Force Broadcast. Ignored if HCR_EL2.TGE == 1 + HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + bool shareable = currEL(tc) == EL1 && EL2Enabled(tc) && + hcr.fb && !hcr.tge; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + shareable, // shareable + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAALE1NXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -1116,7 +2050,8 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime shareable, // shareable - true); // last level only + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1125,15 +2060,19 @@ std::unordered_map TlbiOp64::tlbiOps = { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + TlbiOp64::tlbiRvaa(tc, value, isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + attrs); // attrs } }, - { MISCREG_TLBI_RVAALE1OS, [](ThreadContext *tc, RegVal value) + { MISCREG_TLBI_RVAALE1ISNXS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL0) ? TranslationRegime::EL20 : TranslationRegime::EL10; @@ -1142,7 +2081,39 @@ std::unordered_map TlbiOp64::tlbiOps = { isSecureAtEL(tc, translationEl(regime)), // secure regime, // regime true, // shareable - true); // last level only + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_RVAALE1OS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + const TlbiAttr attrs = fnxsAttrs(tc) ? + TlbiAttr::ExcludeXS : TlbiAttr::None; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + attrs); // attrs + } + }, + + { MISCREG_TLBI_RVAALE1OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL0) ? + TranslationRegime::EL20 : TranslationRegime::EL10; + + TlbiOp64::tlbiRvaa(tc, value, + isSecureAtEL(tc, translationEl(regime)), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs } }, @@ -1156,6 +2127,18 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2E1NXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_RIPAS2E1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiRipaS2(tc, value, @@ -1166,6 +2149,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2E1ISNXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RIPAS2E1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiRipaS2(tc, value, @@ -1176,6 +2170,38 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2E1OSNXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + + { MISCREG_TLBI_RIPAS2E1OS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false); // last level only + } + }, + + { MISCREG_TLBI_RIPAS2E1OSNXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RIPAS2LE1, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiRipaS2(tc, value, @@ -1186,6 +2212,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2LE1NXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + false, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RIPAS2LE1IS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiRipaS2(tc, value, @@ -1196,6 +2233,17 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2LE1ISNXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RIPAS2LE1OS, [](ThreadContext *tc, RegVal value) { TlbiOp64::tlbiRipaS2(tc, value, @@ -1206,12 +2254,23 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RIPAS2LE1OSNXS, [](ThreadContext *tc, RegVal value) + { + tlbiRipaS2(tc, value, + isSecureAtEL(tc, EL1), // secure + TranslationRegime::EL10, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE2, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime false, // shareable @@ -1219,12 +2278,26 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE2NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + false, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE2IS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime true, // shareable @@ -1232,12 +2305,26 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE2ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE2OS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime true, // shareable @@ -1245,12 +2332,26 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE2OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE2, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime false, // shareable @@ -1258,12 +2359,26 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVALE2NXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + false, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE2IS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime true, // shareable @@ -1271,12 +2386,26 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVALE2ISNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE2OS, [](ThreadContext *tc, RegVal value) { const TranslationRegime regime = ELIsInHost(tc, EL2) ? TranslationRegime::EL20 : TranslationRegime::EL2; - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL2), // secure regime, // regime true, // shareable @@ -1284,9 +2413,23 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVALE2OSNXS, [](ThreadContext *tc, RegVal value) + { + const TranslationRegime regime = ELIsInHost(tc, EL2) ? + TranslationRegime::EL20 : TranslationRegime::EL2; + + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL2), // secure + regime, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE3, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime false, // shareable @@ -1294,9 +2437,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE3NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + false, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE3IS, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime true, // shareable @@ -1304,9 +2458,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE3ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVAE3OS, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime true, // shareable @@ -1314,9 +2479,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVAE3OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + true, // shareable + false, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE3, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime false, // shareable @@ -1324,9 +2500,20 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVALE3NXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + false, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE3IS, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime true, // shareable @@ -1334,15 +2521,37 @@ std::unordered_map TlbiOp64::tlbiOps = { } }, + { MISCREG_TLBI_RVALE3ISNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, + { MISCREG_TLBI_RVALE3OS, [](ThreadContext *tc, RegVal value) { - tlbiRva(tc, value, + TlbiOp64::tlbiRva(tc, value, isSecureAtEL(tc, EL3), // secure TranslationRegime::EL3, // regime true, // shareable true); // last level only } }, + + { MISCREG_TLBI_RVALE3OSNXS, [](ThreadContext *tc, RegVal value) + { + TlbiOp64::tlbiRva(tc, value, + isSecureAtEL(tc, EL3), // secure + TranslationRegime::EL3, // regime + true, // shareable + true, // last level only + TlbiAttr::ExcludeXS); // attrs + } + }, }; void diff --git a/src/arch/arm/insts/misc64.hh b/src/arch/arm/insts/misc64.hh index 14ed41cb75..7ce3b188c5 100644 --- a/src/arch/arm/insts/misc64.hh +++ b/src/arch/arm/insts/misc64.hh @@ -39,6 +39,7 @@ #define __ARCH_ARM_INSTS_MISC64_HH__ #include "arch/arm/insts/static_inst.hh" +#include "arch/arm/tlbi_op.hh" #include "arch/arm/types.hh" namespace gem5 @@ -286,42 +287,47 @@ class TlbiOp64 : public MiscRegRegImmOp64 { protected: using TlbiFunc = std::function; + using TlbiAttr = ArmISA::TLBIOp::Attr; static std::unordered_map tlbiOps; static void tlbiAll(ThreadContext *tc, RegVal value, - bool secure, ArmISA::TranslationRegime regime, bool shareable); + bool secure, ArmISA::TranslationRegime regime, bool shareable, + TlbiAttr attrs=TlbiAttr::None); static void tlbiVmall(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool stage2=false); + bool stage2=false, TlbiAttr attrs=TlbiAttr::None); static void tlbiVa(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); static void tlbiVaa(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); static void tlbiAsid(ThreadContext *tc, RegVal value, - bool secure, ArmISA::TranslationRegime regime, bool shareable); + bool secure, ArmISA::TranslationRegime regime, bool shareable, + TlbiAttr attrs=TlbiAttr::None); static void tlbiIpaS2(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); static void tlbiRvaa(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); static void tlbiRva(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); static void tlbiRipaS2(ThreadContext *tc, RegVal value, bool secure, ArmISA::TranslationRegime regime, bool shareable, - bool last_level); + bool last_level, TlbiAttr attrs=TlbiAttr::None); + + static bool fnxsAttrs(ThreadContext *tc); protected: TlbiOp64(const char *mnem, ArmISA::ExtMachInst _machInst, diff --git a/src/arch/arm/isa/formats/aarch64.isa b/src/arch/arm/isa/formats/aarch64.isa index 537393aa23..8be050a492 100644 --- a/src/arch/arm/isa/formats/aarch64.isa +++ b/src/arch/arm/isa/formats/aarch64.isa @@ -399,6 +399,22 @@ namespace Aarch64 "unallocated_hint", machInst); } else if (crn == 0x3 && op1 == 0x3) { switch (op2) { + case 0x1: + if (bits(crm, 1, 0) == 0b10) { + switch (bits(crm, 3, 2)) { + case 0x1: // Non-Shareable + return new Dsb64Local(machInst); + case 0x0: // OuterShareable + case 0x2: // InnerShareable + case 0x3: // FullSystem + return new Dsb64Shareable( + machInst, dec.dvmEnabled); + default: + GEM5_UNREACHABLE; + } + } else { + return new Unknown64(machInst); + } case 0x2: return new Clrex64(machInst); case 0x4: @@ -526,86 +542,86 @@ namespace Aarch64 return new Dcivac(machInst, rt, miscReg); // 64-bit TLBIs split into "Local" // and "Shareable" - case MISCREG_TLBI_ALLE3: - case MISCREG_TLBI_ALLE2: - case MISCREG_TLBI_ALLE1: - case MISCREG_TLBI_VMALLS12E1: - case MISCREG_TLBI_VMALLE1: - case MISCREG_TLBI_VAE3: - case MISCREG_TLBI_VALE3: - case MISCREG_TLBI_VAE2: - case MISCREG_TLBI_VALE2: - case MISCREG_TLBI_VAE1: - case MISCREG_TLBI_VALE1: - case MISCREG_TLBI_ASIDE1: - case MISCREG_TLBI_VAAE1: - case MISCREG_TLBI_VAALE1: - case MISCREG_TLBI_IPAS2E1: - case MISCREG_TLBI_IPAS2LE1: - case MISCREG_TLBI_RVAE1: - case MISCREG_TLBI_RVAAE1: - case MISCREG_TLBI_RVALE1: - case MISCREG_TLBI_RVAALE1: - case MISCREG_TLBI_RIPAS2E1: - case MISCREG_TLBI_RIPAS2LE1: - case MISCREG_TLBI_RVAE2: - case MISCREG_TLBI_RVALE2: - case MISCREG_TLBI_RVAE3: - case MISCREG_TLBI_RVALE3: + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE3) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE2) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLS12E1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE3) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE3) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE2) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE2) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ASIDE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAAE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAALE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2E1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2LE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAAE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAALE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2E1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2LE1) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE2) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE2) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE3) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE3) return new Tlbi64LocalHub( machInst, miscReg, rt, dec.dvmEnabled); - case MISCREG_TLBI_ALLE3IS: - case MISCREG_TLBI_ALLE3OS: - case MISCREG_TLBI_ALLE2IS: - case MISCREG_TLBI_ALLE2OS: - case MISCREG_TLBI_ALLE1IS: - case MISCREG_TLBI_ALLE1OS: - case MISCREG_TLBI_VMALLS12E1IS: - case MISCREG_TLBI_VMALLS12E1OS: - case MISCREG_TLBI_VMALLE1IS: - case MISCREG_TLBI_VMALLE1OS: - case MISCREG_TLBI_VAE3IS: - case MISCREG_TLBI_VAE3OS: - case MISCREG_TLBI_VALE3IS: - case MISCREG_TLBI_VALE3OS: - case MISCREG_TLBI_VAE2IS: - case MISCREG_TLBI_VAE2OS: - case MISCREG_TLBI_VALE2IS: - case MISCREG_TLBI_VALE2OS: - case MISCREG_TLBI_VAE1IS: - case MISCREG_TLBI_VAE1OS: - case MISCREG_TLBI_VALE1IS: - case MISCREG_TLBI_VALE1OS: - case MISCREG_TLBI_ASIDE1IS: - case MISCREG_TLBI_ASIDE1OS: - case MISCREG_TLBI_VAAE1IS: - case MISCREG_TLBI_VAAE1OS: - case MISCREG_TLBI_VAALE1IS: - case MISCREG_TLBI_VAALE1OS: - case MISCREG_TLBI_IPAS2E1IS: - case MISCREG_TLBI_IPAS2E1OS: - case MISCREG_TLBI_IPAS2LE1IS: - case MISCREG_TLBI_IPAS2LE1OS: - case MISCREG_TLBI_RVAE1IS: - case MISCREG_TLBI_RVAE1OS: - case MISCREG_TLBI_RVAAE1IS: - case MISCREG_TLBI_RVAAE1OS: - case MISCREG_TLBI_RVALE1IS: - case MISCREG_TLBI_RVALE1OS: - case MISCREG_TLBI_RVAALE1IS: - case MISCREG_TLBI_RVAALE1OS: - case MISCREG_TLBI_RIPAS2E1IS: - case MISCREG_TLBI_RIPAS2E1OS: - case MISCREG_TLBI_RIPAS2LE1IS: - case MISCREG_TLBI_RIPAS2LE1OS: - case MISCREG_TLBI_RVAE2IS: - case MISCREG_TLBI_RVAE2OS: - case MISCREG_TLBI_RVALE2IS: - case MISCREG_TLBI_RVALE2OS: - case MISCREG_TLBI_RVAE3IS: - case MISCREG_TLBI_RVAE3OS: - case MISCREG_TLBI_RVALE3IS: - case MISCREG_TLBI_RVALE3OS: + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE3IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE3OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE2IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE2OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ALLE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLS12E1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLS12E1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VMALLE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE3IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE3OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE3IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE3OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE2IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE2OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE2IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE2OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VALE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ASIDE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_ASIDE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAAE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAAE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAALE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_VAALE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2E1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2E1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2LE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_IPAS2LE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAAE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAAE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAALE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAALE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2E1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2E1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2LE1IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RIPAS2LE1OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE2IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE2OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE2IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE2OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE3IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVAE3OS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE3IS) + TLBI_CASE_VARIANTS(MISCREG_TLBI_RVALE3OS) return new Tlbi64ShareableHub( machInst, miscReg, rt, dec.dvmEnabled); default: diff --git a/src/arch/arm/pagetable.hh b/src/arch/arm/pagetable.hh index af70adb2ba..2d8c1566e0 100644 --- a/src/arch/arm/pagetable.hh +++ b/src/arch/arm/pagetable.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2012-2013, 2021, 2023 Arm Limited + * Copyright (c) 2010, 2012-2013, 2021, 2023-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -260,6 +260,8 @@ struct TlbEntry : public Serializable bool xn; // Execute Never bool pxn; // Privileged Execute Never (LPAE only) + bool xs; // xs attribute from FEAT_XS + //Construct an entry that maps to physical address addr for SE mode TlbEntry(Addr _asn, Addr _vaddr, Addr _paddr, bool uncacheable, bool read_only) : @@ -272,7 +274,8 @@ struct TlbEntry : public Serializable ns(true), nstid(true), regime(TranslationRegime::EL10), type(TypeTLB::unified), partial(false), nonCacheable(uncacheable), - shareable(false), outerShareable(false), xn(0), pxn(0) + shareable(false), outerShareable(false), xn(0), pxn(0), + xs(true) { // no restrictions by default, hap = 0x3 @@ -289,7 +292,8 @@ struct TlbEntry : public Serializable longDescFormat(false), global(false), valid(false), ns(true), nstid(true), regime(TranslationRegime::EL10), type(TypeTLB::unified), partial(false), nonCacheable(false), - shareable(false), outerShareable(false), xn(0), pxn(0) + shareable(false), outerShareable(false), xn(0), pxn(0), + xs(true) { // no restrictions by default, hap = 0x3 @@ -406,9 +410,9 @@ struct TlbEntry : public Serializable print() const { return csprintf("%#x, asn %d vmn %d ppn %#x size: %#x ap:%d " - "ns:%d nstid:%d g:%d regime:%s", vpn << N, asid, vmid, + "ns:%d nstid:%d g:%d xs: %d regime:%s", vpn << N, asid, vmid, pfn << N, size, ap, ns, nstid, global, - regimeToStr(regime)); + xs, regimeToStr(regime)); } void diff --git a/src/arch/arm/regs/misc.cc b/src/arch/arm/regs/misc.cc index 6cbbd2f6fe..d6953e91b6 100644 --- a/src/arch/arm/regs/misc.cc +++ b/src/arch/arm/regs/misc.cc @@ -783,6 +783,36 @@ std::unordered_map miscRegNumToIdx{ { MiscRegNum64(1, 0, 8, 7, 3), MISCREG_TLBI_VAAE1 }, { MiscRegNum64(1, 0, 8, 7, 5), MISCREG_TLBI_VALE1 }, { MiscRegNum64(1, 0, 8, 7, 7), MISCREG_TLBI_VAALE1 }, + { MiscRegNum64(1, 0, 9, 1, 0), MISCREG_TLBI_VMALLE1OSNXS }, + { MiscRegNum64(1, 0, 9, 1, 1), MISCREG_TLBI_VAE1OSNXS }, + { MiscRegNum64(1, 0, 9, 1, 2), MISCREG_TLBI_ASIDE1OSNXS }, + { MiscRegNum64(1, 0, 9, 1, 3), MISCREG_TLBI_VAAE1OSNXS }, + { MiscRegNum64(1, 0, 9, 1, 5), MISCREG_TLBI_VALE1OSNXS }, + { MiscRegNum64(1, 0, 9, 1, 7), MISCREG_TLBI_VAALE1OSNXS }, + { MiscRegNum64(1, 0, 9, 2, 1), MISCREG_TLBI_RVAE1ISNXS }, + { MiscRegNum64(1, 0, 9, 2, 3), MISCREG_TLBI_RVAAE1ISNXS }, + { MiscRegNum64(1, 0, 9, 2, 5), MISCREG_TLBI_RVALE1ISNXS }, + { MiscRegNum64(1, 0, 9, 2, 7), MISCREG_TLBI_RVAALE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 0), MISCREG_TLBI_VMALLE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 1), MISCREG_TLBI_VAE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 2), MISCREG_TLBI_ASIDE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 3), MISCREG_TLBI_VAAE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 5), MISCREG_TLBI_VALE1ISNXS }, + { MiscRegNum64(1, 0, 9, 3, 7), MISCREG_TLBI_VAALE1ISNXS }, + { MiscRegNum64(1, 0, 9, 5, 1), MISCREG_TLBI_RVAE1OSNXS }, + { MiscRegNum64(1, 0, 9, 5, 3), MISCREG_TLBI_RVAAE1OSNXS }, + { MiscRegNum64(1, 0, 9, 5, 5), MISCREG_TLBI_RVALE1OSNXS }, + { MiscRegNum64(1, 0, 9, 5, 7), MISCREG_TLBI_RVAALE1OSNXS }, + { MiscRegNum64(1, 0, 9, 6, 1), MISCREG_TLBI_RVAE1NXS }, + { MiscRegNum64(1, 0, 9, 6, 3), MISCREG_TLBI_RVAAE1NXS }, + { MiscRegNum64(1, 0, 9, 6, 5), MISCREG_TLBI_RVALE1NXS }, + { MiscRegNum64(1, 0, 9, 6, 7), MISCREG_TLBI_RVAALE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 0), MISCREG_TLBI_VMALLE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 1), MISCREG_TLBI_VAE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 2), MISCREG_TLBI_ASIDE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 3), MISCREG_TLBI_VAAE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 5), MISCREG_TLBI_VALE1NXS }, + { MiscRegNum64(1, 0, 9, 7, 7), MISCREG_TLBI_VAALE1NXS }, { MiscRegNum64(1, 3, 7, 4, 1), MISCREG_DC_ZVA_Xt }, { MiscRegNum64(1, 3, 7, 5, 1), MISCREG_IC_IVAU_Xt }, { MiscRegNum64(1, 3, 7, 10, 1), MISCREG_DC_CVAC_Xt }, @@ -827,6 +857,39 @@ std::unordered_map miscRegNumToIdx{ { MiscRegNum64(1, 4, 8, 7, 4), MISCREG_TLBI_ALLE1 }, { MiscRegNum64(1, 4, 8, 7, 5), MISCREG_TLBI_VALE2 }, { MiscRegNum64(1, 4, 8, 7, 6), MISCREG_TLBI_VMALLS12E1 }, + { MiscRegNum64(1, 4, 9, 0, 1), MISCREG_TLBI_IPAS2E1ISNXS }, + { MiscRegNum64(1, 4, 9, 0, 2), MISCREG_TLBI_RIPAS2E1ISNXS }, + { MiscRegNum64(1, 4, 9, 0, 5), MISCREG_TLBI_IPAS2LE1ISNXS }, + { MiscRegNum64(1, 4, 9, 1, 0), MISCREG_TLBI_ALLE2OSNXS }, + { MiscRegNum64(1, 4, 9, 1, 1), MISCREG_TLBI_VAE2OSNXS }, + { MiscRegNum64(1, 4, 9, 1, 4), MISCREG_TLBI_ALLE1OSNXS }, + { MiscRegNum64(1, 4, 9, 1, 5), MISCREG_TLBI_VALE2OSNXS }, + { MiscRegNum64(1, 4, 9, 1, 6), MISCREG_TLBI_VMALLS12E1OSNXS }, + { MiscRegNum64(1, 4, 9, 0, 6), MISCREG_TLBI_RIPAS2LE1ISNXS }, + { MiscRegNum64(1, 4, 9, 2, 1), MISCREG_TLBI_RVAE2ISNXS }, + { MiscRegNum64(1, 4, 9, 2, 5), MISCREG_TLBI_RVALE2ISNXS }, + { MiscRegNum64(1, 4, 9, 3, 0), MISCREG_TLBI_ALLE2ISNXS }, + { MiscRegNum64(1, 4, 9, 3, 1), MISCREG_TLBI_VAE2ISNXS }, + { MiscRegNum64(1, 4, 9, 3, 4), MISCREG_TLBI_ALLE1ISNXS }, + { MiscRegNum64(1, 4, 9, 3, 5), MISCREG_TLBI_VALE2ISNXS }, + { MiscRegNum64(1, 4, 9, 3, 6), MISCREG_TLBI_VMALLS12E1ISNXS }, + { MiscRegNum64(1, 4, 9, 4, 0), MISCREG_TLBI_IPAS2E1OSNXS }, + { MiscRegNum64(1, 4, 9, 4, 1), MISCREG_TLBI_IPAS2E1NXS }, + { MiscRegNum64(1, 4, 9, 4, 2), MISCREG_TLBI_RIPAS2E1NXS }, + { MiscRegNum64(1, 4, 9, 4, 3), MISCREG_TLBI_RIPAS2E1OSNXS }, + { MiscRegNum64(1, 4, 9, 4, 4), MISCREG_TLBI_IPAS2LE1OSNXS }, + { MiscRegNum64(1, 4, 9, 4, 5), MISCREG_TLBI_IPAS2LE1NXS }, + { MiscRegNum64(1, 4, 9, 4, 6), MISCREG_TLBI_RIPAS2LE1NXS }, + { MiscRegNum64(1, 4, 9, 4, 7), MISCREG_TLBI_RIPAS2LE1OSNXS }, + { MiscRegNum64(1, 4, 9, 5, 1), MISCREG_TLBI_RVAE2OSNXS }, + { MiscRegNum64(1, 4, 9, 5, 5), MISCREG_TLBI_RVALE2OSNXS }, + { MiscRegNum64(1, 4, 9, 6, 1), MISCREG_TLBI_RVAE2NXS }, + { MiscRegNum64(1, 4, 9, 6, 5), MISCREG_TLBI_RVALE2NXS }, + { MiscRegNum64(1, 4, 9, 7, 0), MISCREG_TLBI_ALLE2NXS }, + { MiscRegNum64(1, 4, 9, 7, 1), MISCREG_TLBI_VAE2NXS }, + { MiscRegNum64(1, 4, 9, 7, 4), MISCREG_TLBI_ALLE1NXS }, + { MiscRegNum64(1, 4, 9, 7, 5), MISCREG_TLBI_VALE2NXS }, + { MiscRegNum64(1, 4, 9, 7, 6), MISCREG_TLBI_VMALLS12E1NXS }, { MiscRegNum64(1, 6, 7, 8, 0), MISCREG_AT_S1E3R_Xt }, { MiscRegNum64(1, 6, 7, 8, 1), MISCREG_AT_S1E3W_Xt }, { MiscRegNum64(1, 6, 8, 1, 0), MISCREG_TLBI_ALLE3OS }, @@ -844,6 +907,21 @@ std::unordered_map miscRegNumToIdx{ { MiscRegNum64(1, 6, 8, 7, 0), MISCREG_TLBI_ALLE3 }, { MiscRegNum64(1, 6, 8, 7, 1), MISCREG_TLBI_VAE3 }, { MiscRegNum64(1, 6, 8, 7, 5), MISCREG_TLBI_VALE3 }, + { MiscRegNum64(1, 6, 9, 1, 0), MISCREG_TLBI_ALLE3OSNXS }, + { MiscRegNum64(1, 6, 9, 1, 1), MISCREG_TLBI_VAE3OSNXS }, + { MiscRegNum64(1, 6, 9, 1, 5), MISCREG_TLBI_VALE3OSNXS }, + { MiscRegNum64(1, 6, 9, 2, 1), MISCREG_TLBI_RVAE3ISNXS }, + { MiscRegNum64(1, 6, 9, 2, 5), MISCREG_TLBI_RVALE3ISNXS }, + { MiscRegNum64(1, 6, 9, 3, 0), MISCREG_TLBI_ALLE3ISNXS }, + { MiscRegNum64(1, 6, 9, 3, 1), MISCREG_TLBI_VAE3ISNXS }, + { MiscRegNum64(1, 6, 9, 3, 5), MISCREG_TLBI_VALE3ISNXS }, + { MiscRegNum64(1, 6, 9, 5, 1), MISCREG_TLBI_RVAE3OSNXS }, + { MiscRegNum64(1, 6, 9, 5, 5), MISCREG_TLBI_RVALE3OSNXS }, + { MiscRegNum64(1, 6, 9, 6, 1), MISCREG_TLBI_RVAE3NXS }, + { MiscRegNum64(1, 6, 9, 6, 5), MISCREG_TLBI_RVALE3NXS }, + { MiscRegNum64(1, 6, 9, 7, 0), MISCREG_TLBI_ALLE3NXS }, + { MiscRegNum64(1, 6, 9, 7, 1), MISCREG_TLBI_VAE3NXS }, + { MiscRegNum64(1, 6, 9, 7, 5), MISCREG_TLBI_VALE3NXS }, { MiscRegNum64(2, 0, 0, 0, 2), MISCREG_OSDTRRX_EL1 }, { MiscRegNum64(2, 0, 0, 0, 4), MISCREG_DBGBVR0_EL1 }, { MiscRegNum64(2, 0, 0, 0, 5), MISCREG_DBGBCR0_EL1 }, @@ -1384,6 +1462,30 @@ faultFgtInstEL1(const MiscRegLUTEntry &entry, } } +/** + * Template helper for fine grained traps at EL1 + * for TLBI *NXS + * + * @tparam r_bitfield: register (HFGITR) bitfield + */ +template +Fault +faultFgtTlbiNxsEL1(const MiscRegLUTEntry &entry, + ThreadContext *tc, const MiscRegOp64 &inst) +{ + if (HaveExt(tc, ArmExtension::FEAT_HCX)) { + const HCRX hcrx = tc->readMiscReg(MISCREG_HCRX_EL2); + if (auto fault = faultFgtInstEL1(entry, tc, inst); + fault != NoFault && (!isHcrxEL2Enabled(tc) || !hcrx.fgtnxs)) { + return fault; + } else { + return NoFault; + } + } else { + return NoFault; + } +} + /** * Template helper for fine grained traps at EL1 * @@ -1492,6 +1594,30 @@ faultHcrFgtInstEL1(const MiscRegLUTEntry &entry, } } +/** + * Template helper for fine grained traps at EL1 + * for TLBI *NXS instructions + * + * @tparam g_bitfield: group (HCR) bitfield + * @tparam r_bitfield: register (HFGITR) bitfield + */ +template +Fault +faultTlbiNxsEL1(const MiscRegLUTEntry &entry, + ThreadContext *tc, const MiscRegOp64 &inst) +{ + const HCR hcr = tc->readMiscReg(MISCREG_HCR_EL2); + + if (EL2Enabled(tc) && hcr.*g_bitfield) { + return inst.generateTrap(EL2); + } else if (auto fault = faultFgtTlbiNxsEL1(entry, tc, inst); + fault != NoFault) { + return fault; + } else { + return NoFault; + } +} + Fault faultSpEL0(const MiscRegLUTEntry &entry, ThreadContext *tc, const MiscRegOp64 &inst) @@ -2230,6 +2356,26 @@ faultTlbiOsEL1(const MiscRegLUTEntry &entry, } } +template +Fault +faultTlbiOsNxsEL1(const MiscRegLUTEntry &entry, + ThreadContext *tc, const MiscRegOp64 &inst) +{ + const HCR hcr = tc->readMiscRegNoEffect(MISCREG_HCR_EL2); + const bool el2_enabled = EL2Enabled(tc); + if (el2_enabled && hcr.ttlb) { + return inst.generateTrap(EL2); + } else if (el2_enabled && HaveExt(tc, ArmExtension::FEAT_EVT) && + hcr.ttlbos) { + return inst.generateTrap(EL2); + } else if (auto fault = faultFgtTlbiNxsEL1(entry, tc, inst); + fault != NoFault) { + return fault; + } else { + return NoFault; + } +} + template Fault faultTlbiIsEL1(const MiscRegLUTEntry &entry, @@ -2250,6 +2396,26 @@ faultTlbiIsEL1(const MiscRegLUTEntry &entry, } } +template +Fault +faultTlbiIsNxsEL1(const MiscRegLUTEntry &entry, + ThreadContext *tc, const MiscRegOp64 &inst) +{ + const HCR hcr = tc->readMiscRegNoEffect(MISCREG_HCR_EL2); + const bool el2_enabled = EL2Enabled(tc); + if (el2_enabled && hcr.ttlb) { + return inst.generateTrap(EL2); + } else if (el2_enabled && HaveExt(tc, ArmExtension::FEAT_EVT) && + hcr.ttlbis) { + return inst.generateTrap(EL2); + } else if (auto fault = faultFgtTlbiNxsEL1(entry, tc, inst); + fault != NoFault) { + return fault; + } else { + return NoFault; + } +} + template Fault faultCacheEL1(const MiscRegLUTEntry &entry, @@ -4769,6 +4935,7 @@ ISA::initializeMiscRegMetadata() InitReg(MISCREG_ID_AA64ISAR1_EL1) .reset([p,release=release](){ AA64ISAR1 isar1_el1 = p.id_aa64isar1_el1; + isar1_el1.xs = release->has(ArmExtension::FEAT_XS) ? 0x1 : 0x0; isar1_el1.i8mm = release->has(ArmExtension::FEAT_I8MM) ? 0x1 : 0x0; isar1_el1.apa = release->has(ArmExtension::FEAT_PAuth) ? 0x1 : 0x0; isar1_el1.jscvt = release->has(ArmExtension::FEAT_JSCVT) ? 0x1 : 0x0; @@ -5480,6 +5647,193 @@ ISA::initializeMiscRegMetadata() .monWrite(); InitReg(MISCREG_TLBI_RVALE3OS) .monWrite(); + InitReg(MISCREG_TLBI_VMALLE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbivmalle1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbivae1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_ASIDE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbiaside1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAAE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbivaae1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VALE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbivale1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAALE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbivaale1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VMALLE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbivmalle1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbivae1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_ASIDE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbiaside1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAAE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbivaae1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VALE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbivale1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAALE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbivaale1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VMALLE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbivmalle1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbivae1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_ASIDE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbiaside1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAAE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbivaae1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VALE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbivale1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_VAALE1NXS) + .faultWrite(EL1, faultTlbiNxsEL1<&HCR::ttlb, &HFGITR::tlbivaale1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_IPAS2E1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_IPAS2LE1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE2OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VAE2OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VALE2OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VMALLS12E1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_IPAS2E1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_IPAS2LE1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE2ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VAE2ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VALE2ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VMALLS12E1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_IPAS2E1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_IPAS2LE1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE2NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VAE2NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VALE2NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_VMALLS12E1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_ALLE3OSNXS) + .monWrite(); + InitReg(MISCREG_TLBI_VAE3OSNXS) + .monWrite(); + InitReg(MISCREG_TLBI_VALE3OSNXS) + .monWrite(); + InitReg(MISCREG_TLBI_ALLE3ISNXS) + .monWrite(); + InitReg(MISCREG_TLBI_VAE3ISNXS) + .monWrite(); + InitReg(MISCREG_TLBI_VALE3ISNXS) + .monWrite(); + InitReg(MISCREG_TLBI_ALLE3NXS) + .monWrite(); + InitReg(MISCREG_TLBI_VAE3NXS) + .monWrite(); + InitReg(MISCREG_TLBI_VALE3NXS) + .monWrite(); + + InitReg(MISCREG_TLBI_RVAE1NXS) + .faultWrite(EL1, faultHcrFgtInstEL1<&HCR::ttlb, &HFGITR::tlbirvae1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAAE1NXS) + .faultWrite(EL1, faultHcrFgtInstEL1<&HCR::ttlb, &HFGITR::tlbirvaae1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVALE1NXS) + .faultWrite(EL1, faultHcrFgtInstEL1<&HCR::ttlb, &HFGITR::tlbirvale1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAALE1NXS) + .faultWrite(EL1, faultHcrFgtInstEL1<&HCR::ttlb, &HFGITR::tlbirvaale1>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RIPAS2E1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RIPAS2LE1NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE2NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVALE2NXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE3NXS) + .monWrite(); + InitReg(MISCREG_TLBI_RVALE3NXS) + .monWrite(); + InitReg(MISCREG_TLBI_RVAE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbirvae1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAAE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbirvaae1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVALE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbirvale1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAALE1ISNXS) + .faultWrite(EL1, faultTlbiIsNxsEL1<&HFGITR::tlbirvaale1is>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RIPAS2E1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RIPAS2LE1ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE2ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVALE2ISNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE3ISNXS) + .monWrite(); + InitReg(MISCREG_TLBI_RVALE3ISNXS) + .monWrite(); + InitReg(MISCREG_TLBI_RVAE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbirvae1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAAE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbirvaae1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVALE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbirvale1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RVAALE1OSNXS) + .faultWrite(EL1, faultTlbiOsNxsEL1<&HFGITR::tlbirvaale1os>) + .writes(1).exceptUserMode(); + InitReg(MISCREG_TLBI_RIPAS2E1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RIPAS2LE1OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE2OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVALE2OSNXS) + .hypWrite().monWrite(); + InitReg(MISCREG_TLBI_RVAE3OSNXS) + .monWrite(); + InitReg(MISCREG_TLBI_RVALE3OSNXS) + .monWrite(); InitReg(MISCREG_PMINTENSET_EL1) .allPrivileges().exceptUserMode() .mapsTo(MISCREG_PMINTENSET); diff --git a/src/arch/arm/regs/misc.hh b/src/arch/arm/regs/misc.hh index 93005aa0e1..66ec06829b 100644 --- a/src/arch/arm/regs/misc.hh +++ b/src/arch/arm/regs/misc.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2023 Arm Limited + * Copyright (c) 2010-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -53,6 +53,18 @@ #include "debug/MiscRegs.hh" #include "dev/arm/generic_timer_miscregs_types.hh" +#define TLBI_VARIANTS(TLBI) \ + TLBI, \ + TLBI##NXS + +#define TLBI_CASE_VARIANTS(TLBI) \ + case TLBI: \ + case TLBI##NXS: + +#define TLBI_STR_VARIANTS(TLBI) \ + #TLBI, \ + #TLBI"nxs" + namespace gem5 { @@ -689,84 +701,84 @@ namespace ArmISA MISCREG_AT_S12E0W_Xt, MISCREG_AT_S1E3R_Xt, MISCREG_AT_S1E3W_Xt, - MISCREG_TLBI_VMALLE1IS, - MISCREG_TLBI_VMALLE1OS, - MISCREG_TLBI_VAE1IS, - MISCREG_TLBI_VAE1OS, - MISCREG_TLBI_ASIDE1IS, - MISCREG_TLBI_ASIDE1OS, - MISCREG_TLBI_VAAE1IS, - MISCREG_TLBI_VAAE1OS, - MISCREG_TLBI_VALE1IS, - MISCREG_TLBI_VALE1OS, - MISCREG_TLBI_VAALE1IS, - MISCREG_TLBI_VAALE1OS, - MISCREG_TLBI_VMALLE1, - MISCREG_TLBI_VAE1, - MISCREG_TLBI_ASIDE1, - MISCREG_TLBI_VAAE1, - MISCREG_TLBI_VALE1, - MISCREG_TLBI_VAALE1, - MISCREG_TLBI_IPAS2E1IS, - MISCREG_TLBI_IPAS2E1OS, - MISCREG_TLBI_IPAS2LE1IS, - MISCREG_TLBI_IPAS2LE1OS, - MISCREG_TLBI_ALLE2IS, - MISCREG_TLBI_ALLE2OS, - MISCREG_TLBI_VAE2IS, - MISCREG_TLBI_VAE2OS, - MISCREG_TLBI_ALLE1IS, - MISCREG_TLBI_ALLE1OS, - MISCREG_TLBI_VALE2IS, - MISCREG_TLBI_VALE2OS, - MISCREG_TLBI_VMALLS12E1IS, - MISCREG_TLBI_VMALLS12E1OS, - MISCREG_TLBI_IPAS2E1, - MISCREG_TLBI_IPAS2LE1, - MISCREG_TLBI_ALLE2, - MISCREG_TLBI_VAE2, - MISCREG_TLBI_ALLE1, - MISCREG_TLBI_VALE2, - MISCREG_TLBI_VMALLS12E1, - MISCREG_TLBI_ALLE3IS, - MISCREG_TLBI_ALLE3OS, - MISCREG_TLBI_VAE3IS, - MISCREG_TLBI_VAE3OS, - MISCREG_TLBI_VALE3IS, - MISCREG_TLBI_VALE3OS, - MISCREG_TLBI_ALLE3, - MISCREG_TLBI_VAE3, - MISCREG_TLBI_VALE3, - MISCREG_TLBI_RVAE1, - MISCREG_TLBI_RVAAE1, - MISCREG_TLBI_RVALE1, - MISCREG_TLBI_RVAALE1, - MISCREG_TLBI_RIPAS2E1, - MISCREG_TLBI_RIPAS2LE1, - MISCREG_TLBI_RVAE2, - MISCREG_TLBI_RVALE2, - MISCREG_TLBI_RVAE3, - MISCREG_TLBI_RVALE3, - MISCREG_TLBI_RVAE1IS, - MISCREG_TLBI_RVAAE1IS, - MISCREG_TLBI_RVALE1IS, - MISCREG_TLBI_RVAALE1IS, - MISCREG_TLBI_RIPAS2E1IS, - MISCREG_TLBI_RIPAS2LE1IS, - MISCREG_TLBI_RVAE2IS, - MISCREG_TLBI_RVALE2IS, - MISCREG_TLBI_RVAE3IS, - MISCREG_TLBI_RVALE3IS, - MISCREG_TLBI_RVAE1OS, - MISCREG_TLBI_RVAAE1OS, - MISCREG_TLBI_RVALE1OS, - MISCREG_TLBI_RVAALE1OS, - MISCREG_TLBI_RIPAS2E1OS, - MISCREG_TLBI_RIPAS2LE1OS, - MISCREG_TLBI_RVAE2OS, - MISCREG_TLBI_RVALE2OS, - MISCREG_TLBI_RVAE3OS, - MISCREG_TLBI_RVALE3OS, + TLBI_VARIANTS(MISCREG_TLBI_VMALLE1IS), + TLBI_VARIANTS(MISCREG_TLBI_VMALLE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VAE1IS), + TLBI_VARIANTS(MISCREG_TLBI_VAE1OS), + TLBI_VARIANTS(MISCREG_TLBI_ASIDE1IS), + TLBI_VARIANTS(MISCREG_TLBI_ASIDE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VAAE1IS), + TLBI_VARIANTS(MISCREG_TLBI_VAAE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VALE1IS), + TLBI_VARIANTS(MISCREG_TLBI_VALE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VAALE1IS), + TLBI_VARIANTS(MISCREG_TLBI_VAALE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VMALLE1), + TLBI_VARIANTS(MISCREG_TLBI_VAE1), + TLBI_VARIANTS(MISCREG_TLBI_ASIDE1), + TLBI_VARIANTS(MISCREG_TLBI_VAAE1), + TLBI_VARIANTS(MISCREG_TLBI_VALE1), + TLBI_VARIANTS(MISCREG_TLBI_VAALE1), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2E1IS), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2E1OS), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2LE1IS), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2LE1OS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE2IS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE2OS), + TLBI_VARIANTS(MISCREG_TLBI_VAE2IS), + TLBI_VARIANTS(MISCREG_TLBI_VAE2OS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE1IS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE1OS), + TLBI_VARIANTS(MISCREG_TLBI_VALE2IS), + TLBI_VARIANTS(MISCREG_TLBI_VALE2OS), + TLBI_VARIANTS(MISCREG_TLBI_VMALLS12E1IS), + TLBI_VARIANTS(MISCREG_TLBI_VMALLS12E1OS), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2E1), + TLBI_VARIANTS(MISCREG_TLBI_IPAS2LE1), + TLBI_VARIANTS(MISCREG_TLBI_ALLE2), + TLBI_VARIANTS(MISCREG_TLBI_VAE2), + TLBI_VARIANTS(MISCREG_TLBI_ALLE1), + TLBI_VARIANTS(MISCREG_TLBI_VALE2), + TLBI_VARIANTS(MISCREG_TLBI_VMALLS12E1), + TLBI_VARIANTS(MISCREG_TLBI_ALLE3IS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE3OS), + TLBI_VARIANTS(MISCREG_TLBI_VAE3IS), + TLBI_VARIANTS(MISCREG_TLBI_VAE3OS), + TLBI_VARIANTS(MISCREG_TLBI_VALE3IS), + TLBI_VARIANTS(MISCREG_TLBI_VALE3OS), + TLBI_VARIANTS(MISCREG_TLBI_ALLE3), + TLBI_VARIANTS(MISCREG_TLBI_VAE3), + TLBI_VARIANTS(MISCREG_TLBI_VALE3), + TLBI_VARIANTS(MISCREG_TLBI_RVAE1), + TLBI_VARIANTS(MISCREG_TLBI_RVAAE1), + TLBI_VARIANTS(MISCREG_TLBI_RVALE1), + TLBI_VARIANTS(MISCREG_TLBI_RVAALE1), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2E1), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2LE1), + TLBI_VARIANTS(MISCREG_TLBI_RVAE2), + TLBI_VARIANTS(MISCREG_TLBI_RVALE2), + TLBI_VARIANTS(MISCREG_TLBI_RVAE3), + TLBI_VARIANTS(MISCREG_TLBI_RVALE3), + TLBI_VARIANTS(MISCREG_TLBI_RVAE1IS), + TLBI_VARIANTS(MISCREG_TLBI_RVAAE1IS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE1IS), + TLBI_VARIANTS(MISCREG_TLBI_RVAALE1IS), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2E1IS), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2LE1IS), + TLBI_VARIANTS(MISCREG_TLBI_RVAE2IS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE2IS), + TLBI_VARIANTS(MISCREG_TLBI_RVAE3IS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE3IS), + TLBI_VARIANTS(MISCREG_TLBI_RVAE1OS), + TLBI_VARIANTS(MISCREG_TLBI_RVAAE1OS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE1OS), + TLBI_VARIANTS(MISCREG_TLBI_RVAALE1OS), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2E1OS), + TLBI_VARIANTS(MISCREG_TLBI_RIPAS2LE1OS), + TLBI_VARIANTS(MISCREG_TLBI_RVAE2OS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE2OS), + TLBI_VARIANTS(MISCREG_TLBI_RVAE3OS), + TLBI_VARIANTS(MISCREG_TLBI_RVALE3OS), MISCREG_PMINTENSET_EL1, MISCREG_PMINTENCLR_EL1, MISCREG_PMCR_EL0, @@ -2436,84 +2448,84 @@ namespace ArmISA "at_s12e0w_xt", "at_s1e3r_xt", "at_s1e3w_xt", - "tlbi_vmalle1is", - "tlbi_vmalle1os", - "tlbi_vae1is", - "tlbi_vae1os", - "lbi_aside1is_xt", - "tlbi_aside1os", - "tlbi_vaae1is", - "tlbi_vaae1os", - "tlbi_vale1is", - "tlbi_vale1os", - "tlbi_vaale1is", - "tlbi_vaale1os", - "tlbi_vmalle1", - "tlbi_vae1", - "tlbi_aside1", - "tlbi_vaae1", - "tlbi_vale1", - "tlbi_vaale1", - "tlbi_ipas2e1is", - "tlbi_ipas2e1os", - "tlbi_ipas2le1is", - "tlbi_ipas2le1os", - "tlbi_alle2is", - "tlbi_alle2os", - "tlbi_vae2is", - "tlbi_vae2os", - "tlbi_alle1is", - "tlbi_alle1os", - "tlbi_vale2is", - "tlbi_vale2os", - "tlbi_vmalls12e1is", - "tlbi_vmalls12e1os", - "tlbi_ipas2e1", - "tlbi_ipas2le1", - "tlbi_alle2", - "tlbi_vae2", - "tlbi_alle1", - "tlbi_vale2", - "tlbi_vmalls12e1", - "tlbi_alle3is", - "tlbi_alle3os", - "tlbi_vae3is", - "tlbi_vae3os", - "tlbi_vale3is", - "tlbi_vale3os", - "tlbi_alle3", - "tlbi_vae3", - "tlbi_vale3", - "tlbi_rvae1", - "tlbi_rvaae1", - "tlbi_rvale1", - "tlbi_rvaale1", - "tlbi_ripas2e1", - "tlbi_ripas2le1", - "tlbi_rvae2", - "tlbi_rvale2", - "tlbi_rvae3", - "tlbi_rvale3", - "tlbi_rvae1is", - "tlbi_rvaae1is", - "tlbi_rvale1is", - "tlbi_rvaale1is", - "tlbi_ripas2e1is", - "tlbi_ripas2le1is", - "tlbi_rvae2is", - "tlbi_rvale2is", - "tlbi_rvae3is", - "tlbi_rvale3is", - "tlbi_rvae1os", - "tlbi_rvaae1os", - "tlbi_rvale1os", - "tlbi_rvaale1os", - "tlbi_ripas2e1os", - "tlbi_ripas2le1os", - "tlbi_rvae2os", - "tlbi_rvale2os", - "tlbi_rvae3os", - "tlbi_rvale3os", + TLBI_STR_VARIANTS(tlbi_vmalle1is), + TLBI_STR_VARIANTS(tlbi_vmalle1os), + TLBI_STR_VARIANTS(tlbi_vae1is), + TLBI_STR_VARIANTS(tlbi_vae1os), + TLBI_STR_VARIANTS(lbi_aside1is_xt), + TLBI_STR_VARIANTS(tlbi_aside1os), + TLBI_STR_VARIANTS(tlbi_vaae1is), + TLBI_STR_VARIANTS(tlbi_vaae1os), + TLBI_STR_VARIANTS(tlbi_vale1is), + TLBI_STR_VARIANTS(tlbi_vale1os), + TLBI_STR_VARIANTS(tlbi_vaale1is), + TLBI_STR_VARIANTS(tlbi_vaale1os), + TLBI_STR_VARIANTS(tlbi_vmalle1), + TLBI_STR_VARIANTS(tlbi_vae1), + TLBI_STR_VARIANTS(tlbi_aside1), + TLBI_STR_VARIANTS(tlbi_vaae1), + TLBI_STR_VARIANTS(tlbi_vale1), + TLBI_STR_VARIANTS(tlbi_vaale1), + TLBI_STR_VARIANTS(tlbi_ipas2e1is), + TLBI_STR_VARIANTS(tlbi_ipas2e1os), + TLBI_STR_VARIANTS(tlbi_ipas2le1is), + TLBI_STR_VARIANTS(tlbi_ipas2le1os), + TLBI_STR_VARIANTS(tlbi_alle2is), + TLBI_STR_VARIANTS(tlbi_alle2os), + TLBI_STR_VARIANTS(tlbi_vae2is), + TLBI_STR_VARIANTS(tlbi_vae2os), + TLBI_STR_VARIANTS(tlbi_alle1is), + TLBI_STR_VARIANTS(tlbi_alle1os), + TLBI_STR_VARIANTS(tlbi_vale2is), + TLBI_STR_VARIANTS(tlbi_vale2os), + TLBI_STR_VARIANTS(tlbi_vmalls12e1is), + TLBI_STR_VARIANTS(tlbi_vmalls12e1os), + TLBI_STR_VARIANTS(tlbi_ipas2e1), + TLBI_STR_VARIANTS(tlbi_ipas2le1), + TLBI_STR_VARIANTS(tlbi_alle2), + TLBI_STR_VARIANTS(tlbi_vae2), + TLBI_STR_VARIANTS(tlbi_alle1), + TLBI_STR_VARIANTS(tlbi_vale2), + TLBI_STR_VARIANTS(tlbi_vmalls12e1), + TLBI_STR_VARIANTS(tlbi_alle3is), + TLBI_STR_VARIANTS(tlbi_alle3os), + TLBI_STR_VARIANTS(tlbi_vae3is), + TLBI_STR_VARIANTS(tlbi_vae3os), + TLBI_STR_VARIANTS(tlbi_vale3is), + TLBI_STR_VARIANTS(tlbi_vale3os), + TLBI_STR_VARIANTS(tlbi_alle3), + TLBI_STR_VARIANTS(tlbi_vae3), + TLBI_STR_VARIANTS(tlbi_vale3), + TLBI_STR_VARIANTS(tlbi_rvae1), + TLBI_STR_VARIANTS(tlbi_rvaae1), + TLBI_STR_VARIANTS(tlbi_rvale1), + TLBI_STR_VARIANTS(tlbi_rvaale1), + TLBI_STR_VARIANTS(tlbi_ripas2e1), + TLBI_STR_VARIANTS(tlbi_ripas2le1), + TLBI_STR_VARIANTS(tlbi_rvae2), + TLBI_STR_VARIANTS(tlbi_rvale2), + TLBI_STR_VARIANTS(tlbi_rvae3), + TLBI_STR_VARIANTS(tlbi_rvale3), + TLBI_STR_VARIANTS(tlbi_rvae1is), + TLBI_STR_VARIANTS(tlbi_rvaae1is), + TLBI_STR_VARIANTS(tlbi_rvale1is), + TLBI_STR_VARIANTS(tlbi_rvaale1is), + TLBI_STR_VARIANTS(tlbi_ripas2e1is), + TLBI_STR_VARIANTS(tlbi_ripas2le1is), + TLBI_STR_VARIANTS(tlbi_rvae2is), + TLBI_STR_VARIANTS(tlbi_rvale2is), + TLBI_STR_VARIANTS(tlbi_rvae3is), + TLBI_STR_VARIANTS(tlbi_rvale3is), + TLBI_STR_VARIANTS(tlbi_rvae1os), + TLBI_STR_VARIANTS(tlbi_rvaae1os), + TLBI_STR_VARIANTS(tlbi_rvale1os), + TLBI_STR_VARIANTS(tlbi_rvaale1os), + TLBI_STR_VARIANTS(tlbi_ripas2e1os), + TLBI_STR_VARIANTS(tlbi_ripas2le1os), + TLBI_STR_VARIANTS(tlbi_rvae2os), + TLBI_STR_VARIANTS(tlbi_rvale2os), + TLBI_STR_VARIANTS(tlbi_rvae3os), + TLBI_STR_VARIANTS(tlbi_rvale3os), "pmintenset_el1", "pmintenclr_el1", "pmcr_el0", diff --git a/src/arch/arm/regs/misc_types.hh b/src/arch/arm/regs/misc_types.hh index 0f3a9f558c..5bb3b59847 100644 --- a/src/arch/arm/regs/misc_types.hh +++ b/src/arch/arm/regs/misc_types.hh @@ -133,6 +133,7 @@ namespace ArmISA EndBitUnion(AA64ISAR0) BitUnion64(AA64ISAR1) + Bitfield<59, 56> xs; Bitfield<55, 52> i8mm; Bitfield<43, 40> specres; Bitfield<39, 36> sb; @@ -1088,6 +1089,8 @@ namespace ArmISA BitUnion64(HCRX) Bitfield<15> sctlr2En; Bitfield<14> tcr2En; + Bitfield<4> fgtnxs; + Bitfield<3> fnxs; EndBitUnion(HCRX) BitUnion64(MPAMIDR) diff --git a/src/arch/arm/stage2_lookup.cc b/src/arch/arm/stage2_lookup.cc index 0d9942a744..35239ac4e0 100644 --- a/src/arch/arm/stage2_lookup.cc +++ b/src/arch/arm/stage2_lookup.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2013, 2016, 2018 ARM Limited + * Copyright (c) 2010-2013, 2016, 2018, 2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -162,6 +162,14 @@ Stage2LookUp::mergeTe(BaseMMU::Mode mode) stage1Te.shareable = true; stage1Te.outerShareable = true; } + + if (stage1Te.mtype == TlbEntry::MemoryType::Normal && + stage1Te.innerAttrs == 3 && + stage1Te.outerAttrs == 3) { + stage1Te.xs = false; + } else { + stage1Te.xs = stage1Te.xs && stage2Te->xs; + } stage1Te.updateAttributes(); } diff --git a/src/arch/arm/table_walker.cc b/src/arch/arm/table_walker.cc index f16c065839..f2b9f03a4b 100644 --- a/src/arch/arm/table_walker.cc +++ b/src/arch/arm/table_walker.cc @@ -1479,6 +1479,13 @@ TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, te.attributes |= (uint64_t) attr << 56; } +bool +TableWalker::uncacheableFromAttrs(uint8_t attrs) +{ + return !bits(attrs, 2) || // Write-through + attrs == 0b0100; // NonCacheable +} + void TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, LongDescriptor &l_descriptor) @@ -1511,6 +1518,9 @@ TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, // but for performance reasons not optimal. te.nonCacheable = (attr_hi == 1) || (attr_hi == 2) || (attr_lo == 1) || (attr_lo == 2); + + // To be used when merging stage1 and astage 2 attributes + te.xs = !l_descriptor.fnxs(); } } else { uint8_t attrIndx = l_descriptor.attrIndx(); @@ -1540,32 +1550,31 @@ TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, attr_lo = bits(attr, 3, 0); attr_hi = bits(attr, 7, 4); - // Memory type - te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; - - // Cacheability - te.nonCacheable = false; - if (te.mtype == TlbEntry::MemoryType::Device) { // Device memory - te.nonCacheable = true; - } // Treat write-through memory as uncacheable, this is safe // but for performance reasons not optimal. - switch (attr_hi) { - case 0x1 ... 0x3: // Normal Memory, Outer Write-through transient - case 0x4: // Normal memory, Outer Non-cacheable - case 0x8 ... 0xb: // Normal Memory, Outer Write-through non-transient + switch (attr) { + case 0b00000000 ... 0b00001111: // Device Memory + te.mtype = TlbEntry::MemoryType::Device; te.nonCacheable = true; - } - switch (attr_lo) { - case 0x1 ... 0x3: // Normal Memory, Inner Write-through transient - case 0x9 ... 0xb: // Normal Memory, Inner Write-through non-transient - warn_if(!attr_hi, "Unpredictable behavior"); - [[fallthrough]]; - case 0x4: // Device-nGnRE memory or - // Normal memory, Inner Non-cacheable - case 0x8: // Device-nGRE memory or - // Normal memory, Inner Write-through non-transient + te.xs = !bits(attr, 0); + break; + case 0b01000000: // Normal memory, Non-cacheable + te.mtype = TlbEntry::MemoryType::Normal; te.nonCacheable = true; + te.xs = false; + break; + case 0b10100000: // Normal memory, Write-through + te.mtype = TlbEntry::MemoryType::Normal; + te.nonCacheable = true; + te.xs = false; + break; + default: + te.mtype = TlbEntry::MemoryType::Normal; + te.nonCacheable = uncacheableFromAttrs(attr_hi) || + uncacheableFromAttrs(attr_lo); + // XS is 0 only for write-back regions (cacheable) + te.xs = te.nonCacheable; + break; } te.shareable = sh == 2; @@ -1594,6 +1603,9 @@ TableWalker::memAttrsWalkAArch64(TlbEntry &te) te.nonCacheable = (te.outerAttrs == 0 || te.outerAttrs == 2) && (te.innerAttrs == 0 || te.innerAttrs == 2); } + + // XS is 0 only for write-back regions (cacheable) + te.xs = te.nonCacheable; } void diff --git a/src/arch/arm/table_walker.hh b/src/arch/arm/table_walker.hh index efd9e92d50..f8a8ec1b5d 100644 --- a/src/arch/arm/table_walker.hh +++ b/src/arch/arm/table_walker.hh @@ -678,6 +678,14 @@ class TableWalker : public ClockedObject return !bits(data, 11); } + /** FNXS for FEAT_XS only */ + bool + fnxs() const + { + assert((type() == Block || type() == Page)); + return bits(data, 11); + } + /** Returns true if the access flag (AF) is set. */ bool af() const @@ -1145,6 +1153,7 @@ class TableWalker : public ClockedObject void memAttrsAArch64(ThreadContext *tc, TlbEntry &te, LongDescriptor &lDescriptor); void memAttrsWalkAArch64(TlbEntry &te); + bool uncacheableFromAttrs(uint8_t attrs); static LookupLevel toLookupLevel(uint8_t lookup_level_as_int); diff --git a/src/arch/arm/tlb.cc b/src/arch/arm/tlb.cc index 4bceb28383..900b7d328d 100644 --- a/src/arch/arm/tlb.cc +++ b/src/arch/arm/tlb.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2013, 2016-2023 Arm Limited + * Copyright (c) 2010-2013, 2016-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -162,7 +162,7 @@ TLB::lookup(const Lookup &lookup_data) DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x secure %d " "ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d " - "regime: %s\n", + "xs: %d regime: %s\n", lookup_data.va, lookup_data.asn, retval ? "hit" : "miss", lookup_data.vmid, lookup_data.secure, retval ? retval->pfn : 0, retval ? retval->size : 0, @@ -170,7 +170,8 @@ TLB::lookup(const Lookup &lookup_data) retval ? retval->ap : 0, retval ? retval->ns : 0, retval ? retval->nstid : 0, retval ? retval->global : 0, retval ? retval->asid : 0, - retval ? regimeToStr(retval->regime) : 0); + retval ? retval->xs : 0, + retval ? regimeToStr(retval->regime) : "None"); // Updating stats if this was not a functional lookup if (!lookup_data.functional) { @@ -242,20 +243,20 @@ TLB::insert(TlbEntry &entry) { DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x" " asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d" - " ap:%#x domain:%#x ns:%d nstid:%d, regime: %s\n", entry.pfn, + " ap:%#x domain:%#x ns:%d nstid:%d, xs:%d regime: %s\n", entry.pfn, entry.size, entry.vpn, entry.asid, entry.vmid, entry.N, entry.global, entry.valid, entry.nonCacheable, entry.xn, entry.ap, static_cast(entry.domain), entry.ns, - entry.nstid, regimeToStr(entry.regime)); + entry.nstid, entry.xs, regimeToStr(entry.regime)); if (table[size - 1].valid) DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x " - "size: %#x ap:%d ns:%d nstid:%d g:%d regime: %s\n", + "size: %#x ap:%d ns:%d nstid:%d g:%d xs:%d regime: %s\n", table[size-1].vpn << table[size-1].N, table[size-1].asid, table[size-1].vmid, table[size-1].pfn << table[size-1].N, table[size-1].size, table[size-1].ap, table[size-1].ns, table[size-1].nstid, table[size-1].global, - regimeToStr(table[size-1].regime)); + table[size-1].xs, regimeToStr(table[size-1].regime)); // inserting to MRU position and evicting the LRU one for (int i = size - 1; i > 0; --i) diff --git a/src/arch/arm/tlbi_op.cc b/src/arch/arm/tlbi_op.cc index 1080a64e4b..36d6459044 100644 --- a/src/arch/arm/tlbi_op.cc +++ b/src/arch/arm/tlbi_op.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2023 Arm Limited + * Copyright (c) 2018-2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -45,6 +45,12 @@ namespace gem5 namespace ArmISA { +bool +TLBIOp::match(TlbEntry* te, vmid_t vmid) const +{ + return matchEntry(te, vmid) && (attr != Attr::ExcludeXS || !te->xs); +} + void TLBIALL::operator()(ThreadContext* tc) { @@ -61,7 +67,7 @@ TLBIALL::operator()(ThreadContext* tc) } bool -TLBIALL::match(TlbEntry* te, vmid_t vmid) const +TLBIALL::matchEntry(TlbEntry* te, vmid_t vmid) const { return te->valid && secureLookup == !te->nstid && (te->vmid == vmid || el2Enabled) && @@ -76,9 +82,9 @@ ITLBIALL::operator()(ThreadContext* tc) } bool -ITLBIALL::match(TlbEntry* te, vmid_t vmid) const +ITLBIALL::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIALL::match(te, vmid) && (te->type & TypeTLB::instruction); + return TLBIALL::matchEntry(te, vmid) && (te->type & TypeTLB::instruction); } void @@ -89,9 +95,9 @@ DTLBIALL::operator()(ThreadContext* tc) } bool -DTLBIALL::match(TlbEntry* te, vmid_t vmid) const +DTLBIALL::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIALL::match(te, vmid) && (te->type & TypeTLB::data); + return TLBIALL::matchEntry(te, vmid) && (te->type & TypeTLB::data); } void @@ -107,7 +113,7 @@ TLBIALLEL::operator()(ThreadContext* tc) } bool -TLBIALLEL::match(TlbEntry* te, vmid_t vmid) const +TLBIALLEL::matchEntry(TlbEntry* te, vmid_t vmid) const { return te->valid && secureLookup == !te->nstid && te->checkRegime(targetRegime); @@ -128,7 +134,7 @@ TLBIVMALL::operator()(ThreadContext* tc) } bool -TLBIVMALL::match(TlbEntry* te, vmid_t vmid) const +TLBIVMALL::matchEntry(TlbEntry* te, vmid_t vmid) const { return te->valid && secureLookup == !te->nstid && te->checkRegime(targetRegime) && @@ -148,7 +154,7 @@ TLBIASID::operator()(ThreadContext* tc) } bool -TLBIASID::match(TlbEntry* te, vmid_t vmid) const +TLBIASID::matchEntry(TlbEntry* te, vmid_t vmid) const { return te->valid && te->asid == asid && secureLookup == !te->nstid && @@ -164,9 +170,9 @@ ITLBIASID::operator()(ThreadContext* tc) } bool -ITLBIASID::match(TlbEntry* te, vmid_t vmid) const +ITLBIASID::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIASID::match(te, vmid) && (te->type & TypeTLB::instruction); + return TLBIASID::matchEntry(te, vmid) && (te->type & TypeTLB::instruction); } void @@ -177,9 +183,9 @@ DTLBIASID::operator()(ThreadContext* tc) } bool -DTLBIASID::match(TlbEntry* te, vmid_t vmid) const +DTLBIASID::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIASID::match(te, vmid) && (te->type & TypeTLB::data); + return TLBIASID::matchEntry(te, vmid) && (te->type & TypeTLB::data); } void @@ -194,7 +200,7 @@ TLBIALLN::operator()(ThreadContext* tc) } bool -TLBIALLN::match(TlbEntry* te, vmid_t vmid) const +TLBIALLN::matchEntry(TlbEntry* te, vmid_t vmid) const { return te->valid && te->nstid && te->checkRegime(targetRegime); @@ -226,7 +232,7 @@ TLBIMVAA::operator()(ThreadContext* tc) } bool -TLBIMVAA::match(TlbEntry* te, vmid_t vmid) const +TLBIMVAA::matchEntry(TlbEntry* te, vmid_t vmid) const { TlbEntry::Lookup lookup_data = lookupGen(vmid); @@ -261,7 +267,7 @@ TLBIMVA::operator()(ThreadContext* tc) } bool -TLBIMVA::match(TlbEntry* te, vmid_t vmid) const +TLBIMVA::matchEntry(TlbEntry* te, vmid_t vmid) const { TlbEntry::Lookup lookup_data = lookupGen(vmid); @@ -275,9 +281,9 @@ ITLBIMVA::operator()(ThreadContext* tc) } bool -ITLBIMVA::match(TlbEntry* te, vmid_t vmid) const +ITLBIMVA::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIMVA::match(te, vmid) && (te->type & TypeTLB::instruction); + return TLBIMVA::matchEntry(te, vmid) && (te->type & TypeTLB::instruction); } void @@ -287,9 +293,9 @@ DTLBIMVA::operator()(ThreadContext* tc) } bool -DTLBIMVA::match(TlbEntry* te, vmid_t vmid) const +DTLBIMVA::matchEntry(TlbEntry* te, vmid_t vmid) const { - return TLBIMVA::match(te, vmid) && (te->type & TypeTLB::data); + return TLBIMVA::matchEntry(te, vmid) && (te->type & TypeTLB::data); } void @@ -304,7 +310,7 @@ TLBIIPA::operator()(ThreadContext* tc) } bool -TLBIRMVA::match(TlbEntry* te, vmid_t vmid) const +TLBIRMVA::matchEntry(TlbEntry* te, vmid_t vmid) const { TlbEntry::Lookup lookup_data = lookupGen(vmid); lookup_data.size = rangeSize(); @@ -320,7 +326,7 @@ TLBIRMVA::match(TlbEntry* te, vmid_t vmid) const } bool -TLBIRMVAA::match(TlbEntry* te, vmid_t vmid) const +TLBIRMVAA::matchEntry(TlbEntry* te, vmid_t vmid) const { TlbEntry::Lookup lookup_data = lookupGen(vmid); lookup_data.size = rangeSize(); diff --git a/src/arch/arm/tlbi_op.hh b/src/arch/arm/tlbi_op.hh index 26105abfaf..f505dc7686 100644 --- a/src/arch/arm/tlbi_op.hh +++ b/src/arch/arm/tlbi_op.hh @@ -57,8 +57,14 @@ namespace ArmISA { class TLBIOp { public: - TLBIOp(TranslationRegime _target_regime, bool _secure) - : secureLookup(_secure), targetRegime(_target_regime) + enum class Attr + { + None, + ExcludeXS + }; + + TLBIOp(TranslationRegime _target_regime, bool _secure, Attr _attr) + : secureLookup(_secure), targetRegime(_target_regime), attr(_attr) {} virtual ~TLBIOp() {} @@ -76,7 +82,9 @@ class TLBIOp (*this)(oc); } - virtual bool match(TlbEntry *entry, vmid_t curr_vmid) const = 0; + bool match(TlbEntry *entry, vmid_t curr_vmid) const; + + virtual bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const = 0; /** * Return true if the TLBI op needs to flush stage1 @@ -102,20 +110,22 @@ class TLBIOp bool secureLookup; TranslationRegime targetRegime; + Attr attr; }; /** TLB Invalidate All */ class TLBIALL : public TLBIOp { public: - TLBIALL(TranslationRegime _target_regime, bool _secure) - : TLBIOp(_target_regime, _secure), el2Enabled(false), + TLBIALL(TranslationRegime _target_regime, bool _secure, + Attr _attr=Attr::None) + : TLBIOp(_target_regime, _secure, _attr), el2Enabled(false), currentEL(EL0) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; bool stage2Flush() const override @@ -128,7 +138,7 @@ class TLBIALL : public TLBIOp TLBIALL makeStage2() const { - return TLBIALL(targetRegime, secureLookup); + return TLBIALL(targetRegime, secureLookup, attr); } bool el2Enabled; @@ -145,7 +155,7 @@ class ITLBIALL : public TLBIALL void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate All */ @@ -158,20 +168,20 @@ class DTLBIALL : public TLBIALL void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Implementaton of AArch64 TLBI ALLE(1,2,3)(IS) instructions */ class TLBIALLEL : public TLBIOp { public: - TLBIALLEL(TranslationRegime _target_regime, bool _secure) - : TLBIOp(_target_regime, _secure) + TLBIALLEL(TranslationRegime _target_regime, bool _secure, Attr _attr) + : TLBIOp(_target_regime, _secure, _attr) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; bool stage2Flush() const override @@ -184,7 +194,7 @@ class TLBIALLEL : public TLBIOp TLBIALLEL makeStage2() const { - return TLBIALLEL(targetRegime, secureLookup); + return TLBIALLEL(targetRegime, secureLookup, attr); } }; @@ -193,14 +203,15 @@ class TLBIALLEL : public TLBIOp class TLBIVMALL : public TLBIOp { public: - TLBIVMALL(TranslationRegime _target_regime, bool _secure, bool _stage2) - : TLBIOp(_target_regime, _secure), el2Enabled(false), + TLBIVMALL(TranslationRegime _target_regime, bool _secure, + bool _stage2, Attr _attr) + : TLBIOp(_target_regime, _secure, _attr), el2Enabled(false), stage2(_stage2) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; bool stage2Flush() const override @@ -211,7 +222,7 @@ class TLBIVMALL : public TLBIOp TLBIVMALL makeStage2() const { - return TLBIVMALL(targetRegime, secureLookup, false); + return TLBIVMALL(targetRegime, secureLookup, false, attr); } bool el2Enabled; @@ -222,14 +233,15 @@ class TLBIVMALL : public TLBIOp class TLBIASID : public TLBIOp { public: - TLBIASID(TranslationRegime _target_regime, bool _secure, uint16_t _asid) - : TLBIOp(_target_regime, _secure), asid(_asid), + TLBIASID(TranslationRegime _target_regime, bool _secure, + uint16_t _asid, Attr _attr=Attr::None) + : TLBIOp(_target_regime, _secure, _attr), asid(_asid), el2Enabled(false) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; uint16_t asid; bool el2Enabled; @@ -245,7 +257,7 @@ class ITLBIASID : public TLBIASID void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate by ASID match */ @@ -258,7 +270,7 @@ class DTLBIASID : public TLBIASID void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** TLB Invalidate All, Non-Secure */ @@ -266,12 +278,12 @@ class TLBIALLN : public TLBIOp { public: TLBIALLN(TranslationRegime _target_regime) - : TLBIOp(_target_regime, false) + : TLBIOp(_target_regime, false, Attr::None) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; bool stage2Flush() const override @@ -293,14 +305,14 @@ class TLBIMVAA : public TLBIOp TlbEntry::Lookup lookupGen(vmid_t vmid) const; public: TLBIMVAA(TranslationRegime _target_regime, bool _secure, - Addr _addr, bool last_level) - : TLBIOp(_target_regime, _secure), addr(_addr), + Addr _addr, bool last_level, Attr _attr=Attr::None) + : TLBIOp(_target_regime, _secure, _attr), addr(_addr), lastLevel(last_level) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; Addr addr; bool lastLevel; @@ -314,14 +326,15 @@ class TLBIMVA : public TLBIOp public: TLBIMVA(TranslationRegime _target_regime, bool _secure, - Addr _addr, uint16_t _asid, bool last_level) - : TLBIOp(_target_regime, _secure), addr(_addr), asid(_asid), + Addr _addr, uint16_t _asid, bool last_level, + Attr _attr=Attr::None) + : TLBIOp(_target_regime, _secure, _attr), addr(_addr), asid(_asid), lastLevel(last_level) {} void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; Addr addr; uint16_t asid; @@ -339,7 +352,7 @@ class ITLBIMVA : public TLBIMVA void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** Data TLB Invalidate by VA */ @@ -353,7 +366,7 @@ class DTLBIMVA : public TLBIMVA void operator()(ThreadContext* tc) override; - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; class TLBIRange @@ -416,14 +429,15 @@ class TLBIIPA : public TLBIOp { public: TLBIIPA(TranslationRegime _target_regime, bool _secure, Addr _addr, - bool last_level) - : TLBIOp(_target_regime, _secure), addr(_addr), lastLevel(last_level) + bool last_level, Attr _attr=Attr::None) + : TLBIOp(_target_regime, _secure, _attr), + addr(_addr), lastLevel(last_level) {} void operator()(ThreadContext* tc) override; bool - match(TlbEntry *entry, vmid_t curr_vmid) const override + matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override { panic("This shouldn't be called\n"); } @@ -438,7 +452,7 @@ class TLBIIPA : public TLBIOp virtual TLBIMVAA makeStage2() const { - return TLBIMVAA(targetRegime, secureLookup, addr, lastLevel); + return TLBIMVAA(targetRegime, secureLookup, addr, lastLevel, attr); } Addr addr; @@ -450,12 +464,13 @@ class TLBIRMVA : public TLBIRange, public TLBIMVA { public: TLBIRMVA(TranslationRegime _target_regime, bool _secure, - RegVal val, uint16_t _asid, bool last_level) + RegVal val, uint16_t _asid, bool last_level, Attr _attr) : TLBIRange(val), - TLBIMVA(_target_regime, _secure, startAddress(), _asid, last_level) + TLBIMVA(_target_regime, _secure, startAddress(), + _asid, last_level, _attr) {} - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** TLB Range Invalidate by VA, All ASIDs */ @@ -463,12 +478,12 @@ class TLBIRMVAA : public TLBIRange, public TLBIMVAA { public: TLBIRMVAA(TranslationRegime _target_regime, bool _secure, - RegVal val, bool last_level) + RegVal val, bool last_level, Attr _attr) : TLBIRange(val), - TLBIMVAA(_target_regime, _secure, startAddress(), last_level) + TLBIMVAA(_target_regime, _secure, startAddress(), last_level, _attr) {} - bool match(TlbEntry *entry, vmid_t curr_vmid) const override; + bool matchEntry(TlbEntry *entry, vmid_t curr_vmid) const override; }; /** TLB Range Invalidate by VA, All ASIDs */ @@ -476,15 +491,16 @@ class TLBIRIPA : public TLBIRange, public TLBIIPA { public: TLBIRIPA(TranslationRegime _target_regime, bool _secure, - RegVal val, bool last_level) + RegVal val, bool last_level, Attr _attr) : TLBIRange(val), - TLBIIPA(_target_regime, _secure, startAddress(), last_level) + TLBIIPA(_target_regime, _secure, startAddress(), last_level, _attr) {} virtual TLBIMVAA makeStage2() const { - return TLBIRMVAA(targetRegime, secureLookup, rangeData, lastLevel); + return TLBIRMVAA(targetRegime, secureLookup, rangeData, + lastLevel, attr); } };