From 870f93301fec40ac624d3299269f6f3e2504ebc9 Mon Sep 17 00:00:00 2001 From: Giacomo Travaglini Date: Wed, 9 Jun 2021 15:04:56 +0100 Subject: [PATCH] arch-arm: Move translation logic from the ArmTLB to the ArmMMU This patch is moving most of the TLB code to the MMU. In this way the TLB stops being the main translating agent and becomes a simple "passive" translation cache. All the logic behind virtual memory translation, like * Checking permission/alignment * Issuing page table walks * etc Is now embedded in the MMU model. This will allow us to stack multiple TLBs and to compose arbitrary hierarchies as their sole purpose now is to cache translations JIRA: https://gem5.atlassian.net/browse/GEM5-790 Change-Id: I687c639a56263d5e3bb6633dd8c9666c85edba3a Signed-off-by: Giacomo Travaglini Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/48141 Tested-by: kokoro --- src/arch/arm/ArmMMU.py | 9 +- src/arch/arm/insts/macromem.cc | 8 +- src/arch/arm/insts/macromem.hh | 4 +- src/arch/arm/insts/mem64.cc | 2 +- src/arch/arm/insts/sve_mem.hh | 8 +- src/arch/arm/isa.cc | 68 +- src/arch/arm/isa.hh | 6 +- src/arch/arm/isa/formats/fp.isa | 6 +- src/arch/arm/isa/insts/amo64.isa | 2 +- src/arch/arm/isa/insts/branch.isa | 8 +- src/arch/arm/isa/insts/ldr.isa | 10 +- src/arch/arm/isa/insts/ldr64.isa | 4 +- src/arch/arm/isa/insts/str.isa | 10 +- src/arch/arm/isa/insts/str64.isa | 4 +- src/arch/arm/isa/templates/sve_mem.isa | 8 +- src/arch/arm/mmu.cc | 1462 +++++++++++++++++++++++- src/arch/arm/mmu.hh | 258 ++++- src/arch/arm/stage2_lookup.cc | 31 +- src/arch/arm/stage2_lookup.hh | 17 +- src/arch/arm/table_walker.cc | 78 +- src/arch/arm/table_walker.hh | 15 +- src/arch/arm/tlb.cc | 1326 +-------------------- src/arch/arm/tlb.hh | 212 +--- src/arch/arm/tracers/tarmac_parser.cc | 2 +- 24 files changed, 1846 insertions(+), 1712 deletions(-) diff --git a/src/arch/arm/ArmMMU.py b/src/arch/arm/ArmMMU.py index 1880f6f6c7..84122d572d 100644 --- a/src/arch/arm/ArmMMU.py +++ b/src/arch/arm/ArmMMU.py @@ -1,6 +1,6 @@ # -*- mode:python -*- -# Copyright (c) 2020 ARM Limited +# Copyright (c) 2020-2021 Arm Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -80,10 +80,15 @@ class ArmMMU(BaseMMU): stage2_dtb_walker = Param.ArmTableWalker( ArmStage2TableWalker(), "HW Table walker") + sys = Param.System(Parent.any, "system object parameter") + @classmethod def walkerPorts(cls): - return ["mmu.itb_walker.port", "mmu.dtb_walker.port"] + return ["mmu.itb_walker.port", "mmu.dtb_walker.port", + "mmu.stage2_itb_walker.port", "mmu.stage2_dtb_walker.port"] def connectWalkerPorts(self, iport, dport): self.itb_walker.port = iport self.dtb_walker.port = dport + self.stage2_itb_walker.port = iport + self.stage2_dtb_walker.port = dport diff --git a/src/arch/arm/insts/macromem.cc b/src/arch/arm/insts/macromem.cc index 36df626fec..ce63c8e2c6 100644 --- a/src/arch/arm/insts/macromem.cc +++ b/src/arch/arm/insts/macromem.cc @@ -1146,7 +1146,7 @@ VldMultOp64::VldMultOp64(const char *mnem, ExtMachInst machInst, microOps = new StaticInstPtr[numMicroops]; unsigned uopIdx = 0; - uint32_t memaccessFlags = (TLB::ArmFlags)eSize | TLB::AllowUnaligned; + uint32_t memaccessFlags = (MMU::ArmFlags)eSize | MMU::AllowUnaligned; int i = 0; for (; i < numMemMicroops - 1; ++i) { @@ -1254,7 +1254,7 @@ VstMultOp64::VstMultOp64(const char *mnem, ExtMachInst machInst, } } - uint32_t memaccessFlags = (TLB::ArmFlags)eSize | TLB::AllowUnaligned; + uint32_t memaccessFlags = (MMU::ArmFlags)eSize | MMU::AllowUnaligned; int i = 0; for (; i < numMemMicroops - 1; ++i) { @@ -1322,7 +1322,7 @@ VldSingleOp64::VldSingleOp64(const char *mnem, ExtMachInst machInst, microOps = new StaticInstPtr[numMicroops]; unsigned uopIdx = 0; - uint32_t memaccessFlags = (TLB::ArmFlags)eSize | TLB::AllowUnaligned; + uint32_t memaccessFlags = (MMU::ArmFlags)eSize | MMU::AllowUnaligned; int i = 0; for (; i < numMemMicroops - 1; ++i) { @@ -1401,7 +1401,7 @@ VstSingleOp64::VstSingleOp64(const char *mnem, ExtMachInst machInst, numStructElems, index, i /* step */, replicate); } - uint32_t memaccessFlags = (TLB::ArmFlags)eSize | TLB::AllowUnaligned; + uint32_t memaccessFlags = (MMU::ArmFlags)eSize | MMU::AllowUnaligned; int i = 0; for (; i < numMemMicroops - 1; ++i) { diff --git a/src/arch/arm/insts/macromem.hh b/src/arch/arm/insts/macromem.hh index b1c2438a6a..8541131fd4 100644 --- a/src/arch/arm/insts/macromem.hh +++ b/src/arch/arm/insts/macromem.hh @@ -395,7 +395,7 @@ class MicroMemOp : public MicroIntImmOp MicroMemOp(const char *mnem, ExtMachInst machInst, OpClass __opClass, RegIndex _ura, RegIndex _urb, bool _up, uint8_t _imm) : MicroIntImmOp(mnem, machInst, __opClass, _ura, _urb, _imm), - up(_up), memAccessFlags(TLB::AlignWord) + up(_up), memAccessFlags(MMU::AlignWord) { } @@ -416,7 +416,7 @@ class MicroMemPairOp : public MicroOp bool _up, uint8_t _imm) : MicroOp(mnem, machInst, __opClass), dest(_dreg1), dest2(_dreg2), urb(_base), up(_up), imm(_imm), - memAccessFlags(TLB::AlignWord) + memAccessFlags(MMU::AlignWord) { } diff --git a/src/arch/arm/insts/mem64.cc b/src/arch/arm/insts/mem64.cc index 091987e746..f21c43064e 100644 --- a/src/arch/arm/insts/mem64.cc +++ b/src/arch/arm/insts/mem64.cc @@ -78,7 +78,7 @@ Memory64::setExcAcRel(bool exclusive, bool acrel) if (exclusive) memAccessFlags |= Request::LLSC; else - memAccessFlags |= ArmISA::TLB::AllowUnaligned; + memAccessFlags |= ArmISA::MMU::AllowUnaligned; if (acrel) { flags[IsWriteBarrier] = true; flags[IsReadBarrier] = true; diff --git a/src/arch/arm/insts/sve_mem.hh b/src/arch/arm/insts/sve_mem.hh index 17176d0eb0..167591c523 100644 --- a/src/arch/arm/insts/sve_mem.hh +++ b/src/arch/arm/insts/sve_mem.hh @@ -64,7 +64,7 @@ class SveMemVecFillSpill : public ArmStaticInst IntRegIndex _base, uint64_t _imm) : ArmStaticInst(mnem, _machInst, __opClass), dest(_dest), base(_base), imm(_imm), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { baseIsSP = isSP(_base); } @@ -90,7 +90,7 @@ class SveMemPredFillSpill : public ArmStaticInst IntRegIndex _base, uint64_t _imm) : ArmStaticInst(mnem, _machInst, __opClass), dest(_dest), base(_base), imm(_imm), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { baseIsSP = isSP(_base); } @@ -117,7 +117,7 @@ class SveContigMemSS : public ArmStaticInst IntRegIndex _offset) : ArmStaticInst(mnem, _machInst, __opClass), dest(_dest), gp(_gp), base(_base), offset(_offset), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { baseIsSP = isSP(_base); } @@ -144,7 +144,7 @@ class SveContigMemSI : public ArmStaticInst uint64_t _imm) : ArmStaticInst(mnem, _machInst, __opClass), dest(_dest), gp(_gp), base(_base), imm(_imm), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { baseIsSP = isSP(_base); } diff --git a/src/arch/arm/isa.cc b/src/arch/arm/isa.cc index 8157b0f2ff..cb3d4ecde2 100644 --- a/src/arch/arm/isa.cc +++ b/src/arch/arm/isa.cc @@ -2076,46 +2076,46 @@ ISA::setMiscReg(int misc_reg, RegVal val) misc_reg = MISCREG_IFAR_S; break; case MISCREG_ATS1CPR: - addressTranslation(TLB::S1CTran, BaseMMU::Read, 0, val); + addressTranslation(MMU::S1CTran, BaseMMU::Read, 0, val); return; case MISCREG_ATS1CPW: - addressTranslation(TLB::S1CTran, BaseMMU::Write, 0, val); + addressTranslation(MMU::S1CTran, BaseMMU::Write, 0, val); return; case MISCREG_ATS1CUR: - addressTranslation(TLB::S1CTran, BaseMMU::Read, - TLB::UserMode, val); + addressTranslation(MMU::S1CTran, BaseMMU::Read, + MMU::UserMode, val); return; case MISCREG_ATS1CUW: - addressTranslation(TLB::S1CTran, BaseMMU::Write, - TLB::UserMode, val); + addressTranslation(MMU::S1CTran, BaseMMU::Write, + MMU::UserMode, val); return; case MISCREG_ATS12NSOPR: if (!haveSecurity) panic("Security Extensions required for ATS12NSOPR"); - addressTranslation(TLB::S1S2NsTran, BaseMMU::Read, 0, val); + addressTranslation(MMU::S1S2NsTran, BaseMMU::Read, 0, val); return; case MISCREG_ATS12NSOPW: if (!haveSecurity) panic("Security Extensions required for ATS12NSOPW"); - addressTranslation(TLB::S1S2NsTran, BaseMMU::Write, 0, val); + addressTranslation(MMU::S1S2NsTran, BaseMMU::Write, 0, val); return; case MISCREG_ATS12NSOUR: if (!haveSecurity) panic("Security Extensions required for ATS12NSOUR"); - addressTranslation(TLB::S1S2NsTran, BaseMMU::Read, - TLB::UserMode, val); + addressTranslation(MMU::S1S2NsTran, BaseMMU::Read, + MMU::UserMode, val); return; case MISCREG_ATS12NSOUW: if (!haveSecurity) panic("Security Extensions required for ATS12NSOUW"); - addressTranslation(TLB::S1S2NsTran, BaseMMU::Write, - TLB::UserMode, val); + addressTranslation(MMU::S1S2NsTran, BaseMMU::Write, + MMU::UserMode, val); return; case MISCREG_ATS1HR: - addressTranslation(TLB::HypMode, BaseMMU::Read, 0, val); + addressTranslation(MMU::HypMode, BaseMMU::Read, 0, val); return; case MISCREG_ATS1HW: - addressTranslation(TLB::HypMode, BaseMMU::Write, 0, val); + addressTranslation(MMU::HypMode, BaseMMU::Write, 0, val); return; case MISCREG_TTBCR: { @@ -2252,44 +2252,44 @@ ISA::setMiscReg(int misc_reg, RegVal val) } break; case MISCREG_AT_S1E1R_Xt: - addressTranslation64(TLB::S1E1Tran, BaseMMU::Read, 0, val); + addressTranslation64(MMU::S1E1Tran, BaseMMU::Read, 0, val); return; case MISCREG_AT_S1E1W_Xt: - addressTranslation64(TLB::S1E1Tran, BaseMMU::Write, 0, val); + addressTranslation64(MMU::S1E1Tran, BaseMMU::Write, 0, val); return; case MISCREG_AT_S1E0R_Xt: - addressTranslation64(TLB::S1E0Tran, BaseMMU::Read, - TLB::UserMode, val); + addressTranslation64(MMU::S1E0Tran, BaseMMU::Read, + MMU::UserMode, val); return; case MISCREG_AT_S1E0W_Xt: - addressTranslation64(TLB::S1E0Tran, BaseMMU::Write, - TLB::UserMode, val); + addressTranslation64(MMU::S1E0Tran, BaseMMU::Write, + MMU::UserMode, val); return; case MISCREG_AT_S1E2R_Xt: - addressTranslation64(TLB::S1E2Tran, BaseMMU::Read, 0, val); + addressTranslation64(MMU::S1E2Tran, BaseMMU::Read, 0, val); return; case MISCREG_AT_S1E2W_Xt: - addressTranslation64(TLB::S1E2Tran, BaseMMU::Write, 0, val); + addressTranslation64(MMU::S1E2Tran, BaseMMU::Write, 0, val); return; case MISCREG_AT_S12E1R_Xt: - addressTranslation64(TLB::S12E1Tran, BaseMMU::Read, 0, val); + addressTranslation64(MMU::S12E1Tran, BaseMMU::Read, 0, val); return; case MISCREG_AT_S12E1W_Xt: - addressTranslation64(TLB::S12E1Tran, BaseMMU::Write, 0, val); + addressTranslation64(MMU::S12E1Tran, BaseMMU::Write, 0, val); return; case MISCREG_AT_S12E0R_Xt: - addressTranslation64(TLB::S12E0Tran, BaseMMU::Read, - TLB::UserMode, val); + addressTranslation64(MMU::S12E0Tran, BaseMMU::Read, + MMU::UserMode, val); return; case MISCREG_AT_S12E0W_Xt: - addressTranslation64(TLB::S12E0Tran, BaseMMU::Write, - TLB::UserMode, val); + addressTranslation64(MMU::S12E0Tran, BaseMMU::Write, + MMU::UserMode, val); return; case MISCREG_AT_S1E3R_Xt: - addressTranslation64(TLB::S1E3Tran, BaseMMU::Read, 0, val); + addressTranslation64(MMU::S1E3Tran, BaseMMU::Read, 0, val); return; case MISCREG_AT_S1E3W_Xt: - addressTranslation64(TLB::S1E3Tran, BaseMMU::Write, 0, val); + addressTranslation64(MMU::S1E3Tran, BaseMMU::Write, 0, val); return; case MISCREG_SPSR_EL3: case MISCREG_SPSR_EL2: @@ -2415,7 +2415,7 @@ ISA::unserialize(CheckpointIn &cp) } void -ISA::addressTranslation64(TLB::ArmTranslationType tran_type, +ISA::addressTranslation64(MMU::ArmTranslationType tran_type, BaseMMU::Mode mode, Request::Flags flags, RegVal val) { // If we're in timing mode then doing the translation in @@ -2466,7 +2466,7 @@ ISA::addressTranslation64(TLB::ArmTranslationType tran_type, } void -ISA::addressTranslation(TLB::ArmTranslationType tran_type, +ISA::addressTranslation(MMU::ArmTranslationType tran_type, BaseMMU::Mode mode, Request::Flags flags, RegVal val) { // If we're in timing mode then doing the translation in @@ -2491,8 +2491,8 @@ ISA::addressTranslation(TLB::ArmTranslationType tran_type, HCR hcr = readMiscRegNoEffect(MISCREG_HCR); uint8_t max_paddr_bit = 0; - if (haveLPAE && (ttbcr.eae || tran_type & TLB::HypMode || - ((tran_type & TLB::S1S2NsTran) && hcr.vm) )) { + if (haveLPAE && (ttbcr.eae || tran_type & MMU::HypMode || + ((tran_type & MMU::S1S2NsTran) && hcr.vm) )) { max_paddr_bit = 39; } else { diff --git a/src/arch/arm/isa.hh b/src/arch/arm/isa.hh index cc6e54d5ea..0e384839fb 100644 --- a/src/arch/arm/isa.hh +++ b/src/arch/arm/isa.hh @@ -42,11 +42,11 @@ #define __ARCH_ARM_ISA_HH__ #include "arch/arm/isa_device.hh" +#include "arch/arm/mmu.hh" #include "arch/arm/regs/int.hh" #include "arch/arm/regs/misc.hh" #include "arch/arm/self_debug.hh" #include "arch/arm/system.hh" -#include "arch/arm/tlb.hh" #include "arch/arm/types.hh" #include "arch/arm/utility.hh" #include "arch/generic/isa.hh" @@ -514,9 +514,9 @@ namespace ArmISA void initID32(const ArmISAParams &p); void initID64(const ArmISAParams &p); - void addressTranslation(TLB::ArmTranslationType tran_type, + void addressTranslation(MMU::ArmTranslationType tran_type, BaseMMU::Mode mode, Request::Flags flags, RegVal val); - void addressTranslation64(TLB::ArmTranslationType tran_type, + void addressTranslation64(MMU::ArmTranslationType tran_type, BaseMMU::Mode mode, Request::Flags flags, RegVal val); public: diff --git a/src/arch/arm/isa/formats/fp.isa b/src/arch/arm/isa/formats/fp.isa index b21ef53f1e..ce492c9d92 100644 --- a/src/arch/arm/isa/formats/fp.isa +++ b/src/arch/arm/isa/formats/fp.isa @@ -152,7 +152,7 @@ let {{ if (singleAll) { size = bits(machInst, 7, 6); bool t = bits(machInst, 5); - align = size | TLB::AllowUnaligned; + align = size | MMU::AllowUnaligned; if (width == 1) { regs = t ? 2 : 1; inc = 1; @@ -185,7 +185,7 @@ let {{ } } else { size = bits(machInst, 11, 10); - align = size | TLB::AllowUnaligned; + align = size | MMU::AllowUnaligned; regs = width; unsigned indexAlign = bits(machInst, 7, 4); // If width is 1, inc is always 1. That's overridden later. @@ -251,7 +251,7 @@ let {{ align = bits(machInst, 5, 4); if (align == 0) { // @align wasn't specified, so alignment can be turned off. - align = size | TLB::AllowUnaligned; + align = size | MMU::AllowUnaligned; } else { align = align + 2; } diff --git a/src/arch/arm/isa/insts/amo64.isa b/src/arch/arm/isa/insts/amo64.isa index 27ee9b595a..a6386af117 100644 --- a/src/arch/arm/isa/insts/amo64.isa +++ b/src/arch/arm/isa/insts/amo64.isa @@ -83,7 +83,7 @@ let {{ # Add memory request flags where necessary if self.user: - self.memFlags.append("ArmISA::TLB::UserMode") + self.memFlags.append("ArmISA::MMU::UserMode") sz = self.size*2 if paired else self.size self.memFlags.append("%d" % int(math.log(sz, 2))) diff --git a/src/arch/arm/isa/insts/branch.isa b/src/arch/arm/isa/insts/branch.isa index 33bb1ff5f2..5acca80680 100644 --- a/src/arch/arm/isa/insts/branch.isa +++ b/src/arch/arm/isa/insts/branch.isa @@ -186,16 +186,16 @@ let {{ for isTbh in (0, 1): if isTbh: eaCode = ''' - unsigned memAccessFlags = ArmISA::TLB::AllowUnaligned | - ArmISA::TLB::AlignHalfWord; + unsigned memAccessFlags = ArmISA::MMU::AllowUnaligned | + ArmISA::MMU::AlignHalfWord; EA = Op1 + Op2 * 2 ''' accCode = 'NPC = PC + 2 * (Mem_uh);\n' mnem = "tbh" else: eaCode = ''' - unsigned memAccessFlags = ArmISA::TLB::AllowUnaligned | - ArmISA::TLB::AlignByte; + unsigned memAccessFlags = ArmISA::MMU::AllowUnaligned | + ArmISA::MMU::AlignByte; EA = Op1 + Op2 ''' accCode = 'NPC = PC + 2 * (Mem_ub)' diff --git a/src/arch/arm/isa/insts/ldr.isa b/src/arch/arm/isa/insts/ldr.isa index 3be0e3e902..7a6b5cd41c 100644 --- a/src/arch/arm/isa/insts/ldr.isa +++ b/src/arch/arm/isa/insts/ldr.isa @@ -92,7 +92,7 @@ let {{ super(RfeInst, self).__init__(mnem, post, add, writeback) self.Name = "RFE_" + loadImmClassName(post, add, writeback, 8) - self.memFlags.append("ArmISA::TLB::AlignWord") + self.memFlags.append("ArmISA::MMU::AlignWord") def emit(self): offset = 0 @@ -163,7 +163,7 @@ let {{ # Add memory request flags where necessary self.memFlags.append("%d" % int(math.log(self.size, 2))) if self.user: - self.memFlags.append("ArmISA::TLB::UserMode") + self.memFlags.append("ArmISA::MMU::UserMode") self.instFlags = [] if self.flavor == "dprefetch": @@ -173,7 +173,7 @@ let {{ self.memFlags.append("Request::PREFETCH") self.instFlags = ['IsInstPrefetch'] elif self.flavor == "normal": - self.memFlags.append("ArmISA::TLB::AllowUnaligned") + self.memFlags.append("ArmISA::MMU::AllowUnaligned") if self.flavor in ("exclusive", "acex"): self.memFlags.append("Request::LLSC") @@ -249,9 +249,9 @@ let {{ # Add memory request flags where necessary if self.flavor in ("exclusive", "acex"): self.memFlags.append("Request::LLSC") - self.memFlags.append("ArmISA::TLB::AlignDoubleWord") + self.memFlags.append("ArmISA::MMU::AlignDoubleWord") else: - self.memFlags.append("ArmISA::TLB::AlignWord") + self.memFlags.append("ArmISA::MMU::AlignWord") # Disambiguate the class name for different flavors of loads if self.flavor != "normal": diff --git a/src/arch/arm/isa/insts/ldr64.isa b/src/arch/arm/isa/insts/ldr64.isa index 0a367953ba..848f4d6022 100644 --- a/src/arch/arm/isa/insts/ldr64.isa +++ b/src/arch/arm/isa/insts/ldr64.isa @@ -88,7 +88,7 @@ let {{ if self.flavor not in ("acquire", "acex", "exclusive", "acexp", "exp"): - self.memFlags.append("ArmISA::TLB::AllowUnaligned") + self.memFlags.append("ArmISA::MMU::AllowUnaligned") if self.flavor in ("acquire", "acex", "acexp"): self.instFlags.extend(["IsWriteBarrier", "IsReadBarrier"]) @@ -136,7 +136,7 @@ let {{ if self.user: eaCode += " uint8_t userFlag = 0;\n"\ " if(isUnpriviledgeAccess(xc->tcBase()))\n"\ - " userFlag = ArmISA::TLB::UserMode;" + " userFlag = ArmISA::MMU::UserMode;" self.codeBlobs["ea_code"] = eaCode diff --git a/src/arch/arm/isa/insts/str.isa b/src/arch/arm/isa/insts/str.isa index 48bf153dd7..49463b7b43 100644 --- a/src/arch/arm/isa/insts/str.isa +++ b/src/arch/arm/isa/insts/str.isa @@ -137,7 +137,7 @@ let {{ (newHeader, newDecoder, newExec) = self.fillTemplates(self.name, self.Name, codeBlobs, - ["ArmISA::TLB::AlignWord"], [], 'SrsOp', wbDecl) + ["ArmISA::MMU::AlignWord"], [], 'SrsOp', wbDecl) header_output += newHeader decoder_output += newDecoder @@ -178,13 +178,13 @@ let {{ # Add memory request flags where necessary self.memFlags.append("%d" % int(math.log(self.size, 2))) if self.user: - self.memFlags.append("ArmISA::TLB::UserMode") + self.memFlags.append("ArmISA::MMU::UserMode") if self.flavor in ("exclusive", "relex"): self.instFlags.append("IsStoreConditional") self.memFlags.append("Request::LLSC") elif self.flavor != "fp": - self.memFlags.append("ArmISA::TLB::AllowUnaligned") + self.memFlags.append("ArmISA::MMU::AllowUnaligned") if self.flavor in ("release", "relex"): self.instFlags.extend(["IsWriteBarrier", @@ -263,9 +263,9 @@ let {{ if self.flavor in ("exclusive", "relex"): self.instFlags.append("IsStoreConditional") self.memFlags.append("Request::LLSC") - self.memFlags.append("ArmISA::TLB::AlignDoubleWord") + self.memFlags.append("ArmISA::MMU::AlignDoubleWord") else: - self.memFlags.append("ArmISA::TLB::AlignWord") + self.memFlags.append("ArmISA::MMU::AlignWord") if self.flavor in ("release", "relex"): self.instFlags.extend(["IsWriteBarrier", diff --git a/src/arch/arm/isa/insts/str64.isa b/src/arch/arm/isa/insts/str64.isa index ed9906487c..93718591e3 100644 --- a/src/arch/arm/isa/insts/str64.isa +++ b/src/arch/arm/isa/insts/str64.isa @@ -73,7 +73,7 @@ let {{ if self.flavor not in ("release", "relex", "exclusive", "relexp", "exp"): - self.memFlags.append("ArmISA::TLB::AllowUnaligned") + self.memFlags.append("ArmISA::MMU::AllowUnaligned") if self.micro: self.instFlags.append("IsMicroop") @@ -137,7 +137,7 @@ let {{ if self.user: eaCode += " uint8_t userFlag = 0;\n"\ " if(isUnpriviledgeAccess(xc->tcBase()))\n"\ - " userFlag = ArmISA::TLB::UserMode;" + " userFlag = ArmISA::MMU::UserMode;" self.codeBlobs["ea_code"] = eaCode diff --git a/src/arch/arm/isa/templates/sve_mem.isa b/src/arch/arm/isa/templates/sve_mem.isa index c814157cfa..5b45e41c00 100644 --- a/src/arch/arm/isa/templates/sve_mem.isa +++ b/src/arch/arm/isa/templates/sve_mem.isa @@ -438,7 +438,7 @@ def template SveIndexedMemVIMicroopDeclare {{ dest(_dest), gp(_gp), base(_base), imm(_imm), elemIndex(_elemIndex), numElems(_numElems), firstFault(_firstFault), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { %(set_reg_idx_arr)s; %(constructor)s; @@ -526,7 +526,7 @@ def template SveIndexedMemSVMicroopDeclare {{ offsetIs32(_offsetIs32), offsetIsSigned(_offsetIsSigned), offsetIsScaled(_offsetIsScaled), elemIndex(_elemIndex), numElems(_numElems), firstFault(_firstFault), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { %(set_reg_idx_arr)s; %(constructor)s; @@ -909,7 +909,7 @@ def template SveStructMemSIMicroopDeclare {{ %(base_class)s(mnem, machInst, %(op_class)s), dest(_dest), gp(_gp), base(_base), imm(_imm), numRegs(_numRegs), regIndex(_regIndex), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { %(set_reg_idx_arr)s; %(constructor)s; @@ -1186,7 +1186,7 @@ def template SveStructMemSSMicroopDeclare {{ %(base_class)s(mnem, machInst, %(op_class)s), dest(_dest), gp(_gp), base(_base), offset(_offset), numRegs(_numRegs), regIndex(_regIndex), - memAccessFlags(ArmISA::TLB::AllowUnaligned) + memAccessFlags(ArmISA::MMU::AllowUnaligned) { %(set_reg_idx_arr)s; %(constructor)s; diff --git a/src/arch/arm/mmu.cc b/src/arch/arm/mmu.cc index 30164b6303..4b26a37019 100644 --- a/src/arch/arm/mmu.cc +++ b/src/arch/arm/mmu.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 ARM Limited + * Copyright (c) 2010-2013, 2016-2021 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -11,6 +11,9 @@ * unmodified and in its entirety in all distributions of the software, * modified or unmodified, in source code or in binary form. * + * Copyright (c) 2001-2005 The Regents of The University of Michigan + * All rights reserved. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: redistributions of source code must retain the above copyright @@ -36,8 +39,17 @@ */ #include "arch/arm/mmu.hh" + +#include "arch/arm/isa.hh" +#include "arch/arm/reg_abi.hh" +#include "arch/arm/stage2_lookup.hh" #include "arch/arm/table_walker.hh" #include "arch/arm/tlbi_op.hh" +#include "debug/TLB.hh" +#include "debug/TLBVerbose.hh" +#include "mem/packet_access.hh" +#include "sim/pseudo_inst.hh" +#include "sim/process.hh" namespace gem5 { @@ -47,16 +59,31 @@ using namespace ArmISA; MMU::MMU(const ArmMMUParams &p) : BaseMMU(p), itbStage2(p.stage2_itb), dtbStage2(p.stage2_dtb), - iport(p.itb_walker, p.sys->getRequestorId(p.itb_walker)), - dport(p.dtb_walker, p.sys->getRequestorId(p.dtb_walker)), itbWalker(p.itb_walker), dtbWalker(p.dtb_walker), itbStage2Walker(p.stage2_itb_walker), - dtbStage2Walker(p.stage2_dtb_walker) + dtbStage2Walker(p.stage2_dtb_walker), + test(nullptr), + miscRegContext(0), + s1State(this, false), s2State(this, true), + _attr(0), + stats(this) { - itbWalker->setPort(&iport); - dtbWalker->setPort(&dport); - itbStage2Walker->setPort(&iport); - dtbStage2Walker->setPort(&dport); + // Cache system-level properties + if (FullSystem) { + ArmSystem *arm_sys = dynamic_cast(p.sys); + assert(arm_sys); + haveLPAE = arm_sys->haveLPAE(); + haveVirtualization = arm_sys->haveVirtualization(); + haveLargeAsid64 = arm_sys->haveLargeAsid64(); + physAddrRange = arm_sys->physAddrRange(); + } else { + haveLPAE = false; + haveVirtualization = false; + haveLargeAsid64 = false; + physAddrRange = 48; + } + + m5opRange = p.sys->m5opRange(); } void @@ -70,12 +97,17 @@ MMU::init() itbStage2->setTableWalker(itbStage2Walker); dtbStage2->setTableWalker(dtbStage2Walker); - getITBPtr()->setStage2Tlb(itbStage2); getITBPtr()->setTableWalker(itbWalker); - getDTBPtr()->setStage2Tlb(dtbStage2); getDTBPtr()->setTableWalker(dtbWalker); } +void +MMU::drainResume() +{ + s1State.miscRegValid = false; + s2State.miscRegValid = false; +} + TLB * MMU::getTlb(BaseMMU::Mode mode, bool stage2) const { @@ -92,51 +124,1395 @@ MMU::getTlb(BaseMMU::Mode mode, bool stage2) const } } +TableWalker * +MMU::getTableWalker(BaseMMU::Mode mode, bool stage2) const +{ + if (mode == BaseMMU::Execute) { + if (stage2) + return itbStage2Walker; + else + return itbWalker; + } else { + if (stage2) + return dtbStage2Walker; + else + return dtbWalker; + } +} + bool -MMU::translateFunctional(ThreadContext *tc, Addr vaddr, Addr &paddr) +MMU::translateFunctional(ThreadContext *tc, Addr va, Addr &pa) { - return getDTBPtr()->translateFunctional(tc, vaddr, paddr); -} + CachedState& state = updateMiscReg(tc, NormalTran, false); -Fault -MMU::translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tran_type) -{ - return translateFunctional(req, tc, mode, tran_type, false); -} + auto tlb = getTlb(BaseMMU::Read, state.directToStage2); -Fault -MMU::translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tran_type, - bool stage2) -{ - return getTlb(mode, stage2)->translateFunctional( - req, tc, mode, tran_type); -} + TlbEntry *e = tlb->lookup( + va, state.asid, state.vmid, state.isHyp, state.isSecure, true, false, + state.aarch64 ? state.aarch64EL : EL1, false, BaseMMU::Read); -Fault -MMU::translateAtomic(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, bool stage2) -{ - return getTlb(mode, stage2)->translateAtomic(req, tc, mode); -} - -void -MMU::translateTiming(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, BaseMMU::Mode mode, bool stage2) -{ - return getTlb(mode, stage2)->translateTiming(req, tc, translation, mode); + if (!e) + return false; + pa = e->pAddr(va); + return true; } void MMU::invalidateMiscReg(TLBType type) { - if (type & TLBType::I_TLBS) { - getITBPtr()->invalidateMiscReg(); + s1State.miscRegValid = false; +} + +Fault +MMU::finalizePhysical(const RequestPtr &req, + ThreadContext *tc, Mode mode) const +{ + const Addr paddr = req->getPaddr(); + + if (m5opRange.contains(paddr)) { + uint8_t func; + pseudo_inst::decodeAddrOffset(paddr - m5opRange.start(), func); + req->setLocalAccessor( + [func, mode](ThreadContext *tc, PacketPtr pkt) -> Cycles + { + uint64_t ret; + if (inAArch64(tc)) + pseudo_inst::pseudoInst(tc, func, ret); + else + pseudo_inst::pseudoInst(tc, func, ret); + + if (mode == Read) + pkt->setLE(ret); + + return Cycles(1); + } + ); } - if (type & TLBType::D_TLBS) { - getDTBPtr()->invalidateMiscReg(); + + return NoFault; +} + + +Fault +MMU::translateSe(const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool &delay, bool timing, + CachedState &state) +{ + updateMiscReg(tc, NormalTran, state.isStage2); + Addr vaddr_tainted = req->getVaddr(); + Addr vaddr = 0; + if (state.aarch64) + vaddr = purifyTaggedAddr(vaddr_tainted, tc, state.aarch64EL, + (TCR)state.ttbcr, mode==Execute); + else + vaddr = vaddr_tainted; + Request::Flags flags = req->getFlags(); + + bool is_fetch = (mode == Execute); + bool is_write = (mode == Write); + + if (!is_fetch) { + if (state.sctlr.a || !(flags & AllowUnaligned)) { + if (vaddr & mask(flags & AlignmentMask)) { + // LPAE is always disabled in SE mode + return std::make_shared( + vaddr_tainted, + TlbEntry::DomainType::NoAccess, is_write, + ArmFault::AlignmentFault, state.isStage2, + ArmFault::VmsaTran); + } + } + } + + Addr paddr; + Process *p = tc->getProcessPtr(); + + if (!p->pTable->translate(vaddr, paddr)) + return std::make_shared(vaddr_tainted); + req->setPaddr(paddr); + + return finalizePhysical(req, tc, mode); +} + +Fault +MMU::checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode, + bool stage2) +{ + return checkPermissions(te, req, mode, stage2 ? s2State : s1State); +} + +Fault +MMU::checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode, + CachedState &state) +{ + // a data cache maintenance instruction that operates by MVA does + // not generate a Data Abort exeception due to a Permission fault + if (req->isCacheMaintenance()) { + return NoFault; + } + + Addr vaddr = req->getVaddr(); // 32-bit don't have to purify + Request::Flags flags = req->getFlags(); + bool is_fetch = (mode == Execute); + bool is_write = (mode == Write); + bool is_priv = state.isPriv && !(flags & UserMode); + + // Get the translation type from the actuall table entry + ArmFault::TranMethod tranMethod = te->longDescFormat ? ArmFault::LpaeTran + : ArmFault::VmsaTran; + + // If this is the second stage of translation and the request is for a + // stage 1 page table walk then we need to check the HCR.PTW bit. This + // allows us to generate a fault if the request targets an area marked + // as a device or strongly ordered. + if (state.isStage2 && req->isPTWalk() && state.hcr.ptw && + (te->mtype != TlbEntry::MemoryType::Normal)) { + return std::make_shared( + vaddr, te->domain, is_write, + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2, tranMethod); + } + + // Generate an alignment fault for unaligned data accesses to device or + // strongly ordered memory + if (!is_fetch) { + if (te->mtype != TlbEntry::MemoryType::Normal) { + if (vaddr & mask(flags & AlignmentMask)) { + stats.alignFaults++; + return std::make_shared( + vaddr, TlbEntry::DomainType::NoAccess, is_write, + ArmFault::AlignmentFault, state.isStage2, + tranMethod); + } + } + } + + if (te->nonCacheable) { + // Prevent prefetching from I/O devices. + if (req->isPrefetch()) { + // Here we can safely use the fault status for the short + // desc. format in all cases + return std::make_shared( + vaddr, ArmFault::PrefetchUncacheable, + state.isStage2, tranMethod); + } + } + + if (!te->longDescFormat) { + switch ((state.dacr >> (static_cast(te->domain) * 2)) & 0x3) { + case 0: + stats.domainFaults++; + DPRINTF(TLB, "TLB Fault: Data abort on domain. DACR: %#x" + " domain: %#x write:%d\n", state.dacr, + static_cast(te->domain), is_write); + if (is_fetch) { + // Use PC value instead of vaddr because vaddr might + // be aligned to cache line and should not be the + // address reported in FAR + return std::make_shared( + req->getPC(), + ArmFault::DomainLL + te->lookupLevel, + state.isStage2, tranMethod); + } else + return std::make_shared( + vaddr, te->domain, is_write, + ArmFault::DomainLL + te->lookupLevel, + state.isStage2, tranMethod); + case 1: + // Continue with permissions check + break; + case 2: + panic("UNPRED domain\n"); + case 3: + return NoFault; + } + } + + // The 'ap' variable is AP[2:0] or {AP[2,1],1b'0}, i.e. always three bits + uint8_t ap = te->longDescFormat ? te->ap << 1 : te->ap; + uint8_t hap = te->hap; + + if (state.sctlr.afe == 1 || te->longDescFormat) + ap |= 1; + + bool abt; + bool isWritable = true; + // If this is a stage 2 access (eg for reading stage 1 page table entries) + // then don't perform the AP permissions check, we stil do the HAP check + // below. + if (state.isStage2) { + abt = false; + } else { + switch (ap) { + case 0: + DPRINTF(TLB, "Access permissions 0, checking rs:%#x\n", + (int)state.sctlr.rs); + if (!state.sctlr.xp) { + switch ((int)state.sctlr.rs) { + case 2: + abt = is_write; + break; + case 1: + abt = is_write || !is_priv; + break; + case 0: + case 3: + default: + abt = true; + break; + } + } else { + abt = true; + } + break; + case 1: + abt = !is_priv; + break; + case 2: + abt = !is_priv && is_write; + isWritable = is_priv; + break; + case 3: + abt = false; + break; + case 4: + panic("UNPRED premissions\n"); + case 5: + abt = !is_priv || is_write; + isWritable = false; + break; + case 6: + case 7: + abt = is_write; + isWritable = false; + break; + default: + panic("Unknown permissions %#x\n", ap); + } + } + + bool hapAbt = is_write ? !(hap & 2) : !(hap & 1); + bool xn = te->xn || (isWritable && state.sctlr.wxn) || + (ap == 3 && state.sctlr.uwxn && is_priv); + if (is_fetch && (abt || xn || + (te->longDescFormat && te->pxn && is_priv) || + (state.isSecure && te->ns && state.scr.sif))) { + stats.permsFaults++; + DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. AP:%d " + "priv:%d write:%d ns:%d sif:%d sctlr.afe: %d \n", + ap, is_priv, is_write, te->ns, + state.scr.sif, state.sctlr.afe); + // Use PC value instead of vaddr because vaddr might be aligned to + // cache line and should not be the address reported in FAR + return std::make_shared( + req->getPC(), + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2, tranMethod); + } else if (abt | hapAbt) { + stats.permsFaults++; + DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d priv:%d" + " write:%d\n", ap, is_priv, is_write); + return std::make_shared( + vaddr, te->domain, is_write, + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2 | !abt, tranMethod); + } + return NoFault; +} + +Fault +MMU::checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode, + ThreadContext *tc, bool stage2) +{ + return checkPermissions64(te, req, mode, tc, stage2 ? s2State : s1State); +} + +Fault +MMU::checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode, + ThreadContext *tc, CachedState &state) +{ + assert(state.aarch64); + + // A data cache maintenance instruction that operates by VA does + // not generate a Permission fault unless: + // * It is a data cache invalidate (dc ivac) which requires write + // permissions to the VA, or + // * It is executed from EL0 + if (req->isCacheClean() && state.aarch64EL != EL0 && !state.isStage2) { + return NoFault; + } + + Addr vaddr_tainted = req->getVaddr(); + Addr vaddr = purifyTaggedAddr(vaddr_tainted, tc, state.aarch64EL, + (TCR)state.ttbcr, mode==Execute); + + Request::Flags flags = req->getFlags(); + bool is_fetch = (mode == Execute); + // Cache clean operations require read permissions to the specified VA + bool is_write = !req->isCacheClean() && mode == Write; + bool is_atomic = req->isAtomic(); + GEM5_VAR_USED bool is_priv = state.isPriv && !(flags & UserMode); + + updateMiscReg(tc, state.curTranType, state.isStage2); + + // If this is the second stage of translation and the request is for a + // stage 1 page table walk then we need to check the HCR.PTW bit. This + // allows us to generate a fault if the request targets an area marked + // as a device or strongly ordered. + if (state.isStage2 && req->isPTWalk() && state.hcr.ptw && + (te->mtype != TlbEntry::MemoryType::Normal)) { + return std::make_shared( + vaddr_tainted, te->domain, is_write, + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2, ArmFault::LpaeTran); + } + + // Generate an alignment fault for unaligned accesses to device or + // strongly ordered memory + if (!is_fetch) { + if (te->mtype != TlbEntry::MemoryType::Normal) { + if (vaddr & mask(flags & AlignmentMask)) { + stats.alignFaults++; + return std::make_shared( + vaddr_tainted, + TlbEntry::DomainType::NoAccess, + is_atomic ? false : is_write, + ArmFault::AlignmentFault, state.isStage2, + ArmFault::LpaeTran); + } + } + } + + if (te->nonCacheable) { + // Prevent prefetching from I/O devices. + if (req->isPrefetch()) { + // Here we can safely use the fault status for the short + // desc. format in all cases + return std::make_shared( + vaddr_tainted, + ArmFault::PrefetchUncacheable, + state.isStage2, ArmFault::LpaeTran); + } + } + + uint8_t ap = 0x3 & (te->ap); // 2-bit access protection field + bool grant = false; + + bool wxn = state.sctlr.wxn; + uint8_t xn = te->xn; + uint8_t pxn = te->pxn; + bool r = (!is_write && !is_fetch); + bool w = is_write; + bool x = is_fetch; + + if (ArmSystem::haveEL(tc, EL3) && state.isSecure && + te->ns && state.scr.sif) { + xn = true; + } + + // grant_read is used for faults from an atomic instruction that + // both reads and writes from a memory location. From a ISS point + // of view they count as read if a read to that address would have + // generated the fault; they count as writes otherwise + bool grant_read = true; + DPRINTF(TLBVerbose, "Checking permissions: ap:%d, xn:%d, pxn:%d, r:%d, " + "w:%d, x:%d, is_priv: %d, wxn: %d\n", ap, xn, + pxn, r, w, x, is_priv, wxn); + + if (state.isStage2) { + assert(ArmSystem::haveVirtualization(tc) && state.aarch64EL != EL2); + // In stage 2 we use the hypervisor access permission bits. + // The following permissions are described in ARM DDI 0487A.f + // D4-1802 + uint8_t hap = 0x3 & te->hap; + grant_read = hap & 0x1; + if (is_fetch) { + // sctlr.wxn overrides the xn bit + grant = !wxn && !xn; + } else if (is_atomic) { + grant = hap; + } else if (is_write) { + grant = hap & 0x2; + } else { // is_read + grant = grant_read; + } + } else { + switch (state.aarch64EL) { + case EL0: + { + grant_read = ap & 0x1; + uint8_t perm = (ap << 2) | (xn << 1) | pxn; + switch (perm) { + case 0: + case 1: + case 8: + case 9: + grant = x; + break; + case 4: + case 5: + grant = r || w || (x && !wxn); + break; + case 6: + case 7: + grant = r || w; + break; + case 12: + case 13: + grant = r || x; + break; + case 14: + case 15: + grant = r; + break; + default: + grant = false; + } + } + break; + case EL1: + { + if (checkPAN(tc, ap, req, mode, is_priv, state)) { + grant = false; + grant_read = false; + break; + } + + uint8_t perm = (ap << 2) | (xn << 1) | pxn; + switch (perm) { + case 0: + case 2: + grant = r || w || (x && !wxn); + break; + case 1: + case 3: + case 4: + case 5: + case 6: + case 7: + // regions that are writeable at EL0 should not be + // executable at EL1 + grant = r || w; + break; + case 8: + case 10: + case 12: + case 14: + grant = r || x; + break; + case 9: + case 11: + case 13: + case 15: + grant = r; + break; + default: + grant = false; + } + } + break; + case EL2: + if (state.hcr.e2h && checkPAN(tc, ap, req, mode, is_priv, state)) { + grant = false; + grant_read = false; + break; + } + [[fallthrough]]; + case EL3: + { + uint8_t perm = (ap & 0x2) | xn; + switch (perm) { + case 0: + grant = r || w || (x && !wxn); + break; + case 1: + grant = r || w; + break; + case 2: + grant = r || x; + break; + case 3: + grant = r; + break; + default: + grant = false; + } + } + break; + } + } + + if (!grant) { + if (is_fetch) { + stats.permsFaults++; + DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. " + "AP:%d priv:%d write:%d ns:%d sif:%d " + "sctlr.afe: %d\n", + ap, is_priv, is_write, te->ns, + state.scr.sif, state.sctlr.afe); + // Use PC value instead of vaddr because vaddr might be aligned to + // cache line and should not be the address reported in FAR + return std::make_shared( + req->getPC(), + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2, ArmFault::LpaeTran); + } else { + stats.permsFaults++; + DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d " + "priv:%d write:%d\n", ap, is_priv, is_write); + return std::make_shared( + vaddr_tainted, te->domain, + (is_atomic && !grant_read) ? false : is_write, + ArmFault::PermissionLL + te->lookupLevel, + state.isStage2, ArmFault::LpaeTran); + } + } + + return NoFault; +} + +bool +MMU::checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, Mode mode, + const bool is_priv, CachedState &state) +{ + // The PAN bit has no effect on: + // 1) Instruction accesses. + // 2) Data Cache instructions other than DC ZVA + // 3) Address translation instructions, other than ATS1E1RP and + // ATS1E1WP when ARMv8.2-ATS1E1 is implemented. (Unimplemented in + // gem5) + // 4) Instructions to be treated as unprivileged, unless + // HCR_EL2.{E2H, TGE} == {1, 0} + const AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1); + if (mmfr1.pan && state.cpsr.pan && (ap & 0x1) && + mode != BaseMMU::Execute) { + + if (req->isCacheMaintenance() && + !(req->getFlags() & Request::CACHE_BLOCK_ZERO)) { + // Cache maintenance other than DC ZVA + return false; + } else if (!is_priv && !(state.hcr.e2h && !state.hcr.tge)) { + // Treated as unprivileged unless HCR_EL2.{E2H, TGE} == {1, 0} + return false; + } + return true; + } + + return false; +} + +Fault +MMU::translateMmuOff(ThreadContext *tc, const RequestPtr &req, Mode mode, + ArmTranslationType tran_type, Addr vaddr, bool long_desc_format, + CachedState &state) +{ + bool is_fetch = (mode == Execute); + bool is_atomic = req->isAtomic(); + req->setPaddr(vaddr); + // When the MMU is off the security attribute corresponds to the + // security state of the processor + if (state.isSecure) + req->setFlags(Request::SECURE); + + if (state.aarch64) { + bool selbit = bits(vaddr, 55); + TCR tcr1 = tc->readMiscReg(MISCREG_TCR_EL1); + int topbit = computeAddrTop(tc, selbit, is_fetch, tcr1, currEL(tc)); + int addr_sz = bits(vaddr, topbit, physAddrRange); + if (addr_sz != 0){ + Fault f; + if (is_fetch) + f = std::make_shared(vaddr, + ArmFault::AddressSizeLL, state.isStage2, + ArmFault::LpaeTran); + else + f = std::make_shared( vaddr, + TlbEntry::DomainType::NoAccess, + is_atomic ? false : mode==Write, + ArmFault::AddressSizeLL, state.isStage2, + ArmFault::LpaeTran); + return f; + } + } + + // @todo: double check this (ARM ARM issue C B3.2.1) + if (long_desc_format || state.sctlr.tre == 0 || state.nmrr.ir0 == 0 || + state.nmrr.or0 == 0 || state.prrr.tr0 != 0x2) { + if (!req->isCacheMaintenance()) { + req->setFlags(Request::UNCACHEABLE); + } + req->setFlags(Request::STRICT_ORDER); + } + + // Set memory attributes + TlbEntry temp_te; + temp_te.ns = !state.isSecure; + bool dc = (HaveVirtHostExt(tc) + && state.hcr.e2h == 1 && state.hcr.tge == 1) ? 0: state.hcr.dc; + bool i_cacheability = state.sctlr.i && !state.sctlr.m; + if (state.isStage2 || !dc || state.isSecure || + (state.isHyp && !(tran_type & S1CTran))) { + + temp_te.mtype = is_fetch ? TlbEntry::MemoryType::Normal + : TlbEntry::MemoryType::StronglyOrdered; + temp_te.innerAttrs = i_cacheability? 0x2: 0x0; + temp_te.outerAttrs = i_cacheability? 0x2: 0x0; + temp_te.shareable = true; + temp_te.outerShareable = true; + } else { + temp_te.mtype = TlbEntry::MemoryType::Normal; + temp_te.innerAttrs = 0x3; + temp_te.outerAttrs = 0x3; + temp_te.shareable = false; + temp_te.outerShareable = false; + } + temp_te.setAttributes(long_desc_format); + DPRINTF(TLBVerbose, "(No MMU) setting memory attributes: shareable: " + "%d, innerAttrs: %d, outerAttrs: %d, stage2: %d\n", + temp_te.shareable, temp_te.innerAttrs, temp_te.outerAttrs, + state.isStage2); + setAttr(temp_te.attributes); + + return testTranslation(req, mode, TlbEntry::DomainType::NoAccess, state); +} + +Fault +MMU::translateMmuOn(ThreadContext* tc, const RequestPtr &req, Mode mode, + Translation *translation, bool &delay, bool timing, + bool functional, Addr vaddr, + ArmFault::TranMethod tranMethod, CachedState &state) +{ + TlbEntry *te = NULL; + bool is_fetch = (mode == Execute); + TlbEntry mergeTe; + + Request::Flags flags = req->getFlags(); + Addr vaddr_tainted = req->getVaddr(); + + Fault fault = getResultTe(&te, req, tc, mode, translation, timing, + functional, &mergeTe, state); + // only proceed if we have a valid table entry + if ((te == NULL) && (fault == NoFault)) delay = true; + + // If we have the table entry transfer some of the attributes to the + // request that triggered the translation + if (te != NULL) { + // Set memory attributes + DPRINTF(TLBVerbose, + "Setting memory attributes: shareable: %d, innerAttrs: %d, " + "outerAttrs: %d, mtype: %d, stage2: %d\n", + te->shareable, te->innerAttrs, te->outerAttrs, + static_cast(te->mtype), state.isStage2); + setAttr(te->attributes); + + if (te->nonCacheable && !req->isCacheMaintenance()) + req->setFlags(Request::UNCACHEABLE); + + // Require requests to be ordered if the request goes to + // strongly ordered or device memory (i.e., anything other + // than normal memory requires strict order). + if (te->mtype != TlbEntry::MemoryType::Normal) + req->setFlags(Request::STRICT_ORDER); + + Addr pa = te->pAddr(vaddr); + req->setPaddr(pa); + + if (state.isSecure && !te->ns) { + req->setFlags(Request::SECURE); + } + if (!is_fetch && fault == NoFault && + (vaddr & mask(flags & AlignmentMask)) && + (te->mtype != TlbEntry::MemoryType::Normal)) { + // Unaligned accesses to Device memory should always cause an + // abort regardless of sctlr.a + stats.alignFaults++; + bool is_write = (mode == Write); + return std::make_shared( + vaddr_tainted, + TlbEntry::DomainType::NoAccess, is_write, + ArmFault::AlignmentFault, state.isStage2, + tranMethod); + } + + // Check for a trickbox generated address fault + if (fault == NoFault) + fault = testTranslation(req, mode, te->domain, state); + } + + if (fault == NoFault) { + // Don't try to finalize a physical address unless the + // translation has completed (i.e., there is a table entry). + return te ? finalizePhysical(req, tc, mode) : NoFault; + } else { + return fault; } } +Fault +MMU::translateFs(const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool &delay, bool timing, + ArmTranslationType tran_type, bool functional, + CachedState &state) +{ + // No such thing as a functional timing access + assert(!(timing && functional)); + + Addr vaddr_tainted = req->getVaddr(); + Addr vaddr = 0; + if (state.aarch64) + vaddr = purifyTaggedAddr(vaddr_tainted, tc, state.aarch64EL, + (TCR)state.ttbcr, mode==Execute); + else + vaddr = vaddr_tainted; + Request::Flags flags = req->getFlags(); + + bool is_fetch = (mode == Execute); + bool is_write = (mode == Write); + bool long_desc_format = state.aarch64 || longDescFormatInUse(tc); + ArmFault::TranMethod tranMethod = long_desc_format ? ArmFault::LpaeTran + : ArmFault::VmsaTran; + + DPRINTF(TLBVerbose, + "CPSR is priv:%d UserMode:%d secure:%d S1S2NsTran:%d\n", + state.isPriv, flags & UserMode, state.isSecure, + tran_type & S1S2NsTran); + + DPRINTF(TLB, "translateFs addr %#x, mode %d, st2 %d, scr %#x sctlr %#x " + "flags %#lx tranType 0x%x\n", vaddr_tainted, mode, + state.isStage2, state.scr, state.sctlr, flags, tran_type); + + if (!state.isStage2) { + if ((req->isInstFetch() && (!state.sctlr.i)) || + ((!req->isInstFetch()) && (!state.sctlr.c))){ + if (!req->isCacheMaintenance()) { + req->setFlags(Request::UNCACHEABLE); + } + req->setFlags(Request::STRICT_ORDER); + } + } + if (!is_fetch) { + if (state.sctlr.a || !(flags & AllowUnaligned)) { + if (vaddr & mask(flags & AlignmentMask)) { + stats.alignFaults++; + return std::make_shared( + vaddr_tainted, + TlbEntry::DomainType::NoAccess, is_write, + ArmFault::AlignmentFault, state.isStage2, + tranMethod); + } + } + } + + bool vm = state.hcr.vm; + if (HaveVirtHostExt(tc) && state.hcr.e2h == 1 && state.hcr.tge ==1) + vm = 0; + else if (state.hcr.dc == 1) + vm = 1; + + Fault fault = NoFault; + // If guest MMU is off or hcr.vm=0 go straight to stage2 + if ((state.isStage2 && !vm) || (!state.isStage2 && !state.sctlr.m)) { + fault = translateMmuOff(tc, req, mode, tran_type, vaddr, + long_desc_format, state); + } else { + DPRINTF(TLBVerbose, "Translating %s=%#x context=%d\n", + state.isStage2 ? "IPA" : "VA", vaddr_tainted, state.asid); + // Translation enabled + fault = translateMmuOn(tc, req, mode, translation, delay, timing, + functional, vaddr, tranMethod, state); + } + + // Check for Debug Exceptions + SelfDebug *sd = ArmISA::ISA::getSelfDebug(tc); + + if (sd->enabled() && fault == NoFault) { + fault = sd->testDebug(tc, req, mode); + } + + return fault; +} + +Fault +MMU::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode, + ArmTranslationType tran_type) +{ + return translateAtomic(req, tc, mode, tran_type, false); +} + +Fault +MMU::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode, + ArmTranslationType tran_type, bool stage2) +{ + auto& state = updateMiscReg(tc, tran_type, stage2); + + bool delay = false; + Fault fault; + if (FullSystem) + fault = translateFs(req, tc, mode, NULL, delay, false, + tran_type, false, state); + else + fault = translateSe(req, tc, mode, NULL, delay, false, state); + assert(!delay); + return fault; +} + +Fault +MMU::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode) +{ + return translateFunctional(req, tc, mode, NormalTran, false); +} + +Fault +MMU::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode, + ArmTranslationType tran_type) +{ + return translateFunctional(req, tc, mode, tran_type, false); +} + +Fault +MMU::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode, + ArmTranslationType tran_type, bool stage2) +{ + auto& state = updateMiscReg(tc, tran_type, stage2); + + bool delay = false; + Fault fault; + if (FullSystem) + fault = translateFs(req, tc, mode, NULL, delay, false, + tran_type, true, state); + else + fault = translateSe(req, tc, mode, NULL, delay, false, state); + assert(!delay); + return fault; +} + +void +MMU::translateTiming(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, ArmTranslationType tran_type, + bool stage2) +{ + auto& state = updateMiscReg(tc, tran_type, stage2); + + assert(translation); + + translateComplete(req, tc, translation, mode, tran_type, + stage2, state); +} + +Fault +MMU::translateComplete(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, ArmTranslationType tran_type, + bool call_from_s2) +{ + return translateComplete(req, tc, translation, mode, tran_type, + call_from_s2, s1State); +} + +Fault +MMU::translateComplete(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, ArmTranslationType tran_type, + bool call_from_s2, CachedState &state) +{ + bool delay = false; + Fault fault; + if (FullSystem) + fault = translateFs(req, tc, mode, translation, delay, true, tran_type, + false, state); + else + fault = translateSe(req, tc, mode, translation, delay, true, state); + + DPRINTF(TLBVerbose, "Translation returning delay=%d fault=%d\n", delay, + fault != NoFault); + // If we have a translation, and we're not in the middle of doing a stage + // 2 translation tell the translation that we've either finished or its + // going to take a while. By not doing this when we're in the middle of a + // stage 2 translation we prevent marking the translation as delayed twice, + // one when the translation starts and again when the stage 1 translation + // completes. + + if (translation && (call_from_s2 || !state.stage2Req || req->hasPaddr() || + fault != NoFault)) { + if (!delay) + translation->finish(fault, req, tc, mode); + else + translation->markDelayed(); + } + return fault; +} + +vmid_t +MMU::CachedState::getVMID(ThreadContext *tc) const +{ + AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1); + VTCR_t vtcr = tc->readMiscReg(MISCREG_VTCR_EL2); + vmid_t vmid = 0; + + switch (mmfr1.vmidbits) { + case 0b0000: + // 8 bits + vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48); + break; + case 0b0010: + if (vtcr.vs && ELIs64(tc, EL2)) { + // 16 bits + vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 63, 48); + } else { + // 8 bits + vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48); + } + break; + default: + panic("Reserved ID_AA64MMFR1_EL1.VMIDBits value: %#x", + mmfr1.vmidbits); + } + + return vmid; +} + +MMU::CachedState& +MMU::updateMiscReg(ThreadContext *tc, + ArmTranslationType tran_type, bool stage2) +{ + // check if the regs have changed, or the translation mode is different. + // NOTE: the tran type doesn't affect stage 2 TLB's as they only handle + // one type of translation anyway + + auto& state = stage2 ? s2State : s1State; + if (state.miscRegValid && miscRegContext == tc->contextId() && + ((tran_type == state.curTranType) || stage2)) { + + } else { + DPRINTF(TLBVerbose, "TLB variables changed!\n"); + state.updateMiscReg(tc, tran_type); + + itbStage2->setVMID(state.vmid); + dtbStage2->setVMID(state.vmid); + getITBPtr()->setVMID(state.vmid); + getDTBPtr()->setVMID(state.vmid); + + miscRegContext = tc->contextId(); + } + + if (state.directToStage2) { + s2State.updateMiscReg(tc, tran_type); + return s2State; + } else { + return state; + } +} + +void +MMU::CachedState::updateMiscReg(ThreadContext *tc, + ArmTranslationType tran_type) +{ + cpsr = tc->readMiscReg(MISCREG_CPSR); + + // Dependencies: SCR/SCR_EL3, CPSR + isSecure = ArmISA::isSecure(tc) && + !(tran_type & HypMode) && !(tran_type & S1S2NsTran); + + aarch64EL = tranTypeEL(cpsr, tran_type); + aarch64 = isStage2 ? + ELIs64(tc, EL2) : + ELIs64(tc, aarch64EL == EL0 ? EL1 : aarch64EL); + + hcr = tc->readMiscReg(MISCREG_HCR_EL2); + if (aarch64) { // AArch64 + // determine EL we need to translate in + switch (aarch64EL) { + case EL0: + if (HaveVirtHostExt(tc) && hcr.tge == 1 && hcr.e2h == 1) { + // VHE code for EL2&0 regime + sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2); + ttbcr = tc->readMiscReg(MISCREG_TCR_EL2); + uint64_t ttbr_asid = ttbcr.a1 ? + tc->readMiscReg(MISCREG_TTBR1_EL2) : + tc->readMiscReg(MISCREG_TTBR0_EL2); + asid = bits(ttbr_asid, + (mmu->haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); + + } else { + sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1); + ttbcr = tc->readMiscReg(MISCREG_TCR_EL1); + uint64_t ttbr_asid = ttbcr.a1 ? + tc->readMiscReg(MISCREG_TTBR1_EL1) : + tc->readMiscReg(MISCREG_TTBR0_EL1); + asid = bits(ttbr_asid, + (mmu->haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); + + } + break; + case EL1: + { + sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1); + ttbcr = tc->readMiscReg(MISCREG_TCR_EL1); + uint64_t ttbr_asid = ttbcr.a1 ? + tc->readMiscReg(MISCREG_TTBR1_EL1) : + tc->readMiscReg(MISCREG_TTBR0_EL1); + asid = bits(ttbr_asid, + (mmu->haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); + } + break; + case EL2: + sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2); + ttbcr = tc->readMiscReg(MISCREG_TCR_EL2); + if (hcr.e2h == 1) { + // VHE code for EL2&0 regime + uint64_t ttbr_asid = ttbcr.a1 ? + tc->readMiscReg(MISCREG_TTBR1_EL2) : + tc->readMiscReg(MISCREG_TTBR0_EL2); + asid = bits(ttbr_asid, + (mmu->haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); + } else { + asid = -1; + } + break; + case EL3: + sctlr = tc->readMiscReg(MISCREG_SCTLR_EL3); + ttbcr = tc->readMiscReg(MISCREG_TCR_EL3); + asid = -1; + break; + } + + scr = tc->readMiscReg(MISCREG_SCR_EL3); + isPriv = aarch64EL != EL0; + if (mmu->haveVirtualization) { + vmid = getVMID(tc); + isHyp = aarch64EL == EL2; + isHyp |= tran_type & HypMode; + isHyp &= (tran_type & S1S2NsTran) == 0; + isHyp &= (tran_type & S1CTran) == 0; + bool vm = hcr.vm; + if (HaveVirtHostExt(tc) && hcr.e2h == 1 && hcr.tge ==1) { + vm = 0; + } + + if (hcr.e2h == 1 && (aarch64EL == EL2 + || (hcr.tge ==1 && aarch64EL == EL0))) { + isHyp = true; + directToStage2 = false; + stage2Req = false; + stage2DescReq = false; + } else { + // Work out if we should skip the first stage of translation and go + // directly to stage 2. This value is cached so we don't have to + // compute it for every translation. + bool sec = !isSecure || (isSecure && IsSecureEL2Enabled(tc)); + stage2Req = isStage2 || + (vm && !isHyp && sec && + !(tran_type & S1CTran) && (aarch64EL < EL2) && + !(tran_type & S1E1Tran)); // <--- FIX THIS HACK + stage2DescReq = isStage2 || (vm && !isHyp && sec && + (aarch64EL < EL2)); + directToStage2 = !isStage2 && stage2Req && !sctlr.m; + } + } else { + vmid = 0; + isHyp = false; + directToStage2 = false; + stage2Req = false; + stage2DescReq = false; + } + } else { // AArch32 + sctlr = tc->readMiscReg(snsBankedIndex(MISCREG_SCTLR, tc, + !isSecure)); + ttbcr = tc->readMiscReg(snsBankedIndex(MISCREG_TTBCR, tc, + !isSecure)); + scr = tc->readMiscReg(MISCREG_SCR); + isPriv = cpsr.mode != MODE_USER; + if (longDescFormatInUse(tc)) { + uint64_t ttbr_asid = tc->readMiscReg( + snsBankedIndex(ttbcr.a1 ? MISCREG_TTBR1 : + MISCREG_TTBR0, + tc, !isSecure)); + asid = bits(ttbr_asid, 55, 48); + } else { // Short-descriptor translation table format in use + CONTEXTIDR context_id = tc->readMiscReg(snsBankedIndex( + MISCREG_CONTEXTIDR, tc,!isSecure)); + asid = context_id.asid; + } + prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, tc, + !isSecure)); + nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, tc, + !isSecure)); + dacr = tc->readMiscReg(snsBankedIndex(MISCREG_DACR, tc, + !isSecure)); + hcr = tc->readMiscReg(MISCREG_HCR); + + if (mmu->haveVirtualization) { + vmid = bits(tc->readMiscReg(MISCREG_VTTBR), 55, 48); + isHyp = cpsr.mode == MODE_HYP; + isHyp |= tran_type & HypMode; + isHyp &= (tran_type & S1S2NsTran) == 0; + isHyp &= (tran_type & S1CTran) == 0; + if (isHyp) { + sctlr = tc->readMiscReg(MISCREG_HSCTLR); + } + // Work out if we should skip the first stage of translation and go + // directly to stage 2. This value is cached so we don't have to + // compute it for every translation. + bool sec = !isSecure || (isSecure && IsSecureEL2Enabled(tc)); + stage2Req = hcr.vm && !isStage2 && !isHyp && sec && + !(tran_type & S1CTran); + stage2DescReq = hcr.vm && !isStage2 && !isHyp && sec; + directToStage2 = stage2Req && !sctlr.m; + } else { + vmid = 0; + stage2Req = false; + isHyp = false; + directToStage2 = false; + stage2DescReq = false; + } + } + miscRegValid = true; + curTranType = tran_type; +} + +ExceptionLevel +MMU::tranTypeEL(CPSR cpsr, ArmTranslationType type) +{ + switch (type) { + case S1E0Tran: + case S12E0Tran: + return EL0; + + case S1E1Tran: + case S12E1Tran: + return EL1; + + case S1E2Tran: + return EL2; + + case S1E3Tran: + return EL3; + + case NormalTran: + case S1CTran: + case S1S2NsTran: + case HypMode: + return currEL(cpsr); + + default: + panic("Unknown translation mode!\n"); + } +} + +Fault +MMU::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool timing, bool functional, + bool is_secure, ArmTranslationType tran_type, + bool stage2) +{ + return getTE(te, req, tc, mode, translation, timing, functional, + is_secure, tran_type, stage2 ? s2State : s1State); +} + +Fault +MMU::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool timing, bool functional, + bool is_secure, ArmTranslationType tran_type, + CachedState& state) +{ + // In a 2-stage system, the IPA->PA translation can be started via this + // call so make sure the miscRegs are correct. + if (state.isStage2) { + updateMiscReg(tc, tran_type, true); + } + + Addr vaddr_tainted = req->getVaddr(); + Addr vaddr = 0; + ExceptionLevel target_el = state.aarch64 ? state.aarch64EL : EL1; + if (state.aarch64) { + vaddr = purifyTaggedAddr(vaddr_tainted, tc, target_el, + (TCR)state.ttbcr, mode==Execute); + } else { + vaddr = vaddr_tainted; + } + + auto tlb = getTlb(mode, state.isStage2); + *te = tlb->lookup(vaddr, state.asid, state.vmid, state.isHyp, is_secure, + false, false, target_el, false, mode); + if (*te == NULL) { + if (req->isPrefetch()) { + // if the request is a prefetch don't attempt to fill the TLB or go + // any further with the memory access (here we can safely use the + // fault status for the short desc. format in all cases) + stats.prefetchFaults++; + return std::make_shared( + vaddr_tainted, ArmFault::PrefetchTLBMiss, state.isStage2); + } + + // start translation table walk, pass variables rather than + // re-retreaving in table walker for speed + DPRINTF(TLB, + "TLB Miss: Starting hardware table walker for %#x(%d:%d)\n", + vaddr_tainted, state.asid, state.vmid); + + Fault fault; + fault = getTableWalker(mode, state.isStage2)->walk( + req, tc, state.asid, state.vmid, state.isHyp, mode, + translation, timing, functional, is_secure, + tran_type, state.stage2DescReq); + + // for timing mode, return and wait for table walk, + if (timing || fault != NoFault) { + return fault; + } + + *te = tlb->lookup(vaddr, state.asid, state.vmid, state.isHyp, is_secure, + true, false, target_el, false, mode); + if (!*te) + tlb->printTlb(); + assert(*te); + } + return NoFault; +} + +Fault +MMU::getResultTe(TlbEntry **te, const RequestPtr &req, + ThreadContext *tc, Mode mode, + Translation *translation, bool timing, bool functional, + TlbEntry *mergeTe, CachedState &state) +{ + Fault fault; + + if (state.isStage2) { + // We are already in the stage 2 TLB. Grab the table entry for stage + // 2 only. We are here because stage 1 translation is disabled. + TlbEntry *s2_te = NULL; + // Get the stage 2 table entry + fault = getTE(&s2_te, req, tc, mode, translation, timing, functional, + state.isSecure, state.curTranType, state); + // Check permissions of stage 2 + if ((s2_te != NULL) && (fault == NoFault)) { + if (state.aarch64) + fault = checkPermissions64(s2_te, req, mode, tc, state); + else + fault = checkPermissions(s2_te, req, mode, state); + } + *te = s2_te; + return fault; + } + + TlbEntry *s1Te = NULL; + + Addr vaddr_tainted = req->getVaddr(); + + // Get the stage 1 table entry + fault = getTE(&s1Te, req, tc, mode, translation, timing, functional, + state.isSecure, state.curTranType, state); + // only proceed if we have a valid table entry + if ((s1Te != NULL) && (fault == NoFault)) { + // Check stage 1 permissions before checking stage 2 + if (state.aarch64) + fault = checkPermissions64(s1Te, req, mode, tc, state); + else + fault = checkPermissions(s1Te, req, mode, state); + if (state.stage2Req & (fault == NoFault)) { + Stage2LookUp *s2_lookup = new Stage2LookUp(this, *s1Te, + req, translation, mode, timing, functional, state.isSecure, + state.curTranType); + fault = s2_lookup->getTe(tc, mergeTe); + if (s2_lookup->isComplete()) { + *te = mergeTe; + // We've finished with the lookup so delete it + delete s2_lookup; + } else { + // The lookup hasn't completed, so we can't delete it now. We + // get round this by asking the object to self delete when the + // translation is complete. + s2_lookup->setSelfDelete(); + } + } else { + // This case deals with an S1 hit (or bypass), followed by + // an S2 hit-but-perms issue + if (state.isStage2) { + DPRINTF(TLBVerbose, "s2TLB: reqVa %#x, reqPa %#x, fault %p\n", + vaddr_tainted, req->hasPaddr() ? req->getPaddr() : ~0, + fault); + if (fault != NoFault) { + auto arm_fault = reinterpret_cast(fault.get()); + arm_fault->annotate(ArmFault::S1PTW, false); + arm_fault->annotate(ArmFault::OVA, vaddr_tainted); + } + } + *te = s1Te; + } + } + return fault; +} + +void +MMU::takeOverFrom(BaseMMU *old_mmu) +{ + BaseMMU::takeOverFrom(old_mmu); + + auto *ommu = dynamic_cast(old_mmu); + assert(ommu); + + _attr = ommu->_attr; + + s1State = ommu->s1State; + s2State = ommu->s2State; +} + +void +MMU::setTestInterface(SimObject *_ti) +{ + if (!_ti) { + test = nullptr; + } else { + TlbTestInterface *ti(dynamic_cast(_ti)); + fatal_if(!ti, "%s is not a valid ARM TLB tester\n", _ti->name()); + test = ti; + } +} + +Fault +MMU::testTranslation(const RequestPtr &req, Mode mode, + TlbEntry::DomainType domain, CachedState &state) +{ + if (!test || !req->hasSize() || req->getSize() == 0 || + req->isCacheMaintenance()) { + return NoFault; + } else { + return test->translationCheck(req, state.isPriv, mode, domain); + } +} + +Fault +MMU::testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode, + TlbEntry::DomainType domain, LookupLevel lookup_level, + bool stage2) +{ + return testWalk(pa, size, va, is_secure, mode, domain, lookup_level, + stage2 ? s2State : s1State); +} + +Fault +MMU::testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode, + TlbEntry::DomainType domain, LookupLevel lookup_level, + CachedState &state) +{ + if (!test) { + return NoFault; + } else { + return test->walkCheck(pa, size, va, is_secure, state.isPriv, mode, + domain, lookup_level); + } +} + +MMU::Stats::Stats(statistics::Group *parent) + : statistics::Group(parent), + ADD_STAT(alignFaults, statistics::units::Count::get(), + "Number of MMU faults due to alignment restrictions"), + ADD_STAT(prefetchFaults, statistics::units::Count::get(), + "Number of MMU faults due to prefetch"), + ADD_STAT(domainFaults, statistics::units::Count::get(), + "Number of MMU faults due to domain restrictions"), + ADD_STAT(permsFaults, statistics::units::Count::get(), + "Number of MMU faults due to permissions restrictions") +{ +} + } // namespace gem5 diff --git a/src/arch/arm/mmu.hh b/src/arch/arm/mmu.hh index 14816d75b7..fddaa1f69d 100644 --- a/src/arch/arm/mmu.hh +++ b/src/arch/arm/mmu.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 ARM Limited + * Copyright (c) 2010-2013, 2016, 2019-2021 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -11,6 +11,9 @@ * unmodified and in its entirety in all distributions of the software, * modified or unmodified, in source code or in binary form. * + * Copyright (c) 2001-2005 The Regents of The University of Michigan + * All rights reserved. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: redistributions of source code must retain the above copyright @@ -38,7 +41,6 @@ #ifndef __ARCH_ARM_MMU_HH__ #define __ARCH_ARM_MMU_HH__ -#include "arch/arm/table_walker.hh" #include "arch/arm/tlb.hh" #include "arch/generic/mmu.hh" @@ -49,6 +51,8 @@ namespace gem5 namespace ArmISA { +class TableWalker; + class MMU : public BaseMMU { protected: @@ -65,20 +69,54 @@ class MMU : public BaseMMU } TLB * getTlb(BaseMMU::Mode mode, bool stage2) const; + TableWalker * getTableWalker(BaseMMU::Mode mode, bool stage2) const; protected: TLB *itbStage2; TLB *dtbStage2; - TableWalker::Port iport; - TableWalker::Port dport; - TableWalker *itbWalker; TableWalker *dtbWalker; TableWalker *itbStage2Walker; TableWalker *dtbStage2Walker; public: + enum ArmFlags + { + AlignmentMask = 0x7, + + AlignByte = 0x0, + AlignHalfWord = 0x1, + AlignWord = 0x2, + AlignDoubleWord = 0x3, + AlignQuadWord = 0x4, + AlignOctWord = 0x5, + + AllowUnaligned = 0x8, + // Priv code operating as if it wasn't + UserMode = 0x10 + }; + + enum ArmTranslationType + { + NormalTran = 0, + S1CTran = 0x1, + HypMode = 0x2, + // Secure code operating as if it wasn't (required by some Address + // Translate operations) + S1S2NsTran = 0x4, + // Address translation instructions (eg AT S1E0R_Xt) need to be handled + // in special ways during translation because they could need to act + // like a different EL than the current EL. The following flags are + // for these instructions + S1E0Tran = 0x8, + S1E1Tran = 0x10, + S1E2Tran = 0x20, + S1E3Tran = 0x40, + S12E0Tran = 0x80, + S12E1Tran = 0x100 + }; + enum TLBType { I_TLBS = 0x01, @@ -86,27 +124,131 @@ class MMU : public BaseMMU ALL_TLBS = 0x11 }; + struct CachedState { + explicit CachedState(MMU *_mmu, bool stage2) + : mmu(_mmu), isStage2(stage2) + {} + + void updateMiscReg(ThreadContext *tc, ArmTranslationType tran_type); + + /** Returns the current VMID + * (information stored in the VTTBR_EL2 register) */ + vmid_t getVMID(ThreadContext *tc) const; + + MMU *mmu; + bool isStage2 = false; + CPSR cpsr = 0; + bool aarch64 = false; + ExceptionLevel aarch64EL = EL0; + SCTLR sctlr = 0; + SCR scr = 0; + bool isPriv = false; + bool isSecure = false; + bool isHyp = false; + TTBCR ttbcr = 0; + uint16_t asid = 0; + vmid_t vmid = 0; + PRRR prrr = 0; + NMRR nmrr = 0; + HCR hcr = 0; + uint32_t dacr = 0; + bool miscRegValid = false; + ArmTranslationType curTranType = NormalTran; + + // Indicates whether a stage 2 lookup is also required + bool stage2Req = false; + + // Indicates whether a stage 2 lookup of the table descriptors is + // required. Certain address translation instructions will + // intercept the IPA but the table descriptors still need to be + // translated by the stage2. + bool stage2DescReq = false; + + // Indicates whether all translation requests should + // be routed directly to the stage 2 TLB + bool directToStage2 = false; + }; + MMU(const ArmMMUParams &p); void init() override; - using BaseMMU::translateFunctional; + /** + * Do a functional lookup on the TLB (for debugging) + * and don't modify any internal state + * @param tc thread context to get the context id from + * @param vaddr virtual address to translate + * @param pa returned physical address + * @return if the translation was successful + */ bool translateFunctional(ThreadContext *tc, Addr vaddr, Addr &paddr); Fault translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tran_type); + BaseMMU::Mode mode) override; + + /** + * Do a functional lookup on the TLB (for checker cpu) that + * behaves like a normal lookup without modifying any page table state. + */ + Fault translateFunctional(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Mode mode, ArmTranslationType tran_type); Fault translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tran_type, - bool stage2); + BaseMMU::Mode mode, ArmTranslationType tran_type, bool stage2); - using BaseMMU::translateAtomic; + Fault + translateAtomic(const RequestPtr &req, + ThreadContext *tc, Mode mode) override + { + return translateAtomic(req, tc, mode, NormalTran); + } Fault translateAtomic(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, bool stage2); + BaseMMU::Mode mode, ArmTranslationType tran_type, bool stage2); + Fault translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode, + ArmTranslationType tran_type); - using BaseMMU::translateTiming; + void + translateTiming(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode) override + { + translateTiming(req, tc, translation, mode, NormalTran, false); + } void translateTiming(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode, bool stage2); + void translateTiming( + const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, + ArmTranslationType tran_type, bool stage2); + + Fault translateMmuOff(ThreadContext *tc, const RequestPtr &req, Mode mode, + ArmTranslationType tran_type, Addr vaddr, bool long_desc_format, + CachedState &state); + Fault translateMmuOn(ThreadContext *tc, const RequestPtr &req, Mode mode, + Translation *translation, bool &delay, bool timing, bool functional, + Addr vaddr, ArmFault::TranMethod tranMethod, + CachedState &state); + + Fault translateFs(const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool &delay, + bool timing, ArmTranslationType tran_type, bool functional, + CachedState &state); + Fault translateSe(const RequestPtr &req, ThreadContext *tc, Mode mode, + Translation *translation, bool &delay, bool timing, + CachedState &state); + + Fault translateComplete(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, ArmTranslationType tran_type, + bool call_from_s2); + Fault translateComplete(const RequestPtr &req, ThreadContext *tc, + Translation *translation, Mode mode, ArmTranslationType tran_type, + bool call_from_s2, CachedState &state); + Fault finalizePhysical( + const RequestPtr &req, + ThreadContext *tc, Mode mode) const override; + + void drainResume() override; + + void takeOverFrom(BaseMMU *old_mmu) override; void invalidateMiscReg(TLBType type = ALL_TLBS); @@ -164,8 +306,98 @@ class MMU : public BaseMMU uint64_t getAttr() const { - return getDTBPtr()->getAttr(); + return _attr; } + + /** Accessor functions for memory attributes for last accessed TLB entry + */ + void + setAttr(uint64_t attr) + { + _attr = attr; + } + + /** + * Determine the EL to use for the purpose of a translation given + * a specific translation type. If the translation type doesn't + * specify an EL, we use the current EL. + */ + static ExceptionLevel tranTypeEL(CPSR cpsr, ArmTranslationType type); + + public: + Fault getTE(TlbEntry **te, const RequestPtr &req, + ThreadContext *tc, Mode mode, + Translation *translation, bool timing, bool functional, + bool is_secure, ArmTranslationType tran_type, + bool stage2); + Fault getTE(TlbEntry **te, const RequestPtr &req, + ThreadContext *tc, Mode mode, + Translation *translation, bool timing, bool functional, + bool is_secure, ArmTranslationType tran_type, + CachedState &state); + + Fault getResultTe(TlbEntry **te, const RequestPtr &req, + ThreadContext *tc, Mode mode, + Translation *translation, bool timing, + bool functional, TlbEntry *mergeTe, + CachedState &state); + + Fault checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode, + bool stage2); + Fault checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode, + CachedState &state); + Fault checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode, + ThreadContext *tc, bool stage2); + Fault checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode, + ThreadContext *tc, CachedState &state); + bool checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, + Mode mode, const bool is_priv, CachedState &state); + + public: /* Testing */ + TlbTestInterface *test; + + void setTestInterface(SimObject *ti); + + Fault testTranslation(const RequestPtr &req, Mode mode, + TlbEntry::DomainType domain, CachedState &state); + Fault testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode, + TlbEntry::DomainType domain, + LookupLevel lookup_level, bool stage2); + Fault testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode, + TlbEntry::DomainType domain, + LookupLevel lookup_level, CachedState &state); + + protected: + ContextID miscRegContext; + + public: + CachedState s1State, s2State; + + protected: + uint64_t _attr; // Memory attributes for last accessed TLB entry + + // Cached copies of system-level properties + bool haveLPAE; + bool haveVirtualization; + bool haveLargeAsid64; + uint8_t physAddrRange; + + AddrRange m5opRange; + + CachedState& updateMiscReg( + ThreadContext *tc, ArmTranslationType tran_type, + bool stage2); + + struct Stats : public statistics::Group + { + Stats(statistics::Group *parent); + // Access Stats + mutable statistics::Scalar alignFaults; + mutable statistics::Scalar prefetchFaults; + mutable statistics::Scalar domainFaults; + mutable statistics::Scalar permsFaults; + } stats; + }; template diff --git a/src/arch/arm/stage2_lookup.cc b/src/arch/arm/stage2_lookup.cc index 9ad6688e99..0d9942a744 100644 --- a/src/arch/arm/stage2_lookup.cc +++ b/src/arch/arm/stage2_lookup.cc @@ -55,10 +55,10 @@ using namespace ArmISA; Fault Stage2LookUp::getTe(ThreadContext *tc, TlbEntry *destTe) - { - fault = stage2Tlb->getTE(&stage2Te, req, tc, mode, this, timing, - functional, secure, tranType); + fault = mmu->getTE(&stage2Te, req, tc, mode, this, timing, + functional, secure, tranType, true); + // Call finish if we're done already if ((fault != NoFault) || (stage2Te != NULL)) { // Since we directly requested the table entry (which we need later on @@ -67,19 +67,19 @@ Stage2LookUp::getTe(ThreadContext *tc, TlbEntry *destTe) // entry is now in the TLB this should always hit the cache. if (fault == NoFault) { if (ELIs64(tc, EL2)) - fault = stage2Tlb->checkPermissions64(stage2Te, req, mode, tc); + fault = mmu->checkPermissions64(stage2Te, req, mode, tc, true); else - fault = stage2Tlb->checkPermissions(stage2Te, req, mode); + fault = mmu->checkPermissions(stage2Te, req, mode, true); } - mergeTe(req, mode); + mergeTe(mode); *destTe = stage1Te; } return fault; } void -Stage2LookUp::mergeTe(const RequestPtr &req, BaseMMU::Mode mode) +Stage2LookUp::mergeTe(BaseMMU::Mode mode) { // Check again that we haven't got a fault if (fault == NoFault) { @@ -169,8 +169,9 @@ Stage2LookUp::mergeTe(const RequestPtr &req, BaseMMU::Mode mode) if (fault != NoFault) { // If the second stage of translation generated a fault add the // details of the original stage 1 virtual address - reinterpret_cast(fault.get())->annotate(ArmFault::OVA, - s1Req->getVaddr()); + if (auto arm_fault = reinterpret_cast(fault.get())) { + arm_fault->annotate(ArmFault::OVA, s1Req->getVaddr()); + } } complete = true; } @@ -182,13 +183,14 @@ Stage2LookUp::finish(const Fault &_fault, const RequestPtr &req, fault = _fault; // if we haven't got the table entry get it now if ((fault == NoFault) && (stage2Te == NULL)) { - fault = stage2Tlb->getTE(&stage2Te, req, tc, mode, this, - timing, functional, secure, tranType); + // OLD_LOOK: stage2Tlb + fault = mmu->getTE(&stage2Te, req, tc, mode, this, + timing, functional, secure, tranType, true); } // Now we have the stage 2 table entry we need to merge it with the stage // 1 entry we were given at the start - mergeTe(req, mode); + mergeTe(mode); if (fault != NoFault) { // Returning with a fault requires the original request @@ -196,7 +198,10 @@ Stage2LookUp::finish(const Fault &_fault, const RequestPtr &req, } else if (timing) { // Now notify the original stage 1 translation that we finally have // a result - stage1Tlb->translateComplete(s1Req, tc, transState, mode, tranType, true); + // tran_s1.callFromStage2 = true; + // OLD_LOOK: stage1Tlb + mmu->translateComplete( + s1Req, tc, transState, mode, tranType, true); } // if we have been asked to delete ourselfs do it now if (selfDelete) { diff --git a/src/arch/arm/stage2_lookup.hh b/src/arch/arm/stage2_lookup.hh index 40410ce976..b9b0a9b8d8 100644 --- a/src/arch/arm/stage2_lookup.hh +++ b/src/arch/arm/stage2_lookup.hh @@ -59,15 +59,14 @@ class TLB; class Stage2LookUp : public BaseMMU::Translation { private: - TLB *stage1Tlb; - TLB *stage2Tlb; + MMU *mmu; TlbEntry stage1Te; RequestPtr s1Req; BaseMMU::Translation *transState; BaseMMU::Mode mode; bool timing; bool functional; - TLB::ArmTranslationType tranType; + MMU::ArmTranslationType tranType; TlbEntry *stage2Te; RequestPtr req; Fault fault; @@ -76,22 +75,22 @@ class Stage2LookUp : public BaseMMU::Translation bool secure; public: - Stage2LookUp(TLB *s1Tlb, TLB *s2Tlb, TlbEntry s1Te, const RequestPtr &_req, - BaseMMU::Translation *_transState, BaseMMU::Mode _mode, bool _timing, - bool _functional, bool _secure, TLB::ArmTranslationType _tranType) : - stage1Tlb(s1Tlb), stage2Tlb(s2Tlb), stage1Te(s1Te), s1Req(_req), + Stage2LookUp(MMU *_mmu, TlbEntry s1_te, const RequestPtr &_req, + MMU::Translation *_transState, BaseMMU::Mode _mode, bool _timing, + bool _functional, bool _secure, MMU::ArmTranslationType _tranType) : + mmu(_mmu), stage1Te(s1_te), s1Req(_req), transState(_transState), mode(_mode), timing(_timing), functional(_functional), tranType(_tranType), stage2Te(nullptr), fault(NoFault), complete(false), selfDelete(false), secure(_secure) { req = std::make_shared(); - req->setVirt(s1Te.pAddr(s1Req->getVaddr()), s1Req->getSize(), + req->setVirt(s1_te.pAddr(s1Req->getVaddr()), s1Req->getSize(), s1Req->getFlags(), s1Req->requestorId(), 0); } Fault getTe(ThreadContext *tc, TlbEntry *destTe); - void mergeTe(const RequestPtr &req, BaseMMU::Mode mode); + void mergeTe(BaseMMU::Mode mode); void setSelfDelete() { selfDelete = true; } diff --git a/src/arch/arm/table_walker.cc b/src/arch/arm/table_walker.cc index f1dd348adb..f0ae381d54 100644 --- a/src/arch/arm/table_walker.cc +++ b/src/arch/arm/table_walker.cc @@ -61,7 +61,7 @@ using namespace ArmISA; TableWalker::TableWalker(const Params &p) : ClockedObject(p), requestorId(p.sys->getRequestorId(this)), - port(nullptr), + port(new Port(this, requestorId)), isStage2(p.is_stage2), tlb(NULL), currState(NULL), pending(false), numSquashable(p.num_squash_per_cycle), @@ -128,7 +128,7 @@ TableWalker::WalkerState::WalkerState() : secureLookup(false), rwTable(false), userTable(false), xnTable(false), pxnTable(false), hpd(false), stage2Req(false), stage2Tran(nullptr), timing(false), functional(false), - mode(BaseMMU::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), + mode(BaseMMU::Read), tranType(MMU::NormalTran), l2Desc(l1Desc), delayed(false), tableWalker(nullptr) { } @@ -280,9 +280,9 @@ TableWalker::drainResume() Fault TableWalker::walk(const RequestPtr &_req, ThreadContext *_tc, uint16_t _asid, - vmid_t _vmid, bool _isHyp, BaseMMU::Mode _mode, - BaseMMU::Translation *_trans, bool _timing, bool _functional, - bool secure, TLB::ArmTranslationType tranType, + vmid_t _vmid, bool _isHyp, MMU::Mode _mode, + MMU::Translation *_trans, bool _timing, bool _functional, + bool secure, MMU::ArmTranslationType tranType, bool _stage2Req) { assert(!(_functional && _timing)); @@ -331,7 +331,7 @@ TableWalker::walk(const RequestPtr &_req, ThreadContext *_tc, uint16_t _asid, currState->aarch64 = ELIs64(_tc, EL2); } else { currState->el = - TLB::tranTypeEL(_tc->readMiscReg(MISCREG_CPSR), tranType); + MMU::tranTypeEL(_tc->readMiscReg(MISCREG_CPSR), tranType); currState->aarch64 = ELIs64(_tc, currState->el == EL0 ? EL1 : currState->el); } @@ -533,9 +533,9 @@ TableWalker::processWalkWrapper() } else { // translate the request now that we know it will work stats.walkServiceTime.sample(curTick() - currState->startTime); - tlb->translateTiming(currState->req, currState->tc, - currState->transState, currState->mode); - + mmu->translateTiming(currState->req, currState->tc, + currState->transState, currState->mode, + currState->tranType, isStage2); } // delete the current request @@ -634,7 +634,7 @@ TableWalker::processWalk() // Trickbox address check Fault f; f = testWalk(l1desc_addr, sizeof(uint32_t), - TlbEntry::DomainType::NoAccess, L1); + TlbEntry::DomainType::NoAccess, L1, isStage2); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { @@ -807,7 +807,8 @@ TableWalker::processWalkLPAE() // Trickbox address check Fault f = testWalk(desc_addr, sizeof(uint64_t), - TlbEntry::DomainType::NoAccess, start_lookup_level); + TlbEntry::DomainType::NoAccess, start_lookup_level, + isStage2); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { @@ -1197,7 +1198,7 @@ TableWalker::processWalkAArch64() // Trickbox address check Fault f = testWalk(desc_addr, sizeof(uint64_t), - TlbEntry::DomainType::NoAccess, start_lookup_level); + TlbEntry::DomainType::NoAccess, start_lookup_level, isStage2); if (f) { DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); if (currState->timing) { @@ -1742,7 +1743,8 @@ TableWalker::doL1Descriptor() // Trickbox address check currState->fault = testWalk(l2desc_addr, sizeof(uint32_t), - currState->l1Desc.domain(), L2); + currState->l1Desc.domain(), L2, + isStage2); if (currState->fault) { if (!currState->timing) { @@ -1910,7 +1912,7 @@ TableWalker::doLongDescriptor() // Trickbox address check currState->fault = testWalk( next_desc_addr, sizeof(uint64_t), TlbEntry::DomainType::Client, - toLookupLevel(currState->longDesc.lookupLevel +1)); + toLookupLevel(currState->longDesc.lookupLevel +1), isStage2); if (currState->fault) { if (!currState->timing) { @@ -2054,8 +2056,11 @@ TableWalker::doL1DescriptorWrapper() // Don't finish the translation if a stage 2 look up is underway stats.walkServiceTime.sample(curTick() - currState->startTime); DPRINTF(PageTableWalker, "calling translateTiming again\n"); - tlb->translateTiming(currState->req, currState->tc, - currState->transState, currState->mode); + + mmu->translateTiming(currState->req, currState->tc, + currState->transState, currState->mode, + currState->tranType, isStage2); + stats.walksShortTerminatedAtLevel[0]++; pending = false; @@ -2095,8 +2100,11 @@ TableWalker::doL2DescriptorWrapper() } else { stats.walkServiceTime.sample(curTick() - currState->startTime); DPRINTF(PageTableWalker, "calling translateTiming again\n"); - tlb->translateTiming(currState->req, currState->tc, - currState->transState, currState->mode); + + mmu->translateTiming(currState->req, currState->tc, + currState->transState, currState->mode, + currState->tranType, isStage2); + stats.walksShortTerminatedAtLevel[1]++; } @@ -2172,8 +2180,11 @@ TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) // No additional lookups required DPRINTF(PageTableWalker, "calling translateTiming again\n"); stats.walkServiceTime.sample(curTick() - currState->startTime); - tlb->translateTiming(currState->req, currState->tc, - currState->transState, currState->mode); + + mmu->translateTiming(currState->req, currState->tc, + currState->transState, currState->mode, + currState->tranType, isStage2); + stats.walksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; pending = false; @@ -2221,13 +2232,16 @@ TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, if (isTiming) { auto *tran = new - Stage2Walk(*this, data, event, currState->vaddr); + Stage2Walk(*this, data, event, currState->vaddr, + currState->mode, currState->tranType); currState->stage2Tran = tran; readDataTimed(currState->tc, descAddr, tran, numBytes, flags); fault = tran->fault; } else { fault = readDataUntimed(currState->tc, currState->vaddr, descAddr, data, numBytes, flags, + currState->mode, + currState->tranType, currState->functional); } @@ -2380,10 +2394,10 @@ TableWalker::pendingChange() Fault TableWalker::testWalk(Addr pa, Addr size, TlbEntry::DomainType domain, - LookupLevel lookup_level) + LookupLevel lookup_level, bool stage2) { - return tlb->testWalk(pa, size, currState->vaddr, currState->isSecure, - currState->mode, domain, lookup_level); + return mmu->testWalk(pa, size, currState->vaddr, currState->isSecure, + currState->mode, domain, lookup_level, stage2); } @@ -2410,7 +2424,8 @@ TableWalker::pageSizeNtoStatBin(uint8_t N) Fault TableWalker::readDataUntimed(ThreadContext *tc, Addr vaddr, Addr desc_addr, - uint8_t *data, int num_bytes, Request::Flags flags, bool functional) + uint8_t *data, int num_bytes, Request::Flags flags, BaseMMU::Mode mode, + MMU::ArmTranslationType tran_type, bool functional) { Fault fault; @@ -2418,11 +2433,13 @@ TableWalker::readDataUntimed(ThreadContext *tc, Addr vaddr, Addr desc_addr, auto req = std::make_shared(); req->setVirt(desc_addr, num_bytes, flags | Request::PT_WALK, requestorId, 0); + if (functional) { fault = mmu->translateFunctional(req, tc, BaseMMU::Read, - TLB::NormalTran, true); + tran_type, true); } else { - fault = mmu->translateAtomic(req, tc, BaseMMU::Read, true); + fault = mmu->translateAtomic(req, tc, BaseMMU::Read, + tran_type, true); } // Now do the access. @@ -2459,9 +2476,10 @@ TableWalker::readDataTimed(ThreadContext *tc, Addr desc_addr, } TableWalker::Stage2Walk::Stage2Walk(TableWalker &_parent, - uint8_t *_data, Event *_event, Addr vaddr) + uint8_t *_data, Event *_event, Addr vaddr, BaseMMU::Mode _mode, + MMU::ArmTranslationType tran_type) : data(_data), numBytes(0), event(_event), parent(_parent), - oVAddr(vaddr), fault(NoFault) + oVAddr(vaddr), mode(_mode), tranType(tran_type), fault(NoFault) { req = std::make_shared(); } @@ -2495,7 +2513,7 @@ TableWalker::Stage2Walk::finish(const Fault &_fault, void TableWalker::Stage2Walk::translateTiming(ThreadContext *tc) { - parent.mmu->translateTiming(req, tc, this, BaseMMU::Read, true); + parent.mmu->translateTiming(req, tc, this, mode, tranType, true); } TableWalker::TableWalkerStats::TableWalkerStats(Stats::Group *parent) diff --git a/src/arch/arm/table_walker.hh b/src/arch/arm/table_walker.hh index 165a922950..77fb7fc7de 100644 --- a/src/arch/arm/table_walker.hh +++ b/src/arch/arm/table_walker.hh @@ -41,6 +41,7 @@ #include #include "arch/arm/faults.hh" +#include "arch/arm/mmu.hh" #include "arch/arm/regs/misc.hh" #include "arch/arm/system.hh" #include "arch/arm/tlb.hh" @@ -61,7 +62,6 @@ class ThreadContext; namespace ArmISA { class Translation; class TLB; -class MMU; class TableWalker : public ClockedObject { @@ -828,7 +828,7 @@ class TableWalker : public ClockedObject BaseMMU::Mode mode; /** The translation type that has been requested */ - TLB::ArmTranslationType tranType; + MMU::ArmTranslationType tranType; /** Short-format descriptors */ L1Descriptor l1Desc; @@ -912,12 +912,15 @@ class TableWalker : public ClockedObject Event *event; TableWalker &parent; Addr oVAddr; + BaseMMU::Mode mode; + MMU::ArmTranslationType tranType; public: Fault fault; Stage2Walk(TableWalker &_parent, uint8_t *_data, Event *_event, - Addr vaddr); + Addr vaddr, BaseMMU::Mode mode, + MMU::ArmTranslationType tran_type); void markDelayed() {} @@ -937,6 +940,7 @@ class TableWalker : public ClockedObject Fault readDataUntimed(ThreadContext *tc, Addr vaddr, Addr desc_addr, uint8_t *data, int num_bytes, Request::Flags flags, + BaseMMU::Mode mode, MMU::ArmTranslationType tran_type, bool functional); void readDataTimed(ThreadContext *tc, Addr desc_addr, Stage2Walk *translation, int num_bytes, @@ -1033,11 +1037,10 @@ class TableWalker : public ClockedObject uint16_t asid, vmid_t _vmid, bool _isHyp, BaseMMU::Mode mode, BaseMMU::Translation *_trans, bool timing, bool functional, bool secure, - TLB::ArmTranslationType tranType, bool _stage2Req); + MMU::ArmTranslationType tranType, bool _stage2Req); void setMmu(MMU *_mmu) { mmu = _mmu; } void setTlb(TLB *_tlb) { tlb = _tlb; } - void setPort(Port *_port) { port = _port; } TLB* getTlb() { return tlb; } void memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, uint8_t texcb, bool s); @@ -1101,7 +1104,7 @@ class TableWalker : public ClockedObject static uint8_t pageSizeNtoStatBin(uint8_t N); Fault testWalk(Addr pa, Addr size, TlbEntry::DomainType domain, - LookupLevel lookup_level); + LookupLevel lookup_level, bool stage2); }; } // namespace ArmISA diff --git a/src/arch/arm/tlb.cc b/src/arch/arm/tlb.cc index 703416b71d..7aef0c00ef 100644 --- a/src/arch/arm/tlb.cc +++ b/src/arch/arm/tlb.cc @@ -44,32 +44,14 @@ #include #include -#include "arch/arm/faults.hh" -#include "arch/arm/isa.hh" -#include "arch/arm/pagetable.hh" -#include "arch/arm/reg_abi.hh" -#include "arch/arm/self_debug.hh" -#include "arch/arm/stage2_lookup.hh" -#include "arch/arm/system.hh" #include "arch/arm/table_walker.hh" #include "arch/arm/tlbi_op.hh" #include "arch/arm/utility.hh" -#include "base/compiler.hh" -#include "base/inifile.hh" -#include "base/str.hh" #include "base/trace.hh" -#include "cpu/base.hh" #include "cpu/thread_context.hh" -#include "debug/Checkpoint.hh" #include "debug/TLB.hh" #include "debug/TLBVerbose.hh" -#include "mem/packet_access.hh" -#include "mem/page_table.hh" -#include "mem/request.hh" #include "params/ArmTLB.hh" -#include "sim/full_system.hh" -#include "sim/process.hh" -#include "sim/pseudo_inst.hh" namespace gem5 { @@ -78,29 +60,10 @@ using namespace ArmISA; TLB::TLB(const ArmTLBParams &p) : BaseTLB(p), table(new TlbEntry[p.size]), size(p.size), - isStage2(p.is_stage2), stage2Req(false), stage2DescReq(false), _attr(0), - directToStage2(false), tableWalker(nullptr), stage2Tlb(nullptr), - test(nullptr), stats(this), rangeMRU(1), - aarch64(false), aarch64EL(EL0), isPriv(false), isSecure(false), - isHyp(false), asid(0), vmid(0), hcr(0), dacr(0), - miscRegValid(false), miscRegContext(0), curTranType(NormalTran) + isStage2(p.is_stage2), + tableWalker(nullptr), + stats(this), rangeMRU(1), vmid(0) { - // Cache system-level properties - if (FullSystem) { - ArmSystem *arm_sys = dynamic_cast(p.sys); - assert(arm_sys); - haveLPAE = arm_sys->haveLPAE(); - haveVirtualization = arm_sys->haveVirtualization(); - haveLargeAsid64 = arm_sys->haveLargeAsid64(); - physAddrRange = arm_sys->physAddrRange(); - } else { - haveLPAE = false; - haveVirtualization = false; - haveLargeAsid64 = false; - physAddrRange = 48; - } - - m5opRange = p.sys->m5opRange(); } TLB::~TLB() @@ -115,53 +78,6 @@ TLB::setTableWalker(TableWalker *table_walker) tableWalker->setTlb(this); } -bool -TLB::translateFunctional(ThreadContext *tc, Addr va, Addr &pa) -{ - updateMiscReg(tc); - - if (directToStage2) { - assert(stage2Tlb); - return stage2Tlb->translateFunctional(tc, va, pa); - } - - TlbEntry *e = lookup(va, asid, vmid, isHyp, isSecure, true, false, - aarch64 ? aarch64EL : EL1, false, BaseMMU::Read); - if (!e) - return false; - pa = e->pAddr(va); - return true; -} - -Fault -TLB::finalizePhysical(const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode) const -{ - const Addr paddr = req->getPaddr(); - - if (m5opRange.contains(paddr)) { - uint8_t func; - pseudo_inst::decodeAddrOffset(paddr - m5opRange.start(), func); - req->setLocalAccessor( - [func, mode](ThreadContext *tc, PacketPtr pkt) -> Cycles - { - uint64_t ret; - if (inAArch64(tc)) - pseudo_inst::pseudoInst(tc, func, ret); - else - pseudo_inst::pseudoInst(tc, func, ret); - - if (mode == BaseMMU::Read) - pkt->setLE(ret); - - return Cycles(1); - } - ); - } - - return NoFault; -} - TlbEntry* TLB::lookup(Addr va, uint16_t asn, vmid_t vmid, bool hyp, bool secure, bool functional, bool ignore_asn, ExceptionLevel target_el, @@ -454,35 +370,9 @@ TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup, } } -void -TLB::drainResume() -{ - // We might have unserialized something or switched CPUs, so make - // sure to re-read the misc regs. - miscRegValid = false; -} - void TLB::takeOverFrom(BaseTLB *_otlb) { - TLB *otlb = dynamic_cast(_otlb); - /* Make sure we actually have a valid type */ - if (otlb) { - _attr = otlb->_attr; - directToStage2 = otlb->directToStage2; - stage2Req = otlb->stage2Req; - stage2DescReq = otlb->stage2DescReq; - - /* Sync the stage2 MMU if they exist in both - * the old CPU and the new - */ - if (!isStage2 && - stage2Tlb && otlb->stage2Tlb) { - stage2Tlb->takeOverFrom(otlb->stage2Tlb); - } - } else { - panic("Incompatible TLB type!"); - } } TLB::TlbStats::TlbStats(statistics::Group *parent) @@ -505,14 +395,6 @@ TLB::TlbStats::TlbStats(statistics::Group *parent) "Number of times TLB was flushed by ASID"), ADD_STAT(flushedEntries, statistics::units::Count::get(), "Number of entries that have been flushed from TLB"), - ADD_STAT(alignFaults, statistics::units::Count::get(), - "Number of TLB faults due to alignment restrictions"), - ADD_STAT(prefetchFaults, statistics::units::Count::get(), - "Number of TLB faults due to prefetch"), - ADD_STAT(domainFaults, statistics::units::Count::get(), - "Number of TLB faults due to domain restrictions"), - ADD_STAT(permsFaults, statistics::units::Count::get(), - "Number of TLB faults due to permissions restrictions"), ADD_STAT(readAccesses, statistics::units::Count::get(), "DTB read accesses", readHits + readMisses), ADD_STAT(writeAccesses, statistics::units::Count::get(), "DTB write accesses", @@ -537,1212 +419,10 @@ TLB::regProbePoints() ppRefills.reset(new probing::PMU(getProbeManager(), "Refills")); } -Fault -TLB::translateSe(const RequestPtr &req, ThreadContext *tc, BaseMMU::Mode mode, - BaseMMU::Translation *translation, bool &delay, bool timing) -{ - updateMiscReg(tc); - Addr vaddr_tainted = req->getVaddr(); - Addr vaddr = 0; - if (aarch64) - vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, (TCR)ttbcr, - mode==BaseMMU::Execute); - else - vaddr = vaddr_tainted; - Request::Flags flags = req->getFlags(); - - bool is_fetch = (mode == BaseMMU::Execute); - bool is_write = (mode == BaseMMU::Write); - - if (!is_fetch) { - if (sctlr.a || !(flags & AllowUnaligned)) { - if (vaddr & mask(flags & AlignmentMask)) { - // LPAE is always disabled in SE mode - return std::make_shared( - vaddr_tainted, - TlbEntry::DomainType::NoAccess, is_write, - ArmFault::AlignmentFault, isStage2, - ArmFault::VmsaTran); - } - } - } - - Addr paddr; - Process *p = tc->getProcessPtr(); - - if (!p->pTable->translate(vaddr, paddr)) - return std::make_shared(vaddr_tainted); - req->setPaddr(paddr); - - return finalizePhysical(req, tc, mode); -} - -Fault -TLB::checkPermissions(TlbEntry *te, const RequestPtr &req, BaseMMU::Mode mode) -{ - // a data cache maintenance instruction that operates by MVA does - // not generate a Data Abort exeception due to a Permission fault - if (req->isCacheMaintenance()) { - return NoFault; - } - - Addr vaddr = req->getVaddr(); // 32-bit don't have to purify - Request::Flags flags = req->getFlags(); - bool is_fetch = (mode == BaseMMU::Execute); - bool is_write = (mode == BaseMMU::Write); - bool is_priv = isPriv && !(flags & UserMode); - - // Get the translation type from the actuall table entry - ArmFault::TranMethod tranMethod = te->longDescFormat ? ArmFault::LpaeTran - : ArmFault::VmsaTran; - - // If this is the second stage of translation and the request is for a - // stage 1 page table walk then we need to check the HCR.PTW bit. This - // allows us to generate a fault if the request targets an area marked - // as a device or strongly ordered. - if (isStage2 && req->isPTWalk() && hcr.ptw && - (te->mtype != TlbEntry::MemoryType::Normal)) { - return std::make_shared( - vaddr, te->domain, is_write, - ArmFault::PermissionLL + te->lookupLevel, - isStage2, tranMethod); - } - - // Generate an alignment fault for unaligned data accesses to device or - // strongly ordered memory - if (!is_fetch) { - if (te->mtype != TlbEntry::MemoryType::Normal) { - if (vaddr & mask(flags & AlignmentMask)) { - stats.alignFaults++; - return std::make_shared( - vaddr, TlbEntry::DomainType::NoAccess, is_write, - ArmFault::AlignmentFault, isStage2, - tranMethod); - } - } - } - - if (te->nonCacheable) { - // Prevent prefetching from I/O devices. - if (req->isPrefetch()) { - // Here we can safely use the fault status for the short - // desc. format in all cases - return std::make_shared( - vaddr, ArmFault::PrefetchUncacheable, - isStage2, tranMethod); - } - } - - if (!te->longDescFormat) { - switch ((dacr >> (static_cast(te->domain) * 2)) & 0x3) { - case 0: - stats.domainFaults++; - DPRINTF(TLB, "TLB Fault: Data abort on domain. DACR: %#x" - " domain: %#x write:%d\n", dacr, - static_cast(te->domain), is_write); - if (is_fetch) { - // Use PC value instead of vaddr because vaddr might - // be aligned to cache line and should not be the - // address reported in FAR - return std::make_shared( - req->getPC(), - ArmFault::DomainLL + te->lookupLevel, - isStage2, tranMethod); - } else - return std::make_shared( - vaddr, te->domain, is_write, - ArmFault::DomainLL + te->lookupLevel, - isStage2, tranMethod); - case 1: - // Continue with permissions check - break; - case 2: - panic("UNPRED domain\n"); - case 3: - return NoFault; - } - } - - // The 'ap' variable is AP[2:0] or {AP[2,1],1b'0}, i.e. always three bits - uint8_t ap = te->longDescFormat ? te->ap << 1 : te->ap; - uint8_t hap = te->hap; - - if (sctlr.afe == 1 || te->longDescFormat) - ap |= 1; - - bool abt; - bool isWritable = true; - // If this is a stage 2 access (eg for reading stage 1 page table entries) - // then don't perform the AP permissions check, we stil do the HAP check - // below. - if (isStage2) { - abt = false; - } else { - switch (ap) { - case 0: - DPRINTF(TLB, "Access permissions 0, checking rs:%#x\n", - (int)sctlr.rs); - if (!sctlr.xp) { - switch ((int)sctlr.rs) { - case 2: - abt = is_write; - break; - case 1: - abt = is_write || !is_priv; - break; - case 0: - case 3: - default: - abt = true; - break; - } - } else { - abt = true; - } - break; - case 1: - abt = !is_priv; - break; - case 2: - abt = !is_priv && is_write; - isWritable = is_priv; - break; - case 3: - abt = false; - break; - case 4: - panic("UNPRED premissions\n"); - case 5: - abt = !is_priv || is_write; - isWritable = false; - break; - case 6: - case 7: - abt = is_write; - isWritable = false; - break; - default: - panic("Unknown permissions %#x\n", ap); - } - } - - bool hapAbt = is_write ? !(hap & 2) : !(hap & 1); - bool xn = te->xn || (isWritable && sctlr.wxn) || - (ap == 3 && sctlr.uwxn && is_priv); - if (is_fetch && (abt || xn || - (te->longDescFormat && te->pxn && is_priv) || - (isSecure && te->ns && scr.sif))) { - stats.permsFaults++; - DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. AP:%d " - "priv:%d write:%d ns:%d sif:%d sctlr.afe: %d \n", - ap, is_priv, is_write, te->ns, scr.sif,sctlr.afe); - // Use PC value instead of vaddr because vaddr might be aligned to - // cache line and should not be the address reported in FAR - return std::make_shared( - req->getPC(), - ArmFault::PermissionLL + te->lookupLevel, - isStage2, tranMethod); - } else if (abt | hapAbt) { - stats.permsFaults++; - DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d priv:%d" - " write:%d\n", ap, is_priv, is_write); - return std::make_shared( - vaddr, te->domain, is_write, - ArmFault::PermissionLL + te->lookupLevel, - isStage2 | !abt, tranMethod); - } - return NoFault; -} - - -Fault -TLB::checkPermissions64(TlbEntry *te, const RequestPtr &req, - BaseMMU::Mode mode, ThreadContext *tc) -{ - assert(aarch64); - - // A data cache maintenance instruction that operates by VA does - // not generate a Permission fault unless: - // * It is a data cache invalidate (dc ivac) which requires write - // permissions to the VA, or - // * It is executed from EL0 - if (req->isCacheClean() && aarch64EL != EL0 && !isStage2) { - return NoFault; - } - - Addr vaddr_tainted = req->getVaddr(); - Addr vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, (TCR)ttbcr, - mode==BaseMMU::Execute); - - Request::Flags flags = req->getFlags(); - bool is_fetch = (mode == BaseMMU::Execute); - // Cache clean operations require read permissions to the specified VA - bool is_write = !req->isCacheClean() && mode == BaseMMU::Write; - bool is_atomic = req->isAtomic(); - GEM5_VAR_USED bool is_priv = isPriv && !(flags & UserMode); - - updateMiscReg(tc, curTranType); - - // If this is the second stage of translation and the request is for a - // stage 1 page table walk then we need to check the HCR.PTW bit. This - // allows us to generate a fault if the request targets an area marked - // as a device or strongly ordered. - if (isStage2 && req->isPTWalk() && hcr.ptw && - (te->mtype != TlbEntry::MemoryType::Normal)) { - return std::make_shared( - vaddr_tainted, te->domain, is_write, - ArmFault::PermissionLL + te->lookupLevel, - isStage2, ArmFault::LpaeTran); - } - - // Generate an alignment fault for unaligned accesses to device or - // strongly ordered memory - if (!is_fetch) { - if (te->mtype != TlbEntry::MemoryType::Normal) { - if (vaddr & mask(flags & AlignmentMask)) { - stats.alignFaults++; - return std::make_shared( - vaddr_tainted, - TlbEntry::DomainType::NoAccess, - is_atomic ? false : is_write, - ArmFault::AlignmentFault, isStage2, - ArmFault::LpaeTran); - } - } - } - - if (te->nonCacheable) { - // Prevent prefetching from I/O devices. - if (req->isPrefetch()) { - // Here we can safely use the fault status for the short - // desc. format in all cases - return std::make_shared( - vaddr_tainted, - ArmFault::PrefetchUncacheable, - isStage2, ArmFault::LpaeTran); - } - } - - uint8_t ap = 0x3 & (te->ap); // 2-bit access protection field - bool grant = false; - - bool wxn = sctlr.wxn; - uint8_t xn = te->xn; - uint8_t pxn = te->pxn; - bool r = (!is_write && !is_fetch); - bool w = is_write; - bool x = is_fetch; - - if (ArmSystem::haveEL(tc, EL3) && isSecure && te->ns && scr.sif) - xn = true; - - // grant_read is used for faults from an atomic instruction that - // both reads and writes from a memory location. From a ISS point - // of view they count as read if a read to that address would have - // generated the fault; they count as writes otherwise - bool grant_read = true; - DPRINTF(TLBVerbose, "Checking permissions: ap:%d, xn:%d, pxn:%d, r:%d, " - "w:%d, x:%d, is_priv: %d, wxn: %d\n", ap, xn, - pxn, r, w, x, is_priv, wxn); - - if (isStage2) { - assert(ArmSystem::haveVirtualization(tc) && aarch64EL != EL2); - // In stage 2 we use the hypervisor access permission bits. - // The following permissions are described in ARM DDI 0487A.f - // D4-1802 - uint8_t hap = 0x3 & te->hap; - grant_read = hap & 0x1; - if (is_fetch) { - // sctlr.wxn overrides the xn bit - grant = !wxn && !xn; - } else if (is_atomic) { - grant = hap; - } else if (is_write) { - grant = hap & 0x2; - } else { // is_read - grant = grant_read; - } - } else { - switch (aarch64EL) { - case EL0: - { - grant_read = ap & 0x1; - uint8_t perm = (ap << 2) | (xn << 1) | pxn; - switch (perm) { - case 0: - case 1: - case 8: - case 9: - grant = x; - break; - case 4: - case 5: - grant = r || w || (x && !wxn); - break; - case 6: - case 7: - grant = r || w; - break; - case 12: - case 13: - grant = r || x; - break; - case 14: - case 15: - grant = r; - break; - default: - grant = false; - } - } - break; - case EL1: - { - if (checkPAN(tc, ap, req, mode, is_priv)) { - grant = false; - grant_read = false; - break; - } - - uint8_t perm = (ap << 2) | (xn << 1) | pxn; - switch (perm) { - case 0: - case 2: - grant = r || w || (x && !wxn); - break; - case 1: - case 3: - case 4: - case 5: - case 6: - case 7: - // regions that are writeable at EL0 should not be - // executable at EL1 - grant = r || w; - break; - case 8: - case 10: - case 12: - case 14: - grant = r || x; - break; - case 9: - case 11: - case 13: - case 15: - grant = r; - break; - default: - grant = false; - } - } - break; - case EL2: - if (hcr.e2h && checkPAN(tc, ap, req, mode, is_priv)) { - grant = false; - grant_read = false; - break; - } - [[fallthrough]]; - case EL3: - { - uint8_t perm = (ap & 0x2) | xn; - switch (perm) { - case 0: - grant = r || w || (x && !wxn); - break; - case 1: - grant = r || w; - break; - case 2: - grant = r || x; - break; - case 3: - grant = r; - break; - default: - grant = false; - } - } - break; - } - } - - if (!grant) { - if (is_fetch) { - stats.permsFaults++; - DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. " - "AP:%d priv:%d write:%d ns:%d sif:%d " - "sctlr.afe: %d\n", - ap, is_priv, is_write, te->ns, scr.sif, sctlr.afe); - // Use PC value instead of vaddr because vaddr might be aligned to - // cache line and should not be the address reported in FAR - return std::make_shared( - req->getPC(), - ArmFault::PermissionLL + te->lookupLevel, - isStage2, ArmFault::LpaeTran); - } else { - stats.permsFaults++; - DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d " - "priv:%d write:%d\n", ap, is_priv, is_write); - return std::make_shared( - vaddr_tainted, te->domain, - (is_atomic && !grant_read) ? false : is_write, - ArmFault::PermissionLL + te->lookupLevel, - isStage2, ArmFault::LpaeTran); - } - } - - return NoFault; -} - -bool -TLB::checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, - BaseMMU::Mode mode, const bool is_priv) -{ - // The PAN bit has no effect on: - // 1) Instruction accesses. - // 2) Data Cache instructions other than DC ZVA - // 3) Address translation instructions, other than ATS1E1RP and - // ATS1E1WP when ARMv8.2-ATS1E1 is implemented. (Unimplemented in - // gem5) - // 4) Instructions to be treated as unprivileged, unless - // HCR_EL2.{E2H, TGE} == {1, 0} - const AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1); - if (mmfr1.pan && cpsr.pan && (ap & 0x1) && mode != BaseMMU::Execute) { - if (req->isCacheMaintenance() && - !(req->getFlags() & Request::CACHE_BLOCK_ZERO)) { - // Cache maintenance other than DC ZVA - return false; - } else if (!is_priv && !(hcr.e2h && !hcr.tge)) { - // Treated as unprivileged unless HCR_EL2.{E2H, TGE} == {1, 0} - return false; - } - return true; - } - - return false; -} - -Fault -TLB::translateMmuOff(ThreadContext *tc, const RequestPtr &req, - BaseMMU::Mode mode, TLB::ArmTranslationType tranType, - Addr vaddr, bool long_desc_format) -{ - bool is_fetch = (mode == BaseMMU::Execute); - bool is_atomic = req->isAtomic(); - req->setPaddr(vaddr); - // When the MMU is off the security attribute corresponds to the - // security state of the processor - if (isSecure) - req->setFlags(Request::SECURE); - - if (aarch64) { - bool selbit = bits(vaddr, 55); - TCR tcr1 = tc->readMiscReg(MISCREG_TCR_EL1); - int topbit = computeAddrTop(tc, selbit, is_fetch, tcr1, currEL(tc)); - int addr_sz = bits(vaddr, topbit, physAddrRange); - if (addr_sz != 0){ - Fault f; - if (is_fetch) - f = std::make_shared(vaddr, - ArmFault::AddressSizeLL, isStage2, ArmFault::LpaeTran); - else - f = std::make_shared( vaddr, - TlbEntry::DomainType::NoAccess, - is_atomic ? false : mode==BaseMMU::Write, - ArmFault::AddressSizeLL, isStage2, ArmFault::LpaeTran); - return f; - } - } - - // @todo: double check this (ARM ARM issue C B3.2.1) - if (long_desc_format || sctlr.tre == 0 || nmrr.ir0 == 0 || - nmrr.or0 == 0 || prrr.tr0 != 0x2) { - if (!req->isCacheMaintenance()) { - req->setFlags(Request::UNCACHEABLE); - } - req->setFlags(Request::STRICT_ORDER); - } - - // Set memory attributes - TlbEntry temp_te; - temp_te.ns = !isSecure; - bool dc = (HaveVirtHostExt(tc) - && hcr.e2h == 1 && hcr.tge == 1) ? 0: hcr.dc; - bool i_cacheability = sctlr.i && !sctlr.m; - if (isStage2 || !dc || isSecure || - (isHyp && !(tranType & S1CTran))) { - - temp_te.mtype = is_fetch ? TlbEntry::MemoryType::Normal - : TlbEntry::MemoryType::StronglyOrdered; - temp_te.innerAttrs = i_cacheability? 0x2: 0x0; - temp_te.outerAttrs = i_cacheability? 0x2: 0x0; - temp_te.shareable = true; - temp_te.outerShareable = true; - } else { - temp_te.mtype = TlbEntry::MemoryType::Normal; - temp_te.innerAttrs = 0x3; - temp_te.outerAttrs = 0x3; - temp_te.shareable = false; - temp_te.outerShareable = false; - } - temp_te.setAttributes(long_desc_format); - DPRINTF(TLBVerbose, "(No MMU) setting memory attributes: shareable: " - "%d, innerAttrs: %d, outerAttrs: %d, isStage2: %d\n", - temp_te.shareable, temp_te.innerAttrs, temp_te.outerAttrs, - isStage2); - setAttr(temp_te.attributes); - - return testTranslation(req, mode, TlbEntry::DomainType::NoAccess); -} - -Fault -TLB::translateMmuOn(ThreadContext* tc, const RequestPtr &req, - BaseMMU::Mode mode, BaseMMU::Translation *translation, - bool &delay, bool timing, - bool functional, Addr vaddr, - ArmFault::TranMethod tranMethod) -{ - TlbEntry *te = NULL; - bool is_fetch = (mode == BaseMMU::Execute); - TlbEntry mergeTe; - - Request::Flags flags = req->getFlags(); - Addr vaddr_tainted = req->getVaddr(); - - Fault fault = getResultTe(&te, req, tc, mode, translation, timing, - functional, &mergeTe); - // only proceed if we have a valid table entry - if ((te == NULL) && (fault == NoFault)) delay = true; - - // If we have the table entry transfer some of the attributes to the - // request that triggered the translation - if (te != NULL) { - // Set memory attributes - DPRINTF(TLBVerbose, - "Setting memory attributes: shareable: %d, innerAttrs: %d, " - "outerAttrs: %d, mtype: %d, isStage2: %d\n", - te->shareable, te->innerAttrs, te->outerAttrs, - static_cast(te->mtype), isStage2); - setAttr(te->attributes); - - if (te->nonCacheable && !req->isCacheMaintenance()) - req->setFlags(Request::UNCACHEABLE); - - // Require requests to be ordered if the request goes to - // strongly ordered or device memory (i.e., anything other - // than normal memory requires strict order). - if (te->mtype != TlbEntry::MemoryType::Normal) - req->setFlags(Request::STRICT_ORDER); - - Addr pa = te->pAddr(vaddr); - req->setPaddr(pa); - - if (isSecure && !te->ns) { - req->setFlags(Request::SECURE); - } - if (!is_fetch && fault == NoFault && - (vaddr & mask(flags & AlignmentMask)) && - (te->mtype != TlbEntry::MemoryType::Normal)) { - // Unaligned accesses to Device memory should always cause an - // abort regardless of sctlr.a - stats.alignFaults++; - bool is_write = (mode == BaseMMU::Write); - return std::make_shared( - vaddr_tainted, - TlbEntry::DomainType::NoAccess, is_write, - ArmFault::AlignmentFault, isStage2, - tranMethod); - } - - // Check for a trickbox generated address fault - if (fault == NoFault) - fault = testTranslation(req, mode, te->domain); - } - - if (fault == NoFault) { - // Don't try to finalize a physical address unless the - // translation has completed (i.e., there is a table entry). - return te ? finalizePhysical(req, tc, mode) : NoFault; - } else { - return fault; - } -} - -Fault -TLB::translateFs(const RequestPtr &req, ThreadContext *tc, BaseMMU::Mode mode, - BaseMMU::Translation *translation, bool &delay, bool timing, - TLB::ArmTranslationType tranType, bool functional) -{ - // No such thing as a functional timing access - assert(!(timing && functional)); - - updateMiscReg(tc, tranType); - - Addr vaddr_tainted = req->getVaddr(); - Addr vaddr = 0; - if (aarch64) - vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, (TCR)ttbcr, - mode==BaseMMU::Execute); - else - vaddr = vaddr_tainted; - Request::Flags flags = req->getFlags(); - - bool is_fetch = (mode == BaseMMU::Execute); - bool is_write = (mode == BaseMMU::Write); - bool long_desc_format = aarch64 || longDescFormatInUse(tc); - ArmFault::TranMethod tranMethod = long_desc_format ? ArmFault::LpaeTran - : ArmFault::VmsaTran; - - DPRINTF(TLBVerbose, - "CPSR is priv:%d UserMode:%d secure:%d S1S2NsTran:%d\n", - isPriv, flags & UserMode, isSecure, tranType & S1S2NsTran); - - DPRINTF(TLB, "translateFs addr %#x, mode %d, st2 %d, scr %#x sctlr %#x " - "flags %#lx tranType 0x%x\n", vaddr_tainted, mode, isStage2, - scr, sctlr, flags, tranType); - - if ((req->isInstFetch() && (!sctlr.i)) || - ((!req->isInstFetch()) && (!sctlr.c))){ - if (!req->isCacheMaintenance()) { - req->setFlags(Request::UNCACHEABLE); - } - req->setFlags(Request::STRICT_ORDER); - } - if (!is_fetch) { - if (sctlr.a || !(flags & AllowUnaligned)) { - if (vaddr & mask(flags & AlignmentMask)) { - stats.alignFaults++; - return std::make_shared( - vaddr_tainted, - TlbEntry::DomainType::NoAccess, is_write, - ArmFault::AlignmentFault, isStage2, - tranMethod); - } - } - } - - bool vm = hcr.vm; - if (HaveVirtHostExt(tc) && hcr.e2h == 1 && hcr.tge ==1) - vm = 0; - else if (hcr.dc == 1) - vm = 1; - - Fault fault = NoFault; - // If guest MMU is off or hcr.vm=0 go straight to stage2 - if ((isStage2 && !vm) || (!isStage2 && !sctlr.m)) { - fault = translateMmuOff(tc, req, mode, tranType, vaddr, - long_desc_format); - } else { - DPRINTF(TLBVerbose, "Translating %s=%#x context=%d\n", - isStage2 ? "IPA" : "VA", vaddr_tainted, asid); - // Translation enabled - fault = translateMmuOn(tc, req, mode, translation, delay, timing, - functional, vaddr, tranMethod); - } - - // Check for Debug Exceptions - SelfDebug *sd = ArmISA::ISA::getSelfDebug(tc); - - if (sd->enabled() && fault == NoFault) { - fault = sd->testDebug(tc, req, mode); - } - - return fault; -} - -Fault -TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tranType) -{ - updateMiscReg(tc, tranType); - - if (directToStage2) { - assert(stage2Tlb); - return stage2Tlb->translateAtomic(req, tc, mode, tranType); - } - - bool delay = false; - Fault fault; - if (FullSystem) - fault = translateFs(req, tc, mode, NULL, delay, false, tranType); - else - fault = translateSe(req, tc, mode, NULL, delay, false); - assert(!delay); - return fault; -} - -Fault -TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, TLB::ArmTranslationType tranType) -{ - updateMiscReg(tc, tranType); - - if (directToStage2) { - assert(stage2Tlb); - return stage2Tlb->translateFunctional(req, tc, mode, tranType); - } - - bool delay = false; - Fault fault; - if (FullSystem) - fault = translateFs(req, tc, mode, NULL, delay, false, tranType, true); - else - fault = translateSe(req, tc, mode, NULL, delay, false); - assert(!delay); - return fault; -} - -void -TLB::translateTiming(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, BaseMMU::Mode mode, - TLB::ArmTranslationType tranType) -{ - updateMiscReg(tc, tranType); - - if (directToStage2) { - assert(stage2Tlb); - stage2Tlb->translateTiming(req, tc, translation, mode, tranType); - return; - } - - assert(translation); - - translateComplete(req, tc, translation, mode, tranType, isStage2); -} - -Fault -TLB::translateComplete(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, BaseMMU::Mode mode, - TLB::ArmTranslationType tranType, bool callFromS2) -{ - bool delay = false; - Fault fault; - if (FullSystem) - fault = translateFs(req, tc, mode, translation, delay, true, tranType); - else - fault = translateSe(req, tc, mode, translation, delay, true); - DPRINTF(TLBVerbose, "Translation returning delay=%d fault=%d\n", delay, fault != - NoFault); - // If we have a translation, and we're not in the middle of doing a stage - // 2 translation tell the translation that we've either finished or its - // going to take a while. By not doing this when we're in the middle of a - // stage 2 translation we prevent marking the translation as delayed twice, - // one when the translation starts and again when the stage 1 translation - // completes. - - if (translation && (callFromS2 || !stage2Req || req->hasPaddr() || - fault != NoFault)) { - if (!delay) - translation->finish(fault, req, tc, mode); - else - translation->markDelayed(); - } - return fault; -} - Port * TLB::getTableWalkerPort() { return &tableWalker->getTableWalkerPort(); } -vmid_t -TLB::getVMID(ThreadContext *tc) const -{ - AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1); - VTCR_t vtcr = tc->readMiscReg(MISCREG_VTCR_EL2); - vmid_t vmid = 0; - - switch (mmfr1.vmidbits) { - case 0b0000: - // 8 bits - vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48); - break; - case 0b0010: - if (vtcr.vs && ELIs64(tc, EL2)) { - // 16 bits - vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 63, 48); - } else { - // 8 bits - vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48); - } - break; - default: - panic("Reserved ID_AA64MMFR1_EL1.VMIDBits value: %#x", - mmfr1.vmidbits); - } - - return vmid; -} - -void -TLB::updateMiscReg(ThreadContext *tc, ArmTranslationType tranType) -{ - // check if the regs have changed, or the translation mode is different. - // NOTE: the tran type doesn't affect stage 2 TLB's as they only handle - // one type of translation anyway - if (miscRegValid && miscRegContext == tc->contextId() && - ((tranType == curTranType) || isStage2)) { - return; - } - - DPRINTF(TLBVerbose, "TLB variables changed!\n"); - cpsr = tc->readMiscReg(MISCREG_CPSR); - - // Dependencies: SCR/SCR_EL3, CPSR - isSecure = ArmISA::isSecure(tc) && - !(tranType & HypMode) && !(tranType & S1S2NsTran); - - aarch64EL = tranTypeEL(cpsr, tranType); - aarch64 = isStage2 ? - ELIs64(tc, EL2) : - ELIs64(tc, aarch64EL == EL0 ? EL1 : aarch64EL); - - hcr = tc->readMiscReg(MISCREG_HCR_EL2); - if (aarch64) { // AArch64 - // determine EL we need to translate in - switch (aarch64EL) { - case EL0: - if (HaveVirtHostExt(tc) && hcr.tge == 1 && hcr.e2h == 1) { - // VHE code for EL2&0 regime - sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2); - ttbcr = tc->readMiscReg(MISCREG_TCR_EL2); - uint64_t ttbr_asid = ttbcr.a1 ? - tc->readMiscReg(MISCREG_TTBR1_EL2) : - tc->readMiscReg(MISCREG_TTBR0_EL2); - asid = bits(ttbr_asid, - (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); - - } else { - sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1); - ttbcr = tc->readMiscReg(MISCREG_TCR_EL1); - uint64_t ttbr_asid = ttbcr.a1 ? - tc->readMiscReg(MISCREG_TTBR1_EL1) : - tc->readMiscReg(MISCREG_TTBR0_EL1); - asid = bits(ttbr_asid, - (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); - - } - break; - case EL1: - { - sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1); - ttbcr = tc->readMiscReg(MISCREG_TCR_EL1); - uint64_t ttbr_asid = ttbcr.a1 ? - tc->readMiscReg(MISCREG_TTBR1_EL1) : - tc->readMiscReg(MISCREG_TTBR0_EL1); - asid = bits(ttbr_asid, - (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); - } - break; - case EL2: - sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2); - ttbcr = tc->readMiscReg(MISCREG_TCR_EL2); - if (hcr.e2h == 1) { - // VHE code for EL2&0 regime - uint64_t ttbr_asid = ttbcr.a1 ? - tc->readMiscReg(MISCREG_TTBR1_EL2) : - tc->readMiscReg(MISCREG_TTBR0_EL2); - asid = bits(ttbr_asid, - (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); - } else { - asid = -1; - } - break; - case EL3: - sctlr = tc->readMiscReg(MISCREG_SCTLR_EL3); - ttbcr = tc->readMiscReg(MISCREG_TCR_EL3); - asid = -1; - break; - } - - scr = tc->readMiscReg(MISCREG_SCR_EL3); - isPriv = aarch64EL != EL0; - if (haveVirtualization) { - vmid = getVMID(tc); - isHyp = aarch64EL == EL2; - isHyp |= tranType & HypMode; - isHyp &= (tranType & S1S2NsTran) == 0; - isHyp &= (tranType & S1CTran) == 0; - bool vm = hcr.vm; - if (HaveVirtHostExt(tc) && hcr.e2h == 1 && hcr.tge ==1) { - vm = 0; - } - - if (hcr.e2h == 1 && (aarch64EL == EL2 - || (hcr.tge ==1 && aarch64EL == EL0))) { - isHyp = true; - directToStage2 = false; - stage2Req = false; - stage2DescReq = false; - } else { - // Work out if we should skip the first stage of translation and go - // directly to stage 2. This value is cached so we don't have to - // compute it for every translation. - bool sec = !isSecure || (isSecure && IsSecureEL2Enabled(tc)); - stage2Req = isStage2 || - (vm && !isHyp && sec && - !(tranType & S1CTran) && (aarch64EL < EL2) && - !(tranType & S1E1Tran)); // <--- FIX THIS HACK - stage2DescReq = isStage2 || (vm && !isHyp && sec && - (aarch64EL < EL2)); - directToStage2 = !isStage2 && stage2Req && !sctlr.m; - } - } else { - vmid = 0; - isHyp = false; - directToStage2 = false; - stage2Req = false; - stage2DescReq = false; - } - } else { // AArch32 - sctlr = tc->readMiscReg(snsBankedIndex(MISCREG_SCTLR, tc, - !isSecure)); - ttbcr = tc->readMiscReg(snsBankedIndex(MISCREG_TTBCR, tc, - !isSecure)); - scr = tc->readMiscReg(MISCREG_SCR); - isPriv = cpsr.mode != MODE_USER; - if (longDescFormatInUse(tc)) { - uint64_t ttbr_asid = tc->readMiscReg( - snsBankedIndex(ttbcr.a1 ? MISCREG_TTBR1 : - MISCREG_TTBR0, - tc, !isSecure)); - asid = bits(ttbr_asid, 55, 48); - } else { // Short-descriptor translation table format in use - CONTEXTIDR context_id = tc->readMiscReg(snsBankedIndex( - MISCREG_CONTEXTIDR, tc,!isSecure)); - asid = context_id.asid; - } - prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, tc, - !isSecure)); - nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, tc, - !isSecure)); - dacr = tc->readMiscReg(snsBankedIndex(MISCREG_DACR, tc, - !isSecure)); - hcr = tc->readMiscReg(MISCREG_HCR); - - if (haveVirtualization) { - vmid = bits(tc->readMiscReg(MISCREG_VTTBR), 55, 48); - isHyp = cpsr.mode == MODE_HYP; - isHyp |= tranType & HypMode; - isHyp &= (tranType & S1S2NsTran) == 0; - isHyp &= (tranType & S1CTran) == 0; - if (isHyp) { - sctlr = tc->readMiscReg(MISCREG_HSCTLR); - } - // Work out if we should skip the first stage of translation and go - // directly to stage 2. This value is cached so we don't have to - // compute it for every translation. - bool sec = !isSecure || (isSecure && IsSecureEL2Enabled(tc)); - stage2Req = hcr.vm && !isStage2 && !isHyp && sec && - !(tranType & S1CTran); - stage2DescReq = hcr.vm && !isStage2 && !isHyp && sec; - directToStage2 = stage2Req && !sctlr.m; - } else { - vmid = 0; - stage2Req = false; - isHyp = false; - directToStage2 = false; - stage2DescReq = false; - } - } - miscRegValid = true; - miscRegContext = tc->contextId(); - curTranType = tranType; -} - -ExceptionLevel -TLB::tranTypeEL(CPSR cpsr, ArmTranslationType type) -{ - switch (type) { - case S1E0Tran: - case S12E0Tran: - return EL0; - - case S1E1Tran: - case S12E1Tran: - return EL1; - - case S1E2Tran: - return EL2; - - case S1E3Tran: - return EL3; - - case NormalTran: - case S1CTran: - case S1S2NsTran: - case HypMode: - return currEL(cpsr); - - default: - panic("Unknown translation mode!\n"); - } -} - -Fault -TLB::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, BaseMMU::Translation *translation, bool timing, - bool functional, bool is_secure, TLB::ArmTranslationType tranType) -{ - // In a 2-stage system, the IPA->PA translation can be started via this - // call so make sure the miscRegs are correct. - if (isStage2) { - updateMiscReg(tc, tranType); - } - - Addr vaddr_tainted = req->getVaddr(); - Addr vaddr = 0; - ExceptionLevel target_el = aarch64 ? aarch64EL : EL1; - if (aarch64) { - vaddr = purifyTaggedAddr(vaddr_tainted, tc, target_el, (TCR)ttbcr, - mode==BaseMMU::Execute); - } else { - vaddr = vaddr_tainted; - } - *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el, - false, mode); - if (*te == NULL) { - if (req->isPrefetch()) { - // if the request is a prefetch don't attempt to fill the TLB or go - // any further with the memory access (here we can safely use the - // fault status for the short desc. format in all cases) - stats.prefetchFaults++; - return std::make_shared( - vaddr_tainted, ArmFault::PrefetchTLBMiss, isStage2); - } - - // start translation table walk, pass variables rather than - // re-retreaving in table walker for speed - DPRINTF(TLB, "TLB Miss: Starting hardware table walker for %#x(%d:%d)\n", - vaddr_tainted, asid, vmid); - Fault fault; - fault = tableWalker->walk(req, tc, asid, vmid, isHyp, mode, - translation, timing, functional, is_secure, - tranType, stage2DescReq); - // for timing mode, return and wait for table walk, - if (timing || fault != NoFault) { - return fault; - } - - *te = lookup(vaddr, asid, vmid, isHyp, is_secure, true, false, - target_el, false, mode); - if (!*te) - printTlb(); - assert(*te); - } - return NoFault; -} - -Fault -TLB::getResultTe(TlbEntry **te, const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode, - BaseMMU::Translation *translation, bool timing, bool functional, - TlbEntry *mergeTe) -{ - Fault fault; - - if (isStage2) { - // We are already in the stage 2 TLB. Grab the table entry for stage - // 2 only. We are here because stage 1 translation is disabled. - TlbEntry *s2Te = NULL; - // Get the stage 2 table entry - fault = getTE(&s2Te, req, tc, mode, translation, timing, functional, - isSecure, curTranType); - // Check permissions of stage 2 - if ((s2Te != NULL) && (fault == NoFault)) { - if (aarch64) - fault = checkPermissions64(s2Te, req, mode, tc); - else - fault = checkPermissions(s2Te, req, mode); - } - *te = s2Te; - return fault; - } - - TlbEntry *s1Te = NULL; - - Addr vaddr_tainted = req->getVaddr(); - - // Get the stage 1 table entry - fault = getTE(&s1Te, req, tc, mode, translation, timing, functional, - isSecure, curTranType); - // only proceed if we have a valid table entry - if ((s1Te != NULL) && (fault == NoFault)) { - // Check stage 1 permissions before checking stage 2 - if (aarch64) - fault = checkPermissions64(s1Te, req, mode, tc); - else - fault = checkPermissions(s1Te, req, mode); - if (stage2Req & (fault == NoFault)) { - Stage2LookUp *s2Lookup = new Stage2LookUp(this, stage2Tlb, *s1Te, - req, translation, mode, timing, functional, isSecure, - curTranType); - fault = s2Lookup->getTe(tc, mergeTe); - if (s2Lookup->isComplete()) { - *te = mergeTe; - // We've finished with the lookup so delete it - delete s2Lookup; - } else { - // The lookup hasn't completed, so we can't delete it now. We - // get round this by asking the object to self delete when the - // translation is complete. - s2Lookup->setSelfDelete(); - } - } else { - // This case deals with an S1 hit (or bypass), followed by - // an S2 hit-but-perms issue - if (isStage2) { - DPRINTF(TLBVerbose, "s2TLB: reqVa %#x, reqPa %#x, fault %p\n", - vaddr_tainted, req->hasPaddr() ? req->getPaddr() : ~0, fault); - if (fault != NoFault) { - ArmFault *armFault = reinterpret_cast(fault.get()); - armFault->annotate(ArmFault::S1PTW, false); - armFault->annotate(ArmFault::OVA, vaddr_tainted); - } - } - *te = s1Te; - } - } - return fault; -} - -void -TLB::setTestInterface(SimObject *_ti) -{ - if (!_ti) { - test = nullptr; - } else { - TlbTestInterface *ti(dynamic_cast(_ti)); - fatal_if(!ti, "%s is not a valid ARM TLB tester\n", _ti->name()); - test = ti; - } -} - -Fault -TLB::testTranslation(const RequestPtr &req, BaseMMU::Mode mode, - TlbEntry::DomainType domain) -{ - if (!test || !req->hasSize() || req->getSize() == 0 || - req->isCacheMaintenance()) { - return NoFault; - } else { - return test->translationCheck(req, isPriv, mode, domain); - } -} - -Fault -TLB::testWalk(Addr pa, Addr size, Addr va, bool is_secure, BaseMMU::Mode mode, - TlbEntry::DomainType domain, LookupLevel lookup_level) -{ - if (!test) { - return NoFault; - } else { - return test->walkCheck(pa, size, va, is_secure, isPriv, mode, - domain, lookup_level); - } -} - } // namespace gem5 diff --git a/src/arch/arm/tlb.hh b/src/arch/arm/tlb.hh index 72e88037c0..1c0549d298 100644 --- a/src/arch/arm/tlb.hh +++ b/src/arch/arm/tlb.hh @@ -59,7 +59,6 @@ class ThreadContext; namespace ArmISA { class TableWalker; -class Stage2LookUp; class TLB; class TLBIALL; @@ -108,67 +107,12 @@ class TlbTestInterface class TLB : public BaseTLB { - public: - enum ArmFlags - { - AlignmentMask = 0x7, - - AlignByte = 0x0, - AlignHalfWord = 0x1, - AlignWord = 0x2, - AlignDoubleWord = 0x3, - AlignQuadWord = 0x4, - AlignOctWord = 0x5, - - AllowUnaligned = 0x8, - // Priv code operating as if it wasn't - UserMode = 0x10 - }; - - enum ArmTranslationType - { - NormalTran = 0, - S1CTran = 0x1, - HypMode = 0x2, - // Secure code operating as if it wasn't (required by some Address - // Translate operations) - S1S2NsTran = 0x4, - // Address translation instructions (eg AT S1E0R_Xt) need to be handled - // in special ways during translation because they could need to act - // like a different EL than the current EL. The following flags are - // for these instructions - S1E0Tran = 0x8, - S1E1Tran = 0x10, - S1E2Tran = 0x20, - S1E3Tran = 0x40, - S12E0Tran = 0x80, - S12E1Tran = 0x100 - }; - - /** - * Determine the EL to use for the purpose of a translation given - * a specific translation type. If the translation type doesn't - * specify an EL, we use the current EL. - */ - static ExceptionLevel tranTypeEL(CPSR cpsr, ArmTranslationType type); - protected: TlbEntry* table; // the Page Table int size; // TLB Size bool isStage2; // Indicates this TLB is part of the second stage MMU - bool stage2Req; // Indicates whether a stage 2 lookup is also required - // Indicates whether a stage 2 lookup of the table descriptors is required. - // Certain address translation instructions will intercept the IPA but the - // table descriptors still need to be translated by the stage2. - bool stage2DescReq; - uint64_t _attr; // Memory attributes for last accessed TLB entry - bool directToStage2; // Indicates whether all translation requests should - // be routed directly to the stage 2 TLB TableWalker *tableWalker; - TLB *stage2Tlb; - - TlbTestInterface *test; struct TlbStats : public statistics::Group { @@ -186,10 +130,6 @@ class TLB : public BaseTLB mutable statistics::Scalar flushTlbMvaAsid; mutable statistics::Scalar flushTlbAsid; mutable statistics::Scalar flushedEntries; - mutable statistics::Scalar alignFaults; - mutable statistics::Scalar prefetchFaults; - mutable statistics::Scalar domainFaults; - mutable statistics::Scalar permsFaults; statistics::Formula readAccesses; statistics::Formula writeAccesses; @@ -203,6 +143,7 @@ class TLB : public BaseTLB probing::PMUUPtr ppRefills; int rangeMRU; //On lookup, only move entries ahead when outside rangeMRU + vmid_t vmid; public: using Params = ArmTLBParams; @@ -231,36 +172,16 @@ class TLB : public BaseTLB void takeOverFrom(BaseTLB *otlb) override; - void setTestInterface(SimObject *ti); - - void setStage2Tlb(TLB *stage2_tlb) { stage2Tlb = stage2_tlb; } - void setTableWalker(TableWalker *table_walker); TableWalker *getTableWalker() { return tableWalker; } int getsize() const { return size; } + void setVMID(vmid_t _vmid) { vmid = _vmid; } + void insert(Addr vaddr, TlbEntry &pte); - Fault getTE(TlbEntry **te, const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode, - BaseMMU::Translation *translation, - bool timing, bool functional, - bool is_secure, ArmTranslationType tranType); - - Fault getResultTe(TlbEntry **te, const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode, - BaseMMU::Translation *translation, bool timing, - bool functional, TlbEntry *mergeTe); - - Fault checkPermissions(TlbEntry *te, const RequestPtr &req, - BaseMMU::Mode mode); - Fault checkPermissions64(TlbEntry *te, const RequestPtr &req, - BaseMMU::Mode mode, ThreadContext *tc); - bool checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, - BaseMMU::Mode mode, const bool is_priv); - /** Reset the entire TLB. Used for CPU switching to prevent stale * translations after multiple switches */ @@ -314,87 +235,27 @@ class TLB : public BaseTLB panic("demapPage() is not implemented.\n"); } - /** - * Do a functional lookup on the TLB (for debugging) - * and don't modify any internal state - * @param tc thread context to get the context id from - * @param vaddr virtual address to translate - * @param pa returned physical address - * @return if the translation was successful - */ - bool translateFunctional(ThreadContext *tc, Addr vaddr, Addr &paddr); - - /** - * Do a functional lookup on the TLB (for checker cpu) that - * behaves like a normal lookup without modifying any page table state. - */ - Fault translateFunctional(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, ArmTranslationType tranType); Fault - translateFunctional(const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode) override + translateAtomic(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Mode mode) override { - return translateFunctional(req, tc, mode, NormalTran); + panic("unimplemented"); } - /** Accessor functions for memory attributes for last accessed TLB entry - */ - void - setAttr(uint64_t attr) - { - _attr = attr; - } - - uint64_t - getAttr() const - { - return _attr; - } - - Fault translateMmuOff(ThreadContext *tc, const RequestPtr &req, - BaseMMU::Mode mode, TLB::ArmTranslationType tranType, - Addr vaddr, bool long_desc_format); - Fault translateMmuOn(ThreadContext *tc, const RequestPtr &req, - BaseMMU::Mode mode, BaseMMU::Translation *translation, bool &delay, - bool timing, bool functional, - Addr vaddr, ArmFault::TranMethod tranMethod); - - Fault translateFs(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, BaseMMU::Translation *translation, - bool &delay, bool timing, ArmTranslationType tranType, - bool functional = false); - Fault translateSe(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, BaseMMU::Translation *translation, - bool &delay, bool timing); - - Fault translateAtomic(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Mode mode, ArmTranslationType tranType); - - Fault - translateAtomic(const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode) override - { - return translateAtomic(req, tc, mode, NormalTran); - } - void translateTiming( - const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, BaseMMU::Mode mode, - ArmTranslationType tranType); void translateTiming(const RequestPtr &req, ThreadContext *tc, BaseMMU::Translation *translation, BaseMMU::Mode mode) override { - translateTiming(req, tc, translation, mode, NormalTran); + panic("unimplemented"); } - Fault translateComplete(const RequestPtr &req, ThreadContext *tc, - BaseMMU::Translation *translation, BaseMMU::Mode mode, - ArmTranslationType tranType, bool callFromS2); - Fault finalizePhysical( - const RequestPtr &req, - ThreadContext *tc, BaseMMU::Mode mode) const override; - void drainResume() override; + Fault + finalizePhysical(const RequestPtr &req, ThreadContext *tc, + BaseMMU::Mode mode) const override + { + panic("unimplemented"); + } void regProbePoints() override; @@ -414,45 +275,8 @@ class TLB : public BaseTLB // Writing to misc registers needs to invalidate them. // translateFunctional/translateSe/translateFs checks if they are // invalid and call updateMiscReg if necessary. -protected: - CPSR cpsr; - bool aarch64; - ExceptionLevel aarch64EL; - SCTLR sctlr; - SCR scr; - bool isPriv; - bool isSecure; - bool isHyp; - TTBCR ttbcr; - uint16_t asid; - vmid_t vmid; - PRRR prrr; - NMRR nmrr; - HCR hcr; - uint32_t dacr; - bool miscRegValid; - ContextID miscRegContext; - ArmTranslationType curTranType; - // Cached copies of system-level properties - bool haveLPAE; - bool haveVirtualization; - bool haveLargeAsid64; - uint8_t physAddrRange; - - AddrRange m5opRange; - - void updateMiscReg(ThreadContext *tc, - ArmTranslationType tranType = NormalTran); - - /** Returns the current VMID - * (information stored in the VTTBR_EL2 register) */ - vmid_t getVMID(ThreadContext *tc) const; - -public: - void invalidateMiscReg() { miscRegValid = false; } - -private: + private: /** Remove any entries that match both a va and asn * @param mva virtual address to flush * @param asn contextid/asn to flush on match @@ -463,14 +287,6 @@ private: void _flushMva(Addr mva, uint64_t asn, bool secure_lookup, bool ignore_asn, ExceptionLevel target_el, bool in_host); - - public: /* Testing */ - Fault testTranslation(const RequestPtr &req, BaseMMU::Mode mode, - TlbEntry::DomainType domain); - - Fault testWalk(Addr pa, Addr size, Addr va, bool is_secure, - BaseMMU::Mode mode, TlbEntry::DomainType domain, - LookupLevel lookup_level); }; } // namespace ArmISA diff --git a/src/arch/arm/tracers/tarmac_parser.cc b/src/arch/arm/tracers/tarmac_parser.cc index 473f9b7df5..e9553142be 100644 --- a/src/arch/arm/tracers/tarmac_parser.cc +++ b/src/arch/arm/tracers/tarmac_parser.cc @@ -985,7 +985,7 @@ TarmacParserRecord::dump() std::ostream &outs = Trace::output(); uint64_t written_data = 0; - unsigned mem_flags = 3 | ArmISA::TLB::AllowUnaligned; + unsigned mem_flags = 3 | ArmISA::MMU::AllowUnaligned; ISetState isetstate;