diff --git a/src/dev/arm/RealView.py b/src/dev/arm/RealView.py index a0540ec50c..bff79c15df 100644 --- a/src/dev/arm/RealView.py +++ b/src/dev/arm/RealView.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009-2022 Arm Limited +# Copyright (c) 2009-2022, 2024 Arm Limited # All rights reserved. # # The license below extends only to copyright in the software and shall @@ -1283,6 +1283,7 @@ class VExpress_GEM5_Base(RealView): 95 : HDLCD 96- 98: GPU (reserved) 100-103: PCI + 106 : SMMU event queue 130 : System Watchdog (SP805) 256-319: MSI frame 0 (gem5-specific, SPIs) 320-511: Unused @@ -1508,7 +1509,10 @@ class VExpress_GEM5_Base(RealView): if hasattr(self, "smmu"): m5.fatal("A SMMU has already been instantiated\n") - self.smmu = SMMUv3(reg_map=AddrRange(0x2B400000, size=0x00020000)) + self.smmu = SMMUv3( + reg_map=AddrRange(0x2B400000, size=0x00020000), + eventq_irq=ArmSPI(num=106), + ) self.smmu.request = bus.cpu_side_ports self.smmu.control = bus.mem_side_ports diff --git a/src/dev/arm/SConscript b/src/dev/arm/SConscript index 0a68e480f3..f516df842a 100644 --- a/src/dev/arm/SConscript +++ b/src/dev/arm/SConscript @@ -87,6 +87,7 @@ Source('kmi.cc', tags='arm isa') Source('smmu_v3.cc', tags='arm isa'); Source('smmu_v3_caches.cc', tags='arm isa'); Source('smmu_v3_cmdexec.cc', tags='arm isa'); +Source('smmu_v3_defs.cc', tags='arm isa'); Source('smmu_v3_events.cc', tags='arm isa'); Source('smmu_v3_ports.cc', tags='arm isa'); Source('smmu_v3_proc.cc', tags='arm isa'); diff --git a/src/dev/arm/SMMUv3.py b/src/dev/arm/SMMUv3.py index 28c2c6fe24..ec29700814 100644 --- a/src/dev/arm/SMMUv3.py +++ b/src/dev/arm/SMMUv3.py @@ -1,4 +1,4 @@ -# Copyright (c) 2013, 2018-2020 ARM Limited +# Copyright (c) 2013, 2018-2020, 2024 Arm Limited # All rights reserved # # The license below extends only to copyright in the software and shall @@ -100,13 +100,6 @@ class SMMUv3(ClockedObject): reg_map = Param.AddrRange("Address range for control registers") system = Param.System(Parent.any, "System this device is part of") - irq_interface_enable = Param.Bool( - False, - "This flag enables software to program SMMU_IRQ_CTRL and " - "SMMU_IRQ_CTRLACK as if the model implemented architectural " - "interrupt sources", - ) - device_interfaces = VectorParam.SMMUv3DeviceInterface( [], "Responder interfaces" ) @@ -200,6 +193,13 @@ class SMMUv3(ClockedObject): # [7:0] (0 = SMMUv3.0) (1 = SMMUv3.1) smmu_aidr = Param.UInt32(0, "SMMU_AIDR register") + eventq_irq = Param.ArmSPI( + NULL, + "Event Queue Interrupt. If set to NULL it means a wired " + "implementation of the interrupt is not supported. " + "In that case MSIs should be used", + ) + def generateDeviceTree(self, state): reg_addr = self.reg_map.start reg_size = self.reg_map.size() @@ -210,6 +210,22 @@ class SMMUv3(ClockedObject): "reg", state.addrCells(reg_addr) + state.sizeCells(reg_size) ) ) + + gic = self._parent.unproxy(self).gic + + wired_interrupts = [] + if self.eventq_irq != NULL: + wired_interrupts += self.eventq_irq.generateFdtProperty(gic) + + if wired_interrupts: + node.append(FdtPropertyWords("interrupts", wired_interrupts)) + node.append( + FdtPropertyWords( + "interrupt-names", + ["eventq"], + ) + ) + node.append(FdtPropertyWords("#iommu-cells", [1])) node.appendPhandle(self) diff --git a/src/dev/arm/smmu_v3.cc b/src/dev/arm/smmu_v3.cc index 8ce8bd92b2..4d06a8cd79 100644 --- a/src/dev/arm/smmu_v3.cc +++ b/src/dev/arm/smmu_v3.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018-2020 ARM Limited + * Copyright (c) 2013, 2018-2020, 2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -48,6 +48,7 @@ #include "base/types.hh" #include "debug/Checkpoint.hh" #include "debug/SMMUv3.hh" +#include "dev/arm/base_gic.hh" #include "dev/arm/smmu_v3_transl.hh" #include "mem/packet_access.hh" #include "sim/system.hh" @@ -62,7 +63,7 @@ SMMUv3::SMMUv3(const SMMUv3Params ¶ms) : requestPort(name() + ".request", *this), tableWalkPort(name() + ".walker", *this), controlPort(name() + ".control", *this, params.reg_map), - irqInterfaceEnable(params.irq_interface_enable), + eventqInterrupt(params.eventq_irq ? params.eventq_irq->get() : nullptr), tlb(params.tlb_entries, params.tlb_assoc, params.tlb_policy, this), configCache(params.cfg_entries, params.cfg_assoc, params.cfg_policy, this), ipaCache(params.ipa_entries, params.ipa_assoc, params.ipa_policy, this), @@ -618,10 +619,9 @@ SMMUv3::writeControl(PacketPtr pkt) break; case offsetof(SMMURegs, irq_ctrl): assert(pkt->getSize() == sizeof(uint32_t)); - if (irqInterfaceEnable) { - warn("SMMUv3::%s No support for interrupt sources", __func__); - regs.irq_ctrl = regs.irq_ctrlack = pkt->getLE(); - } + warn("SMMUv3::%s No support for GERROR and PRI interrupt sources", + __func__); + regs.irq_ctrl = regs.irq_ctrlack = pkt->getLE(); break; case offsetof(SMMURegs, cr1): diff --git a/src/dev/arm/smmu_v3.hh b/src/dev/arm/smmu_v3.hh index 8721352c47..be8525f19d 100644 --- a/src/dev/arm/smmu_v3.hh +++ b/src/dev/arm/smmu_v3.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018-2020 ARM Limited + * Copyright (c) 2013, 2018-2020, 2024 ARM Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -79,6 +79,8 @@ namespace gem5 { +class ArmInterruptPin; + class SMMUTranslationProcess; class SMMUv3 : public ClockedObject @@ -97,7 +99,9 @@ class SMMUv3 : public ClockedObject SMMUTableWalkPort tableWalkPort; SMMUControlPort controlPort; - const bool irqInterfaceEnable; + // This could be nullptr if wired implementation of the + // event queue interrupt is not supported + ArmInterruptPin * const eventqInterrupt; ARMArchTLB tlb; ConfigCache configCache; diff --git a/src/dev/arm/smmu_v3_defs.cc b/src/dev/arm/smmu_v3_defs.cc new file mode 100644 index 0000000000..cdf7546824 --- /dev/null +++ b/src/dev/arm/smmu_v3_defs.cc @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024 Arm Limited + * All rights reserved + * + * The license below extends only to copyright in the software and shall + * not be construed as granting a license to any other intellectual + * property including but not limited to intellectual property relating + * to a hardware implementation of the functionality of the software + * licensed hereunder. You may use the software subject to the license + * terms below provided that you ensure that this notice is replicated + * unmodified and in its entirety in all distributions of the software, + * modified or unmodified, in source code or in binary form. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer; + * redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution; + * neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "dev/arm/smmu_v3_defs.hh" + +namespace gem5 +{ + +std::string +SMMUEvent::print() const +{ + return csprintf("type=%#x sid=%#x ssid=%#x va=%#08x\n", + data.dw0.eventType, data.dw0.streamId, data.dw0.substreamId, + data.dw2.inputAddr); +} + +} // namespace gem5 diff --git a/src/dev/arm/smmu_v3_defs.hh b/src/dev/arm/smmu_v3_defs.hh index 8b7dfe02cf..c20343d9e0 100644 --- a/src/dev/arm/smmu_v3_defs.hh +++ b/src/dev/arm/smmu_v3_defs.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018-2019 ARM Limited + * Copyright (c) 2013, 2018-2019, 2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -102,10 +102,42 @@ enum Q_BASE_ADDR_MASK = 0x0000ffffffffffe0ULL, Q_BASE_SIZE_MASK = 0x000000000000001fULL, - E_BASE_ENABLE_MASK = 0x8000000000000000ULL, E_BASE_ADDR_MASK = 0x0000fffffffffffcULL, }; +BitUnion32(IDR0) + Bitfield<0> s2p; + Bitfield<1> s1p; + Bitfield<3, 2> ttf; + Bitfield<4> cohacc; + Bitfield<5> btm; + Bitfield<7, 6> httu; + Bitfield<8> dormhint; + Bitfield<9> hyp; + Bitfield<10> ats; + Bitfield<11> ns1ats; + Bitfield<12> asid16; + Bitfield<13> msi; + Bitfield<14> sev; + Bitfield<15> atos; + Bitfield<16> pri; + Bitfield<17> vmw; + Bitfield<18> vmid16; + Bitfield<19> cd2l; + Bitfield<20> vatos; + Bitfield<22, 21> ttEndian; + Bitfield<23> atsRecErr; + Bitfield<25, 24> stallModel; + Bitfield<26> termModel; + Bitfield<28, 27> stLevel; +EndBitUnion(IDR0) + +BitUnion32(IRQCtrl) + Bitfield<0> gerrorIrqEn; + Bitfield<1> priqIrqEn; + Bitfield<2> eventqIrqEn; +EndBitUnion(IRQCtrl) + union SMMURegs { uint8_t data[SMMU_REG_SIZE]; @@ -384,25 +416,41 @@ struct SMMUCommand } }; -enum SMMUEventTypes -{ - EVT_FAULT = 0x0001, -}; - -enum SMMUEventFlags -{ - EVF_WRITE = 0x0001, -}; - struct SMMUEvent { - uint16_t type; - uint16_t stag; - uint32_t flags; - uint32_t streamId; - uint32_t substreamId; - uint64_t va; - uint64_t ipa; + struct Data { + BitUnion64(DWORD0) + Bitfield<7, 0> eventType; + Bitfield<11> ssv; + Bitfield<31, 12> substreamId; + Bitfield<63, 32> streamId; + EndBitUnion(DWORD0) + DWORD0 dw0; + + BitUnion64(DWORD1) + Bitfield<16, 0> stag; + Bitfield<33> pnu; + Bitfield<34> ind; + Bitfield<35> rnw; + Bitfield<38> nsipa; + Bitfield<39> s2; + Bitfield<41, 40> clss; + EndBitUnion(DWORD1) + DWORD1 dw1; + + BitUnion64(DWORD2) + Bitfield<63, 0> inputAddr; + EndBitUnion(DWORD2) + DWORD2 dw2; + + BitUnion64(DWORD3) + Bitfield<51, 3> fetchAddr; + Bitfield<51, 12> ipa; + EndBitUnion(DWORD3) + DWORD3 dw3; + } data; + + std::string print() const; }; enum diff --git a/src/dev/arm/smmu_v3_transl.cc b/src/dev/arm/smmu_v3_transl.cc index 85554f8bd0..2d519375b0 100644 --- a/src/dev/arm/smmu_v3_transl.cc +++ b/src/dev/arm/smmu_v3_transl.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018-2019, 2021 Arm Limited + * Copyright (c) 2013, 2018-2019, 2021, 2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -41,6 +41,7 @@ #include "debug/SMMUv3.hh" #include "debug/SMMUv3Hazard.hh" #include "dev/arm/amba.hh" +#include "dev/arm/base_gic.hh" #include "dev/arm/smmu_v3.hh" #include "sim/system.hh" @@ -184,7 +185,7 @@ SMMUTranslationProcess::main(Yield &yield) tr = smmuTranslation(yield); - if (tr.fault == FAULT_NONE) + if (!tr.isFaulting()) ifcTLBUpdate(yield, tr); hazard4kRelease(); @@ -213,7 +214,7 @@ SMMUTranslationProcess::main(Yield &yield) tr = smmuTranslation(yield); - if (tr.fault == FAULT_NONE) { + if (!tr.isFaulting()) { ifcTLBUpdate(yield, tr); issuePrefetch(next4k); @@ -222,20 +223,18 @@ SMMUTranslationProcess::main(Yield &yield) hazard4kRelease(); } - if (tr.fault == FAULT_NONE) + if (!tr.isFaulting()) microTLBUpdate(yield, tr); } hazardIdHold(yield); hazardIdRelease(); - if (tr.fault != FAULT_NONE) - panic("Translation Fault (addr=%#x, size=%#x, sid=%d, ssid=%d, " - "isWrite=%d, isPrefetch=%d, isAtsRequest=%d)\n", - request.addr, request.size, request.sid, request.ssid, - request.isWrite, request.isPrefetch, request.isAtsRequest); - - completeTransaction(yield, tr); + if (tr.isFaulting()) { + abortTransaction(yield, tr); + } else { + completeTransaction(yield, tr); + } } } @@ -243,7 +242,7 @@ SMMUTranslationProcess::TranslResult SMMUTranslationProcess::bypass(Addr addr) const { TranslResult tr; - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = addr; tr.addrMask = 0; tr.writable = 1; @@ -297,7 +296,7 @@ SMMUTranslationProcess::smmuTranslation(Yield &yield) // Free PTW slot doSemaphoreUp(smmu.ptwSem); - if (tr.fault == FAULT_NONE) + if (!tr.isFaulting()) smmuTLBUpdate(yield, tr); } @@ -336,8 +335,8 @@ SMMUTranslationProcess::microTLBLookup(Yield &yield, TranslResult &tr) "micro TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x paddr=%#x\n", request.addr, e->vaMask, request.sid, request.ssid, e->pa); - tr.fault = FAULT_NONE; - tr.addr = e->pa + (request.addr & ~e->vaMask);; + tr.fault = Fault(FAULT_NONE); + tr.addr = e->pa + (request.addr & ~e->vaMask);; tr.addrMask = e->vaMask; tr.writable = e->permissions; @@ -370,7 +369,7 @@ SMMUTranslationProcess::ifcTLBLookup(Yield &yield, TranslResult &tr, "paddr=%#x\n", request.addr, e->vaMask, request.sid, request.ssid, e->pa); - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = e->pa + (request.addr & ~e->vaMask);; tr.addrMask = e->vaMask; tr.writable = e->permissions; @@ -402,7 +401,7 @@ SMMUTranslationProcess::smmuTLBLookup(Yield &yield, TranslResult &tr) "SMMU TLB hit vaddr=%#x amask=%#x asid=%#x vmid=%#x paddr=%#x\n", request.addr, e->vaMask, context.asid, context.vmid, e->pa); - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = e->pa + (request.addr & ~e->vaMask);; tr.addrMask = e->vaMask; tr.writable = e->permissions; @@ -414,7 +413,7 @@ void SMMUTranslationProcess::microTLBUpdate(Yield &yield, const TranslResult &tr) { - assert(tr.fault == FAULT_NONE); + assert(!tr.isFaulting()); if (!ifc.microTLBEnable) return; @@ -446,7 +445,7 @@ void SMMUTranslationProcess::ifcTLBUpdate(Yield &yield, const TranslResult &tr) { - assert(tr.fault == FAULT_NONE); + assert(!tr.isFaulting()); if (!ifc.mainTLBEnable) return; @@ -483,7 +482,7 @@ void SMMUTranslationProcess::smmuTLBUpdate(Yield &yield, const TranslResult &tr) { - assert(tr.fault == FAULT_NONE); + assert(!tr.isFaulting()); if (!smmu.tlbEnable) return; @@ -632,7 +631,7 @@ SMMUTranslationProcess::findConfig(Yield &yield, // Now fetch stage 1 config. if (context.stage1Enable) { ContextDescriptor cd; - doReadCD(yield, cd, ste, request.sid, request.ssid); + tr = doReadCD(yield, cd, ste, request.sid, request.ssid); tc.ttb0 = cd.dw1.ttb0 << CD_TTB_SHIFT; tc.ttb1 = cd.dw2.ttb1 << CD_TTB_SHIFT; @@ -647,7 +646,7 @@ SMMUTranslationProcess::findConfig(Yield &yield, tc.t0sz = 0; } - return true; + return !tr.isFaulting(); } void @@ -767,7 +766,7 @@ SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, DPRINTF(SMMUv3, "S1 PTE not valid - fault\n"); TranslResult tr; - tr.fault = FAULT_TRANSLATION; + tr.fault = Fault(FAULT_TRANSLATION, FaultClass::IN, false); return tr; } @@ -777,7 +776,7 @@ SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, DPRINTF(SMMUv3, "S1 page not writable - fault\n"); TranslResult tr; - tr.fault = FAULT_PERMISSION; + tr.fault = Fault(FAULT_PERMISSION, FaultClass::IN, false); return tr; } @@ -788,7 +787,7 @@ SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, if (context.stage2Enable) { TranslResult s2tr = translateStage2(yield, walkPtr, false); - if (s2tr.fault != FAULT_NONE) + if (s2tr.isFaulting()) return s2tr; walkPtr = s2tr.addr; @@ -799,15 +798,17 @@ SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, } TranslResult tr; - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addrMask = pt_ops->pageMask(pte, level); tr.addr = walkPtr + (addr & ~tr.addrMask); tr.writable = pt_ops->isWritable(pte, level, false); if (context.stage2Enable) { TranslResult s2tr = translateStage2(yield, tr.addr, true); - if (s2tr.fault != FAULT_NONE) + if (s2tr.isFaulting()) { + s2tr.fault.clss = FaultClass::IN; return s2tr; + } tr = combineTranslations(tr, s2tr); } @@ -852,7 +853,7 @@ SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, DPRINTF(SMMUv3, " S2 PTE not valid - fault\n"); TranslResult tr; - tr.fault = FAULT_TRANSLATION; + tr.fault = Fault(FAULT_TRANSLATION, FaultClass::TT, true, addr); return tr; } @@ -862,7 +863,7 @@ SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, DPRINTF(SMMUv3, " S2 PTE not writable = fault\n"); TranslResult tr; - tr.fault = FAULT_PERMISSION; + tr.fault = Fault(FAULT_PERMISSION, FaultClass::TT, true, addr); return tr; } @@ -877,7 +878,7 @@ SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, } TranslResult tr; - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addrMask = pt_ops->pageMask(pte, level); tr.addr = walkPtr + (addr & ~tr.addrMask); tr.writable = pt_ops->isWritable(pte, level, true); @@ -913,7 +914,7 @@ SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) TranslResult tr; if (walk_ep) { if (walk_ep->leaf) { - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); tr.addrMask = walk_ep->vaMask; tr.writable = walk_ep->permissions; @@ -924,8 +925,9 @@ SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) Addr table_addr = context.ttb0; if (context.stage2Enable) { TranslResult s2tr = translateStage2(yield, table_addr, false); - if (s2tr.fault != FAULT_NONE) + if (s2tr.isFaulting()) { return s2tr; + } table_addr = s2tr.addr; } @@ -935,7 +937,7 @@ SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) table_addr); } - if (tr.fault == FAULT_NONE) + if (!tr.isFaulting()) DPRINTF(SMMUv3, "Translated vaddr %#x to paddr %#x\n", addr, tr.addr); return tr; @@ -957,7 +959,7 @@ SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) if (ipa_ep) { TranslResult tr; - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = ipa_ep->pa + (addr & ~ipa_ep->ipaMask); tr.addrMask = ipa_ep->ipaMask; tr.writable = ipa_ep->permissions; @@ -995,7 +997,7 @@ SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) TranslResult tr; if (walk_ep) { if (walk_ep->leaf) { - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); tr.addrMask = walk_ep->vaMask; tr.writable = walk_ep->permissions; @@ -1009,7 +1011,7 @@ SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) context.httb); } - if (tr.fault == FAULT_NONE) + if (!tr.isFaulting()) DPRINTF(SMMUv3, " Translated %saddr %#x to paddr %#x\n", context.stage1Enable ? "ip" : "v", addr, tr.addr); @@ -1034,13 +1036,13 @@ SMMUTranslationProcess::TranslResult SMMUTranslationProcess::combineTranslations(const TranslResult &s1tr, const TranslResult &s2tr) const { - if (s2tr.fault != FAULT_NONE) + if (s2tr.isFaulting()) return s2tr; - assert(s1tr.fault == FAULT_NONE); + assert(!s1tr.isFaulting()); TranslResult tr; - tr.fault = FAULT_NONE; + tr.fault = Fault(FAULT_NONE); tr.addr = s2tr.addr; tr.addrMask = s1tr.addrMask | s2tr.addrMask; tr.writable = s1tr.writable & s2tr.writable; @@ -1229,11 +1231,51 @@ SMMUTranslationProcess::issuePrefetch(Addr addr) proc->scheduleWakeup(smmu.clockEdge(Cycles(1))); } +void +SMMUTranslationProcess::abortTransaction(Yield &yield, + const TranslResult &tr) +{ + DPRINTF(SMMUv3, "Translation Fault (addr=%#x, size=%#x, sid=%d, ssid=%d, " + "isWrite=%d, isPrefetch=%d, isAtsRequest=%d)\n", + request.addr, request.size, request.sid, request.ssid, + request.isWrite, request.isPrefetch, request.isAtsRequest); + + // If eventq is not enabled, silently discard event + // TODO: Handle full queue (we are currently aborting + // in send event) + if (smmu.regs.cr0 & CR0_EVENTQEN_MASK) { + SMMUEvent event = generateEvent(tr); + + sendEvent(yield, event); + } + + ifc.xlateSlotsRemaining++; + smmu.scheduleDeviceRetries(); + + if (smmu.system.isAtomicMode()) { + request.pkt->makeAtomicResponse(); + } else if (smmu.system.isTimingMode()) { + request.pkt->makeTimingResponse(); + } else { + panic("Not in atomic or timing mode"); + } + + request.pkt->setBadAddress(); + + SMMUAction a; + // Send the bad address response to the client device + a.type = ACTION_SEND_RESP; + a.pkt = request.pkt; + a.ifc = &ifc; + a.delay = 0; + yield(a); +} + void SMMUTranslationProcess::completeTransaction(Yield &yield, const TranslResult &tr) { - assert(tr.fault == FAULT_NONE); + assert(!tr.isFaulting()); unsigned numRequestorBeats = request.isWrite ? (request.size + (smmu.requestPortWidth-1)) @@ -1304,6 +1346,32 @@ SMMUTranslationProcess::completePrefetch(Yield &yield) yield(a); } +SMMUEvent +SMMUTranslationProcess::generateEvent(const TranslResult &tr) +{ + SMMUEvent event; + switch (tr.fault.type) { + case FAULT_PERMISSION: + case FAULT_TRANSLATION: + event.data.dw0.streamId = request.sid; + event.data.dw0.substreamId = request.ssid; + event.data.dw1.rnw = !request.isWrite; + event.data.dw2.inputAddr = request.addr; + event.data.dw1.s2 = tr.fault.stage2; + if (tr.fault.stage2) { + // Only support non-secure mode in the SMMU + event.data.dw1.nsipa = true; + event.data.dw3.ipa = tr.fault.ipa; + } + event.data.dw1.clss = tr.fault.clss; + break; + default: + panic("Unsupported fault: %d\n", tr.fault.type); + } + + return event; +} + void SMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev) { @@ -1315,23 +1383,45 @@ SMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev) Addr event_addr = (smmu.regs.eventq_base & Q_BASE_ADDR_MASK) + - (smmu.regs.eventq_prod & sizeMask) * sizeof(ev); + (smmu.regs.eventq_prod & sizeMask) * sizeof(ev.data); - DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): type=%#x stag=%#x " - "flags=%#x sid=%#x ssid=%#x va=%#08x ipa=%#x\n", - event_addr, smmu.regs.eventq_prod, ev.type, ev.stag, - ev.flags, ev.streamId, ev.substreamId, ev.va, ev.ipa); + DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): %s\n", + event_addr, smmu.regs.eventq_prod, ev.print()); + + bool empty_queue = (smmu.regs.eventq_prod & sizeMask) == + (smmu.regs.eventq_cons & sizeMask); // This deliberately resets the overflow field in eventq_prod! smmu.regs.eventq_prod = (smmu.regs.eventq_prod + 1) & sizeMask; - doWrite(yield, event_addr, &ev, sizeof(ev)); + doWrite(yield, event_addr, &ev.data, sizeof(ev.data)); - if (!(smmu.regs.eventq_irq_cfg0 & E_BASE_ENABLE_MASK)) - panic("eventq msi not enabled\n"); + // Send an event queue interrupt when transitioning from empty to + // non empty queue + if (IRQCtrl irq_ctrl = smmu.regs.irq_ctrl; + irq_ctrl.eventqIrqEn && empty_queue) { - doWrite(yield, smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK, - &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1)); + sendEventInterrupt(yield); + } +} + +void +SMMUTranslationProcess::sendEventInterrupt(Yield &yield) +{ + Addr msi_addr = smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK; + + // Check if MSIs are enabled by inspecting the SMMU_IDR.MSI bit + // According to the SMMUv3 spec, using an address equal to 0 + // disables the sending of the MSI + if (IDR0 idr0 = smmu.regs.idr0; idr0.msi && msi_addr != 0) { + DPRINTF(SMMUv3, "Raise Event queue MSI\n"); + doWrite(yield, msi_addr, + &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1)); + } + if (smmu.eventqInterrupt) { + DPRINTF(SMMUv3, "Raise Event queue wired interrupt\n"); + smmu.eventqInterrupt->raise(); + } } void @@ -1402,12 +1492,13 @@ SMMUTranslationProcess::doReadSTE(Yield &yield, smmu.stats.steFetches++; } -void +SMMUTranslationProcess::TranslResult SMMUTranslationProcess::doReadCD(Yield &yield, ContextDescriptor &cd, const StreamTableEntry &ste, uint32_t sid, uint32_t ssid) { + TranslResult tr; Addr cd_addr = 0; if (ste.dw0.s1cdmax == 0) { @@ -1426,8 +1517,15 @@ SMMUTranslationProcess::doReadCD(Yield &yield, uint64_t l2_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + bits(ssid, 24, split) * sizeof(l2_ptr); - if (context.stage2Enable) - l2_addr = translateStage2(yield, l2_addr, false).addr; + if (context.stage2Enable) { + tr = translateStage2(yield, l2_addr, false); + if (tr.isFaulting()) { + tr.fault.clss = FaultClass::CD; + return tr; + } + + l2_addr = tr.addr; + } DPRINTF(SMMUv3, "Read L1CD at %#x\n", l2_addr); @@ -1443,8 +1541,15 @@ SMMUTranslationProcess::doReadCD(Yield &yield, } } - if (context.stage2Enable) - cd_addr = translateStage2(yield, cd_addr, false).addr; + if (context.stage2Enable) { + tr = translateStage2(yield, cd_addr, false); + if (tr.isFaulting()) { + tr.fault.clss = FaultClass::CD; + return tr; + } + + cd_addr = tr.addr; + } DPRINTF(SMMUv3, "Read CD at %#x\n", cd_addr); @@ -1464,6 +1569,7 @@ SMMUTranslationProcess::doReadCD(Yield &yield, panic("CD @ %#x not valid\n", cd_addr); smmu.stats.cdFetches++; + return tr; } void diff --git a/src/dev/arm/smmu_v3_transl.hh b/src/dev/arm/smmu_v3_transl.hh index 156d3e6fcc..93877a54b1 100644 --- a/src/dev/arm/smmu_v3_transl.hh +++ b/src/dev/arm/smmu_v3_transl.hh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2018-2019 ARM Limited + * Copyright (c) 2013, 2018-2019, 2024 Arm Limited * All rights reserved * * The license below extends only to copyright in the software and shall @@ -83,19 +83,73 @@ class SMMUTranslationProcess : public SMMUProcess uint8_t s2t0sz; }; - enum FaultType + enum FaultType : uint8_t { FAULT_NONE, - FAULT_TRANSLATION, // F_TRANSLATION - FAULT_PERMISSION, // F_PERMISSION + FAULT_UUT = 0x1, // F_UUT = Unsupported Upstream Transaction + FAULT_BAD_STREAMID = 0x2, // C_BAD_STREAMID = Transaction streamID out of range + FAULT_STE_FETCH = 0x3, // F_STE_FETCH = Fetch of STE caused external abort + FAULT_BAD_STE = 0x4, // C_BAD_STE = Invalid STE + FAULT_BAD_ATS_TREQ = 0x5, // F_BAD_ATS_TREQ + FAULT_STREAM_DISABLED = 0x6, // F_STREAM_DISABLED = Non-substream trans disabled + FAULT_TRANSL_FORBIDDEN = 0x7, // F_TRANSL_FORBIDDEN = SMMU bypass not allowed + FAULT_BAD_SUBSTREAMID = 0x8, // F_BAD_SUBSTREAMID = Bad substreamID + FAULT_CD_FETCH = 0x9, // F_CD_FETCH = Fetch of CD caused external abort + FAULT_BAD_CD = 0xa, // C_BAD_CD = Invalid CD + FAULT_WALK_EABT = 0xb, // F_WALK_EABT = Table walk/update caused external abort + FAULT_TRANSLATION = 0x10, // F_TRANSLATION = Translation Fault + FAULT_ADDR_SIZE = 0x11, // F_ADDR_SIZE = Address Size fault + FAULT_ACCESS = 0x12, // F_ACCESS = Access flag fault + FAULT_PERMISSION = 0x13, // F_PERMISSION = Permission fault + FAULT_TLB_CONFLICT = 0x20, // F_TLB_CONFLICT = TLB conflict + FAULT_CFG_CONFLICT = 0x21, // F_CFG_CONFLICT = Config cache conflict + FAULT_PAGE_REQUEST = 0x24, // E_PAGE_REQUEST + FAULT_VMS_FETCH = 0x25, // F_VMS_FETCH + }; + + /* The class of the operation that caused the fault */ + enum FaultClass + { + CD = 0x0, // CD fetch + TT = 0x1, // Stage1 translation table fetch + IN = 0x2, // Input address caused fault + RESERVED = 0x3 + }; + + struct Fault + { + explicit Fault(FaultType _type, + FaultClass _clss=FaultClass::RESERVED, + bool _stage2=false, Addr _ipa=0) + : type(_type), clss(_clss), stage2(_stage2), ipa(_ipa) + {} + + Fault(const Fault &rhs) = default; + Fault& operator=(const Fault &rhs) = default; + + bool isFaulting() const { return type != FAULT_NONE; } + + FaultType type; + FaultClass clss; + bool stage2; + Addr ipa; }; struct TranslResult { - FaultType fault; - Addr addr; - Addr addrMask; - bool writable; + TranslResult() + : fault(FaultType::FAULT_NONE), + addr(0), addrMask(0), writable(false) + {} + + TranslResult& operator=(const TranslResult &rhs) = default; + + bool isFaulting() const { return fault.isFaulting(); } + + Fault fault; + Addr addr; + Addr addrMask; + bool writable; }; SMMUv3DeviceInterface &ifc; @@ -166,14 +220,17 @@ class SMMUTranslationProcess : public SMMUProcess void issuePrefetch(Addr addr); + void abortTransaction(Yield &yield, const TranslResult &tr); void completeTransaction(Yield &yield, const TranslResult &tr); void completePrefetch(Yield &yield); + SMMUEvent generateEvent(const TranslResult &tr); void sendEvent(Yield &yield, const SMMUEvent &ev); + void sendEventInterrupt(Yield &yield); void doReadSTE(Yield &yield, StreamTableEntry &ste, uint32_t sid); - void doReadCD(Yield &yield, ContextDescriptor &cd, - const StreamTableEntry &ste, uint32_t sid, uint32_t ssid); + TranslResult doReadCD(Yield &yield, ContextDescriptor &cd, + const StreamTableEntry &ste, uint32_t sid, uint32_t ssid); void doReadConfig(Yield &yield, Addr addr, void *ptr, size_t size, uint32_t sid, uint32_t ssid); void doReadPTE(Yield &yield, Addr va, Addr addr, void *ptr,