diff --git a/src/arch/x86/pagetable_walker.cc b/src/arch/x86/pagetable_walker.cc index cf82727a97..7c386dd02f 100644 --- a/src/arch/x86/pagetable_walker.cc +++ b/src/arch/x86/pagetable_walker.cc @@ -40,6 +40,7 @@ #include "arch/x86/pagetable.hh" #include "arch/x86/pagetable_walker.hh" #include "arch/x86/tlb.hh" +#include "arch/x86/vtophys.hh" #include "base/bitfield.hh" #include "cpu/thread_context.hh" #include "cpu/base.hh" @@ -66,329 +67,37 @@ BitUnion64(PageTableEntry) Bitfield<0> p; EndBitUnion(PageTableEntry) -Fault -Walker::doNext(PacketPtr &write) -{ - assert(state != Ready && state != Waiting); - write = NULL; - PageTableEntry pte; - if (size == 8) - pte = read->get(); - else - pte = read->get(); - VAddr vaddr = entry.vaddr; - bool uncacheable = pte.pcd; - Addr nextRead = 0; - bool doWrite = false; - bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX; - switch(state) { - case LongPML4: - DPRINTF(PageTableWalker, - "Got long mode PML4 entry %#016x.\n", (uint64_t)pte); - nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * size; - doWrite = !pte.a; - pte.a = 1; - entry.writable = pte.w; - entry.user = pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - entry.noExec = pte.nx; - nextState = LongPDP; - break; - case LongPDP: - DPRINTF(PageTableWalker, - "Got long mode PDP entry %#016x.\n", (uint64_t)pte); - nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * size; - doWrite = !pte.a; - pte.a = 1; - entry.writable = entry.writable && pte.w; - entry.user = entry.user && pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - nextState = LongPD; - break; - case LongPD: - DPRINTF(PageTableWalker, - "Got long mode PD entry %#016x.\n", (uint64_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = entry.writable && pte.w; - entry.user = entry.user && pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - if (!pte.ps) { - // 4 KB page - entry.size = 4 * (1 << 10); - nextRead = - ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * size; - nextState = LongPTE; - break; - } else { - // 2 MB page - entry.size = 2 * (1 << 20); - entry.paddr = (uint64_t)pte & (mask(31) << 21); - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 12); - entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - } - case LongPTE: - DPRINTF(PageTableWalker, - "Got long mode PTE entry %#016x.\n", (uint64_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = entry.writable && pte.w; - entry.user = entry.user && pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - entry.paddr = (uint64_t)pte & (mask(40) << 12); - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 12); - entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - case PAEPDP: - DPRINTF(PageTableWalker, - "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte); - nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * size; - if (!pte.p) { - stop(); - return pageFault(pte.p); - } - nextState = PAEPD; - break; - case PAEPD: - DPRINTF(PageTableWalker, - "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = pte.w; - entry.user = pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - if (!pte.ps) { - // 4 KB page - entry.size = 4 * (1 << 10); - nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * size; - nextState = PAEPTE; - break; - } else { - // 2 MB page - entry.size = 2 * (1 << 20); - entry.paddr = (uint64_t)pte & (mask(31) << 21); - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 12); - entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - } - case PAEPTE: - DPRINTF(PageTableWalker, - "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = entry.writable && pte.w; - entry.user = entry.user && pte.u; - if (badNX || !pte.p) { - stop(); - return pageFault(pte.p); - } - entry.paddr = (uint64_t)pte & (mask(40) << 12); - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 7); - entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - case PSEPD: - DPRINTF(PageTableWalker, - "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = pte.w; - entry.user = pte.u; - if (!pte.p) { - stop(); - return pageFault(pte.p); - } - if (!pte.ps) { - // 4 KB page - entry.size = 4 * (1 << 10); - nextRead = - ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size; - nextState = PTE; - break; - } else { - // 4 MB page - entry.size = 4 * (1 << 20); - entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22; - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 12); - entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - } - case PD: - DPRINTF(PageTableWalker, - "Got legacy mode PD entry %#08x.\n", (uint32_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = pte.w; - entry.user = pte.u; - if (!pte.p) { - stop(); - return pageFault(pte.p); - } - // 4 KB page - entry.size = 4 * (1 << 10); - nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * size; - nextState = PTE; - break; - case PTE: - DPRINTF(PageTableWalker, - "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte); - doWrite = !pte.a; - pte.a = 1; - entry.writable = pte.w; - entry.user = pte.u; - if (!pte.p) { - stop(); - return pageFault(pte.p); - } - entry.paddr = (uint64_t)pte & (mask(20) << 12); - entry.uncacheable = uncacheable; - entry.global = pte.g; - entry.patBit = bits(pte, 7); - entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); - tlb->insert(entry.vaddr, entry); - stop(); - return NoFault; - default: - panic("Unknown page table walker state %d!\n"); - } - PacketPtr oldRead = read; - //If we didn't return, we're setting up another read. - Request::Flags flags = oldRead->req->getFlags(); - flags.set(Request::UNCACHEABLE, uncacheable); - RequestPtr request = - new Request(nextRead, oldRead->getSize(), flags); - read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast); - read->allocate(); - //If we need to write, adjust the read packet to write the modified value - //back to memory. - if (doWrite) { - write = oldRead; - write->set(pte); - write->cmd = MemCmd::WriteReq; - write->setDest(Packet::Broadcast); - } else { - write = NULL; - delete oldRead->req; - delete oldRead; - } - return NoFault; -} - Fault Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation, RequestPtr _req, BaseTLB::Mode _mode) { - assert(state == Ready); - tc = _tc; - req = _req; - Addr vaddr = req->getVaddr(); - mode = _mode; - translation = _translation; - - VAddr addr = vaddr; - - //Figure out what we're doing. - CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3); - Addr top = 0; - // Check if we're in long mode or not - Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER); - size = 8; - if (efer.lma) { - // Do long mode. - state = LongPML4; - top = (cr3.longPdtb << 12) + addr.longl4 * size; - enableNX = efer.nxe; + // TODO: in timing mode, instead of blocking when there are other + // outstanding requests, see if this request can be coalesced with + // another one (i.e. either coalesce or start walk) + WalkerState * newState = new WalkerState(this, _translation, _req); + newState->initState(_tc, _mode, sys->getMemoryMode() == Enums::timing); + if (currStates.size()) { + assert(newState->isTiming()); + DPRINTF(PageTableWalker, "Walks in progress: %d\n", currStates.size()); + currStates.push_back(newState); + return NoFault; } else { - // We're in some flavor of legacy mode. - CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4); - if (cr4.pae) { - // Do legacy PAE. - state = PAEPDP; - top = (cr3.paePdtb << 5) + addr.pael3 * size; - enableNX = efer.nxe; - } else { - size = 4; - top = (cr3.pdtb << 12) + addr.norml2 * size; - if (cr4.pse) { - // Do legacy PSE. - state = PSEPD; - } else { - // Do legacy non PSE. - state = PD; - } - enableNX = false; + currStates.push_back(newState); + Fault fault = newState->startWalk(); + if (!newState->isTiming()) { + currStates.pop_front(); + delete newState; } - } - - nextState = Ready; - entry.vaddr = vaddr; - - Request::Flags flags = Request::PHYSICAL; - if (cr3.pcd) - flags.set(Request::UNCACHEABLE); - RequestPtr request = new Request(top, size, flags); - read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast); - read->allocate(); - Enums::MemoryMode memMode = sys->getMemoryMode(); - if (memMode == Enums::timing) { - nextState = state; - state = Waiting; - timingFault = NoFault; - sendPackets(); - } else if (memMode == Enums::atomic) { - Fault fault; - do { - port.sendAtomic(read); - PacketPtr write = NULL; - fault = doNext(write); - assert(fault == NoFault || read == NULL); - state = nextState; - nextState = Ready; - if (write) - port.sendAtomic(write); - } while(read); - state = Ready; - nextState = Waiting; return fault; - } else { - panic("Unrecognized memory system mode.\n"); } - return NoFault; +} + +Fault +Walker::startFunctional(ThreadContext * _tc, Addr &addr, Addr &pageSize, + BaseTLB::Mode _mode) +{ + funcState.initState(_tc, _mode); + return funcState.startFunctional(addr, pageSize); } bool @@ -400,60 +109,33 @@ Walker::WalkerPort::recvTiming(PacketPtr pkt) bool Walker::recvTiming(PacketPtr pkt) { - if (pkt->isResponse() && !pkt->wasNacked()) { - assert(inflight); - assert(state == Waiting); - assert(!read); - inflight--; - if (pkt->isRead()) { - state = nextState; - nextState = Ready; - PacketPtr write = NULL; - read = pkt; - timingFault = doNext(write); - state = Waiting; - assert(timingFault == NoFault || read == NULL); - if (write) { - writes.push_back(write); + if (pkt->isResponse() || pkt->wasNacked()) { + WalkerSenderState * senderState = + dynamic_cast(pkt->senderState); + pkt->senderState = senderState->saved; + WalkerState * senderWalk = senderState->senderWalk; + bool walkComplete = senderWalk->recvPacket(pkt); + delete senderState; + if (walkComplete) { + std::list::iterator iter; + for (iter = currStates.begin(); iter != currStates.end(); iter++) { + WalkerState * walkerState = *(iter); + if (walkerState == senderWalk) { + iter = currStates.erase(iter); + break; + } } - sendPackets(); - } else { - sendPackets(); - } - if (inflight == 0 && read == NULL && writes.size() == 0) { - state = Ready; - nextState = Waiting; - if (timingFault == NoFault) { - /* - * Finish the translation. Now that we now the right entry is - * in the TLB, this should work with no memory accesses. - * There could be new faults unrelated to the table walk like - * permissions violations, so we'll need the return value as - * well. - */ - bool delayedResponse; - Fault fault = tlb->translate(req, tc, NULL, mode, - delayedResponse, true); - assert(!delayedResponse); - // Let the CPU continue. - translation->finish(fault, req, tc, mode); - } else { - // There was a fault during the walk. Let the CPU know. - translation->finish(timingFault, req, tc, mode); - } - } - } else if (pkt->wasNacked()) { - pkt->reinitNacked(); - if (!port.sendTiming(pkt)) { - inflight--; - retrying = true; - if (pkt->isWrite()) { - writes.push_back(pkt); - } else { - assert(!read); - read = pkt; + delete senderWalk; + // Since we block requests when another is outstanding, we + // need to check if there is a waiting request to be serviced + if (currStates.size()) { + WalkerState * newState = currStates.front(); + if (!newState->wasStarted()) + newState->startWalk(); } } + } else { + DPRINTF(PageTableWalker, "Received strange packet\n"); } return true; } @@ -493,41 +175,19 @@ Walker::WalkerPort::recvRetry() void Walker::recvRetry() { - retrying = false; - sendPackets(); + std::list::iterator iter; + for (iter = currStates.begin(); iter != currStates.end(); iter++) { + WalkerState * walkerState = *(iter); + if (walkerState->isRetrying()) { + walkerState->retry(); + } + } } -void -Walker::sendPackets() +bool Walker::sendTiming(WalkerState* sendingState, PacketPtr pkt) { - //If we're already waiting for the port to become available, just return. - if (retrying) - return; - - //Reads always have priority - if (read) { - PacketPtr pkt = read; - read = NULL; - inflight++; - if (!port.sendTiming(pkt)) { - retrying = true; - read = pkt; - inflight--; - return; - } - } - //Send off as many of the writes as we can. - while (writes.size()) { - PacketPtr write = writes.back(); - writes.pop_back(); - inflight++; - if (!port.sendTiming(write)) { - retrying = true; - writes.push_back(write); - inflight--; - return; - } - } + pkt->senderState = new WalkerSenderState(sendingState, pkt->senderState); + return port.sendTiming(pkt); } Port * @@ -539,8 +199,512 @@ Walker::getPort(const std::string &if_name, int idx) panic("No page table walker port named %s!\n", if_name); } +void +Walker::WalkerState::initState(ThreadContext * _tc, + BaseTLB::Mode _mode, bool _isTiming) +{ + assert(state == Ready); + started = false; + tc = _tc; + mode = _mode; + timing = _isTiming; +} + Fault -Walker::pageFault(bool present) +Walker::WalkerState::startWalk() +{ + Fault fault = NoFault; + assert(started == false); + started = true; + setupWalk(req->getVaddr()); + if (timing) { + nextState = state; + state = Waiting; + timingFault = NoFault; + sendPackets(); + } else { + do { + walker->port.sendAtomic(read); + PacketPtr write = NULL; + fault = stepWalk(write); + assert(fault == NoFault || read == NULL); + state = nextState; + nextState = Ready; + if (write) + walker->port.sendAtomic(write); + } while(read); + state = Ready; + nextState = Waiting; + } + return fault; +} + +Fault +Walker::WalkerState::startFunctional(Addr &addr, Addr &pageSize) +{ + Fault fault = NoFault; + assert(started == false); + started = true; + setupWalk(addr); + + do { + walker->port.sendFunctional(read); + // On a functional access (page table lookup), writes should + // not happen so this pointer is ignored after stepWalk + PacketPtr write = NULL; + fault = stepWalk(write); + assert(fault == NoFault || read == NULL); + state = nextState; + nextState = Ready; + } while(read); + pageSize = entry.size; + addr = entry.paddr; + + return fault; +} + +Fault +Walker::WalkerState::stepWalk(PacketPtr &write) +{ + assert(state != Ready && state != Waiting); + Fault fault = NoFault; + write = NULL; + PageTableEntry pte; + if (dataSize == 8) + pte = read->get(); + else + pte = read->get(); + VAddr vaddr = entry.vaddr; + bool uncacheable = pte.pcd; + Addr nextRead = 0; + bool doWrite = false; + bool doTLBInsert = false; + bool doEndWalk = false; + bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX; + switch(state) { + case LongPML4: + DPRINTF(PageTableWalker, + "Got long mode PML4 entry %#016x.\n", (uint64_t)pte); + nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * dataSize; + doWrite = !pte.a; + pte.a = 1; + entry.writable = pte.w; + entry.user = pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + entry.noExec = pte.nx; + nextState = LongPDP; + break; + case LongPDP: + DPRINTF(PageTableWalker, + "Got long mode PDP entry %#016x.\n", (uint64_t)pte); + nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * dataSize; + doWrite = !pte.a; + pte.a = 1; + entry.writable = entry.writable && pte.w; + entry.user = entry.user && pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + nextState = LongPD; + break; + case LongPD: + DPRINTF(PageTableWalker, + "Got long mode PD entry %#016x.\n", (uint64_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = entry.writable && pte.w; + entry.user = entry.user && pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + if (!pte.ps) { + // 4 KB page + entry.size = 4 * (1 << 10); + nextRead = + ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * dataSize; + nextState = LongPTE; + break; + } else { + // 2 MB page + entry.size = 2 * (1 << 20); + entry.paddr = (uint64_t)pte & (mask(31) << 21); + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 12); + entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + } + case LongPTE: + DPRINTF(PageTableWalker, + "Got long mode PTE entry %#016x.\n", (uint64_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = entry.writable && pte.w; + entry.user = entry.user && pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + entry.paddr = (uint64_t)pte & (mask(40) << 12); + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 12); + entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + case PAEPDP: + DPRINTF(PageTableWalker, + "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte); + nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * dataSize; + if (!pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + nextState = PAEPD; + break; + case PAEPD: + DPRINTF(PageTableWalker, + "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = pte.w; + entry.user = pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + if (!pte.ps) { + // 4 KB page + entry.size = 4 * (1 << 10); + nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * dataSize; + nextState = PAEPTE; + break; + } else { + // 2 MB page + entry.size = 2 * (1 << 20); + entry.paddr = (uint64_t)pte & (mask(31) << 21); + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 12); + entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + } + case PAEPTE: + DPRINTF(PageTableWalker, + "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = entry.writable && pte.w; + entry.user = entry.user && pte.u; + if (badNX || !pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + entry.paddr = (uint64_t)pte & (mask(40) << 12); + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 7); + entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + case PSEPD: + DPRINTF(PageTableWalker, + "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = pte.w; + entry.user = pte.u; + if (!pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + if (!pte.ps) { + // 4 KB page + entry.size = 4 * (1 << 10); + nextRead = + ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize; + nextState = PTE; + break; + } else { + // 4 MB page + entry.size = 4 * (1 << 20); + entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22; + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 12); + entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + } + case PD: + DPRINTF(PageTableWalker, + "Got legacy mode PD entry %#08x.\n", (uint32_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = pte.w; + entry.user = pte.u; + if (!pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + // 4 KB page + entry.size = 4 * (1 << 10); + nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize; + nextState = PTE; + break; + case PTE: + DPRINTF(PageTableWalker, + "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte); + doWrite = !pte.a; + pte.a = 1; + entry.writable = pte.w; + entry.user = pte.u; + if (!pte.p) { + doEndWalk = true; + fault = pageFault(pte.p); + break; + } + entry.paddr = (uint64_t)pte & (mask(20) << 12); + entry.uncacheable = uncacheable; + entry.global = pte.g; + entry.patBit = bits(pte, 7); + entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); + doTLBInsert = true; + doEndWalk = true; + break; + default: + panic("Unknown page table walker state %d!\n"); + } + if (doEndWalk) { + if (doTLBInsert) + if (!functional) + walker->tlb->insert(entry.vaddr, entry); + endWalk(); + } else { + PacketPtr oldRead = read; + //If we didn't return, we're setting up another read. + Request::Flags flags = oldRead->req->getFlags(); + flags.set(Request::UNCACHEABLE, uncacheable); + RequestPtr request = + new Request(nextRead, oldRead->getSize(), flags); + read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast); + read->allocate(); + // If we need to write, adjust the read packet to write the modified + // value back to memory. + if (doWrite) { + write = oldRead; + write->set(pte); + write->cmd = MemCmd::WriteReq; + write->setDest(Packet::Broadcast); + } else { + write = NULL; + delete oldRead->req; + delete oldRead; + } + } + return fault; +} + +void +Walker::WalkerState::endWalk() +{ + nextState = Ready; + delete read->req; + delete read; + read = NULL; +} + +void +Walker::WalkerState::setupWalk(Addr vaddr) +{ + VAddr addr = vaddr; + CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3); + // Check if we're in long mode or not + Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER); + dataSize = 8; + Addr topAddr; + if (efer.lma) { + // Do long mode. + state = LongPML4; + topAddr = (cr3.longPdtb << 12) + addr.longl4 * dataSize; + enableNX = efer.nxe; + } else { + // We're in some flavor of legacy mode. + CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4); + if (cr4.pae) { + // Do legacy PAE. + state = PAEPDP; + topAddr = (cr3.paePdtb << 5) + addr.pael3 * dataSize; + enableNX = efer.nxe; + } else { + dataSize = 4; + topAddr = (cr3.pdtb << 12) + addr.norml2 * dataSize; + if (cr4.pse) { + // Do legacy PSE. + state = PSEPD; + } else { + // Do legacy non PSE. + state = PD; + } + enableNX = false; + } + } + + nextState = Ready; + entry.vaddr = vaddr; + + Request::Flags flags = Request::PHYSICAL; + if (cr3.pcd) + flags.set(Request::UNCACHEABLE); + RequestPtr request = new Request(topAddr, dataSize, flags); + read = new Packet(request, MemCmd::ReadExReq, Packet::Broadcast); + read->allocate(); +} + +bool +Walker::WalkerState::recvPacket(PacketPtr pkt) +{ + if (pkt->isResponse() && !pkt->wasNacked()) { + assert(inflight); + assert(state == Waiting); + assert(!read); + inflight--; + if (pkt->isRead()) { + state = nextState; + nextState = Ready; + PacketPtr write = NULL; + read = pkt; + timingFault = stepWalk(write); + state = Waiting; + assert(timingFault == NoFault || read == NULL); + if (write) { + writes.push_back(write); + } + sendPackets(); + } else { + sendPackets(); + } + if (inflight == 0 && read == NULL && writes.size() == 0) { + state = Ready; + nextState = Waiting; + if (timingFault == NoFault) { + /* + * Finish the translation. Now that we now the right entry is + * in the TLB, this should work with no memory accesses. + * There could be new faults unrelated to the table walk like + * permissions violations, so we'll need the return value as + * well. + */ + bool delayedResponse; + Fault fault = walker->tlb->translate(req, tc, NULL, mode, + delayedResponse, true); + assert(!delayedResponse); + // Let the CPU continue. + translation->finish(fault, req, tc, mode); + } else { + // There was a fault during the walk. Let the CPU know. + translation->finish(timingFault, req, tc, mode); + } + return true; + } + } else if (pkt->wasNacked()) { + DPRINTF(PageTableWalker, "Request was nacked. Entering retry state\n"); + pkt->reinitNacked(); + if (!walker->sendTiming(this, pkt)) { + inflight--; + retrying = true; + if (pkt->isWrite()) { + writes.push_back(pkt); + } else { + assert(!read); + read = pkt; + } + } + } + return false; +} + +void +Walker::WalkerState::sendPackets() +{ + //If we're already waiting for the port to become available, just return. + if (retrying) + return; + + //Reads always have priority + if (read) { + PacketPtr pkt = read; + read = NULL; + inflight++; + if (!walker->sendTiming(this, pkt)) { + retrying = true; + read = pkt; + inflight--; + return; + } + } + //Send off as many of the writes as we can. + while (writes.size()) { + PacketPtr write = writes.back(); + writes.pop_back(); + inflight++; + if (!walker->sendTiming(this, write)) { + retrying = true; + writes.push_back(write); + inflight--; + return; + } + } +} + +bool +Walker::WalkerState::isRetrying() +{ + return retrying; +} + +bool +Walker::WalkerState::isTiming() +{ + return timing; +} + +bool +Walker::WalkerState::wasStarted() +{ + return started; +} + +void +Walker::WalkerState::retry() +{ + retrying = false; + sendPackets(); +} + +Fault +Walker::WalkerState::pageFault(bool present) { DPRINTF(PageTableWalker, "Raising page fault.\n"); HandyM5Reg m5reg = tc->readMiscRegNoEffect(MISCREG_M5_REG); @@ -549,7 +713,7 @@ Walker::pageFault(bool present) return new PageFault(entry.vaddr, present, mode, m5reg.cpl == 3, false); } -} +/* end namespace X86ISA */ } X86ISA::Walker * X86PagetableWalkerParams::create() diff --git a/src/arch/x86/pagetable_walker.hh b/src/arch/x86/pagetable_walker.hh index 68f85be935..007c577ae9 100644 --- a/src/arch/x86/pagetable_walker.hh +++ b/src/arch/x86/pagetable_walker.hh @@ -45,6 +45,7 @@ #include "arch/x86/pagetable.hh" #include "arch/x86/tlb.hh" #include "base/types.hh" +#include "base/fast_alloc.hh" #include "mem/mem_object.hh" #include "mem/packet.hh" #include "params/X86PagetableWalker.hh" @@ -56,70 +57,8 @@ namespace X86ISA { class Walker : public MemObject { - public: - enum State { - Ready, - Waiting, - // Long mode - LongPML4, LongPDP, LongPD, LongPTE, - // PAE legacy mode - PAEPDP, PAEPD, PAEPTE, - // Non PAE legacy mode with and without PSE - PSEPD, PD, PTE - }; - - // Act on the current state and determine what to do next. The global - // read should be the packet that just came back from a read and write - // should be NULL. When the function returns, read is either NULL - // if the machine is finished, or points to a packet to initiate - // the next read. If any write is required to update an "accessed" - // bit, write will point to a packet to do the write. Otherwise it - // will be NULL. The return value is whatever fault was incurred - // during this stage of the lookup. - Fault doNext(PacketPtr &write); - - // Kick off the state machine. - Fault start(ThreadContext * _tc, BaseTLB::Translation *translation, - RequestPtr req, BaseTLB::Mode mode); - // Clean up after the state machine. - void - stop() - { - nextState = Ready; - delete read->req; - delete read; - read = NULL; - } - protected: - - /* - * State having to do with sending packets. - */ - PacketPtr read; - std::vector writes; - - // How many memory operations are in flight. - unsigned inflight; - - bool retrying; - - /* - * The fault, if any, that's waiting to be delivered in timing mode. - */ - Fault timingFault; - - /* - * Functions for dealing with packets. - */ - bool recvTiming(PacketPtr pkt); - void recvRetry(); - - void sendPackets(); - - /* - * Port for accessing memory - */ + // Port for accessing memory class WalkerPort : public Port { public: @@ -146,31 +85,106 @@ namespace X86ISA } }; + friend class WalkerPort; + WalkerPort port; Port *getPort(const std::string &if_name, int idx = -1); - friend class WalkerPort; + // State to track each walk of the page table + class WalkerState : public FastAlloc + { + private: + enum State { + Ready, + Waiting, + // Long mode + LongPML4, LongPDP, LongPD, LongPTE, + // PAE legacy mode + PAEPDP, PAEPD, PAEPTE, + // Non PAE legacy mode with and without PSE + PSEPD, PD, PTE + }; - WalkerPort port; + protected: + Walker * walker; + ThreadContext *tc; + RequestPtr req; + State state; + State nextState; + int dataSize; + bool enableNX; + unsigned inflight; + TlbEntry entry; + PacketPtr read; + std::vector writes; + Fault timingFault; + TLB::Translation * translation; + BaseTLB::Mode mode; + bool functional; + bool timing; + bool retrying; + bool started; + public: + WalkerState(Walker * _walker, BaseTLB::Translation *_translation, + RequestPtr _req, bool _isFunctional = false) : + walker(_walker), req(_req), state(Ready), + nextState(Ready), inflight(0), + translation(_translation), + functional(_isFunctional), timing(false), + retrying(false), started(false) + { + } + void initState(ThreadContext * _tc, BaseTLB::Mode _mode, + bool _isTiming = false); + Fault startWalk(); + Fault startFunctional(Addr &addr, Addr &pageSize); + bool recvPacket(PacketPtr pkt); + bool isRetrying(); + bool wasStarted(); + bool isTiming(); + void retry(); + std::string name() const {return walker->name();} + + private: + void setupWalk(Addr vaddr); + Fault stepWalk(PacketPtr &write); + void sendPackets(); + void endWalk(); + Fault pageFault(bool present); + }; + + friend class WalkerState; + // State for timing and atomic accesses (need multiple per walker in + // the case of multiple outstanding requests in timing mode) + std::list currStates; + // State for functional accesses (only need one of these per walker) + WalkerState funcState; + + struct WalkerSenderState : public Packet::SenderState + { + WalkerState * senderWalk; + Packet::SenderState * saved; + WalkerSenderState(WalkerState * _senderWalk, + Packet::SenderState * _saved) : + senderWalk(_senderWalk), saved(_saved) {} + }; + + public: + // Kick off the state machine. + Fault start(ThreadContext * _tc, BaseTLB::Translation *translation, + RequestPtr req, BaseTLB::Mode mode); + Fault startFunctional(ThreadContext * _tc, Addr &addr, + Addr &pageSize, BaseTLB::Mode mode); + + protected: // The TLB we're supposed to load. TLB * tlb; System * sys; - BaseTLB::Translation * translation; - /* - * State machine state. - */ - ThreadContext * tc; - RequestPtr req; - State state; - State nextState; - int size; - bool enableNX; - BaseTLB::Mode mode; - bool user; - TlbEntry entry; - - Fault pageFault(bool present); + // Functions for dealing with packets. + bool recvTiming(PacketPtr pkt); + void recvRetry(); + bool sendTiming(WalkerState * sendingState, PacketPtr pkt); public: @@ -182,11 +196,8 @@ namespace X86ISA typedef X86PagetableWalkerParams Params; Walker(const Params *params) : - MemObject(params), - read(NULL), inflight(0), retrying(false), - port(name() + ".port", this), - tlb(NULL), sys(params->system), - tc(NULL), state(Ready), nextState(Ready) + MemObject(params), port(name() + ".port", this), + funcState(this, NULL, NULL, true), tlb(NULL), sys(params->system) { } }; diff --git a/src/arch/x86/tlb.cc b/src/arch/x86/tlb.cc index 7fa2e172cc..a02c5e6a30 100644 --- a/src/arch/x86/tlb.cc +++ b/src/arch/x86/tlb.cc @@ -725,6 +725,12 @@ TLB::doMmuRegWrite(ThreadContext *tc, Packet *pkt) return tc->getCpuPtr()->ticks(1); } +Walker * +TLB::getWalker() +{ + return walker; +} + #endif void diff --git a/src/arch/x86/tlb.hh b/src/arch/x86/tlb.hh index 0e96b26b82..965d331ba8 100644 --- a/src/arch/x86/tlb.hh +++ b/src/arch/x86/tlb.hh @@ -89,6 +89,8 @@ namespace X86ISA protected: Walker * walker; + public: + Walker *getWalker(); #endif public: