gem5/src/dev/dma_device.cc

/*
 * Copyright (c) 2012, 2015, 2017, 2019 ARM Limited
 * All rights reserved.
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Copyright (c) 2006 The Regents of The University of Michigan
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "dev/dma_device.hh"

#include <algorithm>
#include <cassert>
#include <cstring>
#include <utility>

#include "base/logging.hh"
#include "base/trace.hh"
#include "debug/DMA.hh"
#include "debug/Drain.hh"
#include "sim/clocked_object.hh"
#include "sim/system.hh"

namespace gem5
{

DmaPort::DmaPort(ClockedObject *dev, System *s,
                 uint32_t sid, uint32_t ssid)
    : RequestPort(dev->name() + ".dma"),
      device(dev), sys(s), requestorId(s->getRequestorId(dev)),
      sendEvent([this]{ sendDma(); }, dev->name()),
      defaultSid(sid), defaultSSid(ssid), cacheLineSize(s->cacheLineSize())
{ }

void
DmaPort::handleRespPacket(PacketPtr pkt, Tick delay)
{
    // Should always see a response with a sender state.
    assert(pkt->isResponse());
    warn_if(pkt->isError(), "Response pkt error.");

    // Get the DMA sender state.
    auto *state = dynamic_cast<DmaReqState*>(pkt->senderState);
    assert(state);

    handleResp(state, pkt->getAddr(), pkt->req->getSize(), delay);

    delete pkt;
}

void
DmaPort::handleResp(DmaReqState *state, Addr addr, Addr size, Tick delay)
{
    assert(pendingCount != 0);
    pendingCount--;
    DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d,"  \
            " tot: %d sched %d\n",
            MemCmd(state->cmd).toString(), addr, size,
            state->numBytes, state->totBytes,
            state->completionEvent ?
            state->completionEvent->scheduled() : 0);

    // Update the number of bytes received based on the request rather
    // than the packet as the latter could be rounded up to line sizes.
    state->numBytes += size;
    assert(state->totBytes >= state->numBytes);

    bool all_bytes = (state->totBytes == state->numBytes);
    if (state->aborted) {
        // If this request was aborted, check to see if its in flight accesses
        // have finished. There may be packets for more than one request in
        // flight at a time, so check for finished requests, or no more
        // packets.
        if (all_bytes || pendingCount == 0) {
            // If yes, signal its abort event (if any) and delete the state.
            if (state->abortEvent) {
                device->schedule(state->abortEvent, curTick());
            }
            delete state;
        }
    } else if (all_bytes) {
        // If we have reached the end of this DMA request, then signal the
        // completion and delete the sate.
        if (state->completionEvent) {
            delay += state->delay;
            device->schedule(state->completionEvent, curTick() + delay);
        }
        delete state;
    }

    // We might be drained at this point, if so signal the drain event.
    if (pendingCount == 0)
        signalDrainDone();
}

PacketPtr
DmaPort::DmaReqState::createPacket()
{
    RequestPtr req = std::make_shared<Request>(
            gen.addr(), gen.size(), flags, id);
    req->setStreamId(sid);
    req->setSubstreamId(ssid);
    req->taskId(context_switch_task_id::DMA);

    PacketPtr pkt = new Packet(req, cmd);

    if (data)
        pkt->dataStatic(data + gen.complete());

    pkt->senderState = this;
    return pkt;
}

bool
DmaPort::recvTimingResp(PacketPtr pkt)
{
    // We shouldn't ever get a cacheable block in Modified state.
    assert(pkt->req->isUncacheable() ||
           !(pkt->cacheResponding() && !pkt->hasSharers()));

    handleRespPacket(pkt);

    return true;
}

DmaDevice::DmaDevice(const Params &p)
    : PioDevice(p), dmaPort(this, sys, p.sid, p.ssid)
{ }

void
DmaDevice::init()
{
    panic_if(!dmaPort.isConnected(),
             "DMA port of %s not connected to anything!", name());
    PioDevice::init();
}

DrainState
DmaPort::drain()
{
    if (pendingCount == 0) {
        return DrainState::Drained;
    } else {
        DPRINTF(Drain, "DmaPort not drained\n");
        return DrainState::Draining;
    }
}

void
DmaPort::recvReqRetry()
{
    retryPending = false;
    if (transmitList.size())
        trySendTimingReq();
}

void
DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
                   uint8_t *data, uint32_t sid, uint32_t ssid, Tick delay,
                   Request::Flags flag)
{
    DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
            event ? event->scheduled() : -1);

    // One DMA request sender state for every action, that is then
    // split into many requests and packets based on the block size,
    // i.e. cache line size.
    transmitList.push_back(
            new DmaReqState(cmd, addr, cacheLineSize, size,
                data, flag, requestorId, sid, ssid, event, delay));

    // In zero time, also initiate the sending of the packets for the request
    // we have just created. For atomic this involves actually completing all
    // the requests.
    sendDma();
}

void
DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
                   uint8_t *data, Tick delay, Request::Flags flag)
{
    dmaAction(cmd, addr, size, event, data,
              defaultSid, defaultSSid, delay, flag);
}

void
DmaPort::abortPending()
{
    if (inRetry) {
        delete inRetry;
        inRetry = nullptr;
    }

    if (pendingCount && !transmitList.empty()) {
        auto *state = transmitList.front();
        if (state->numBytes != state->gen.complete()) {
            // In flight packets refer to the transmission at the front of the
            // list, and not a transmission whose packets have all been sent
            // but not completed. Preserve the state so the packets don't have
            // dangling pointers.
            transmitList.pop_front();
            state->aborted = true;
        }
    }

    // Get rid of requests that haven't started yet.
    while (!transmitList.empty()) {
        auto *state = transmitList.front();
        if (state->abortEvent)
            device->schedule(state->abortEvent, curTick());
        delete state;
        transmitList.pop_front();
    }

    if (sendEvent.scheduled())
        device->deschedule(sendEvent);

    if (pendingCount == 0)
        signalDrainDone();
}

void
DmaPort::trySendTimingReq()
{
    // Send the next packet for the first DMA request on the transmit list,
    // and schedule the following send if it is successful
    DmaReqState *state = transmitList.front();

    PacketPtr pkt = inRetry ? inRetry : state->createPacket();
    inRetry = nullptr;

    DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
            pkt->getAddr());

    // Check if this was the last packet now, since hypothetically the packet
    // response may come immediately, and state may be deleted.
    bool last = state->gen.last();
    if (sendTimingReq(pkt)) {
        pendingCount++;
    } else {
        retryPending = true;
        inRetry = pkt;
    }
    if (!retryPending) {
        state->gen.next();
        // If that was the last packet from this request, pop it from the list.
        if (last)
            transmitList.pop_front();
        DPRINTF(DMA, "-- Done\n");
        // If there is more to do, then do so.
        if (!transmitList.empty()) {
            // This should ultimately wait for as many cycles as the device
            // needs to send the packet, but currently the port does not have
            // any known width so simply wait a single cycle.
            device->schedule(sendEvent, device->clockEdge(Cycles(1)));
        }
    } else {
        DPRINTF(DMA, "-- Failed, waiting for retry\n");
    }

    DPRINTF(DMA, "TransmitList: %d, retryPending: %d\n",
            transmitList.size(), retryPending ? 1 : 0);
}

bool
DmaPort::sendAtomicReq(DmaReqState *state)
{
    PacketPtr pkt = state->createPacket();
    DPRINTF(DMA, "Sending  DMA for addr: %#x size: %d\n",
            state->gen.addr(), state->gen.size());
    pendingCount++;
    Tick lat = sendAtomic(pkt);

    // Check if we're done, since handleResp may delete state.
    bool done = !state->gen.next();
    handleRespPacket(pkt, lat);
    return done;
}

bool
DmaPort::sendAtomicBdReq(DmaReqState *state)
{
    bool done = false;
    pendingCount++;

    auto bd_it = memBackdoors.contains(state->gen.addr());
    if (bd_it == memBackdoors.end()) {
        // We don't have a backdoor for this address, so use a packet.

        PacketPtr pkt = state->createPacket();
        DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
                state->gen.addr(), state->gen.size());

        MemBackdoorPtr bd = nullptr;
        Tick lat = sendAtomicBackdoor(pkt, bd);

        // If we got a backdoor, record it.
        if (bd && memBackdoors.insert(bd->range(), bd) != memBackdoors.end()) {
            // Invalidation callback which finds this backdoor and removes it.
            auto callback = [this](const MemBackdoor &backdoor) {
                for (auto it = memBackdoors.begin();
                        it != memBackdoors.end(); it++) {
                    if (it->second == &backdoor) {
                        memBackdoors.erase(it);
                        return;
                    }
                }
                panic("Got invalidation for unknown memory backdoor.");
            };
            bd->addInvalidationCallback(callback);
        }

        // Check if we're done now, since handleResp may delete state.
        done = !state->gen.next();
        handleRespPacket(pkt, lat);
    } else {
        // We have a backdoor that can at least partially satisfy this request.
        DPRINTF(DMA, "Handling DMA for addr: %#x size %d through backdoor\n",
                state->gen.addr(), state->gen.size());

        const auto *bd = bd_it->second;
        // Offset of this access into the backdoor.
        const Addr offset = state->gen.addr() - bd->range().start();
        // How many bytes we still need.
        const Addr remaining = state->totBytes - state->gen.complete();
        // How many bytes this backdoor can provide, starting from offset.
        const Addr available = bd->range().size() - offset;

        // How many bytes we're going to handle through this backdoor.
        const Addr handled = std::min(remaining, available);

        // If there's a buffer for data, read/write it.
        if (state->data) {
            uint8_t *bd_data = bd->ptr() + offset;
            uint8_t *state_data = state->data + state->gen.complete();
            if (MemCmd(state->cmd).isRead())
                memcpy(state_data, bd_data, handled);
            else
                memcpy(bd_data, state_data, handled);
        }

        // Advance the chunk generator past this region of memory.
        state->gen.setNext(state->gen.addr() + handled);

        // Check if we're done now, since handleResp may delete state.
        done = !state->gen.next();
        handleResp(state, state->gen.addr(), handled);
    }

    return done;
}

void
DmaPort::sendDma()
{
    // Some kind of selection between access methods. More work is going to
    // have to be done to make switching actually work.
    assert(transmitList.size());

    if (sys->isTimingMode()) {
        // If we are either waiting for a retry or are still waiting after
        // sending the last packet, then do not proceed.
        if (retryPending || sendEvent.scheduled()) {
            DPRINTF(DMA, "Can't send immediately, waiting to send\n");
            return;
        }

        trySendTimingReq();
    } else if (sys->isAtomicMode()) {
        const bool bypass = sys->bypassCaches();

        // Send everything there is to send in zero time.
        while (!transmitList.empty()) {
            DmaReqState *state = transmitList.front();
            transmitList.pop_front();

            bool done = state->gen.done();
            while (!done)
                done = bypass ? sendAtomicBdReq(state) : sendAtomicReq(state);
        }
    } else {
        panic("Unknown memory mode.");
    }
}

Port &
DmaDevice::getPort(const std::string &if_name, PortID idx)
{
    if (if_name == "dma") {
        return dmaPort;
    }
    return PioDevice::getPort(if_name, idx);
}

DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
                         unsigned max_req_size,
                         unsigned max_pending,
                         Request::Flags flags)
    : maxReqSize(max_req_size), fifoSize(size),
      reqFlags(flags), port(_port), cacheLineSize(port.sys->cacheLineSize()),
      buffer(size)
{
    freeRequests.resize(max_pending);
    for (auto &e : freeRequests)
        e.reset(new DmaDoneEvent(this, max_req_size));

}

DmaReadFifo::~DmaReadFifo()
{
    for (auto &p : pendingRequests) {
        DmaDoneEvent *e(p.release());

        if (e->done()) {
            delete e;
        } else {
            // We can't kill in-flight DMAs, so we'll just transfer
            // ownership to the event queue so that they get freed
            // when they are done.
            e->kill();
        }
    }
}

void
DmaReadFifo::serialize(CheckpointOut &cp) const
{
    assert(pendingRequests.empty());

    SERIALIZE_CONTAINER(buffer);
    SERIALIZE_SCALAR(endAddr);
    SERIALIZE_SCALAR(nextAddr);
}

void
DmaReadFifo::unserialize(CheckpointIn &cp)
{
    UNSERIALIZE_CONTAINER(buffer);
    UNSERIALIZE_SCALAR(endAddr);
    UNSERIALIZE_SCALAR(nextAddr);
}

bool
DmaReadFifo::tryGet(uint8_t *dst, size_t len)
{
    if (buffer.size() >= len) {
        buffer.read(dst, len);
        resumeFill();
        return true;
    } else {
        return false;
    }
}

void
DmaReadFifo::get(uint8_t *dst, size_t len)
{
    panic_if(!tryGet(dst, len), "Buffer underrun in DmaReadFifo::get()");
}

void
DmaReadFifo::startFill(Addr start, size_t size)
{
    assert(atEndOfBlock());

    nextAddr = start;
    endAddr = start + size;
    resumeFill();
}

void
DmaReadFifo::stopFill()
{
    // Prevent new DMA requests by setting the next address to the end
    // address. Pending requests will still complete.
    nextAddr = endAddr;

    // Flag in-flight accesses as canceled. This prevents their data
    // from being written to the FIFO.
    for (auto &p : pendingRequests)
        p->cancel();
}

void
DmaReadFifo::resumeFill()
{
    // Don't try to fetch more data if we are draining. This ensures
    // that the DMA engine settles down before we checkpoint it.
    if (drainState() == DrainState::Draining)
        return;

    const bool old_eob(atEndOfBlock());

    if (port.sys->bypassCaches())
        resumeFillBypass();
    else
        resumeFillTiming();

    if (!old_eob && atEndOfBlock())
        onEndOfBlock();
}

void
DmaReadFifo::resumeFillBypass()
{
    const size_t fifo_space = buffer.capacity() - buffer.size();
    if (fifo_space >= cacheLineSize || buffer.capacity() < cacheLineSize) {
        const size_t block_remaining = endAddr - nextAddr;
        const size_t xfer_size = std::min(fifo_space, block_remaining);
        std::vector<uint8_t> tmp_buffer(xfer_size);

        assert(pendingRequests.empty());
        DPRINTF(DMA, "Direct bypass startAddr=%#x xfer_size=%#x " \
                "fifo_space=%#x block_remaining=%#x\n",
                nextAddr, xfer_size, fifo_space, block_remaining);

        port.dmaAction(MemCmd::ReadReq, nextAddr, xfer_size, nullptr,
                tmp_buffer.data(), 0, reqFlags);

        buffer.write(tmp_buffer.begin(), xfer_size);
        nextAddr += xfer_size;
    }
}

void
DmaReadFifo::resumeFillTiming()
{
    size_t size_pending(0);
    for (auto &e : pendingRequests)
        size_pending += e->requestSize();

    while (!freeRequests.empty() && !atEndOfBlock()) {
        const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
        if (buffer.size() + size_pending + req_size > fifoSize)
            break;

        DmaDoneEventUPtr event(std::move(freeRequests.front()));
        freeRequests.pop_front();
        assert(event);

        event->reset(req_size);
        port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
                       event->data(), 0, reqFlags);
        nextAddr += req_size;
        size_pending += req_size;

        pendingRequests.emplace_back(std::move(event));
    }
}

void
DmaReadFifo::dmaDone()
{
    const bool old_active(isActive());

    handlePending();
    resumeFill();

    if (old_active && !isActive())
        onIdle();
}

void
DmaReadFifo::handlePending()
{
    while (!pendingRequests.empty() && pendingRequests.front()->done()) {
        // Get the first finished pending request
        DmaDoneEventUPtr event(std::move(pendingRequests.front()));
        pendingRequests.pop_front();

        if (!event->canceled())
            buffer.write(event->data(), event->requestSize());

        // Move the event to the list of free requests
        freeRequests.emplace_back(std::move(event));
    }

    if (pendingRequests.empty())
        signalDrainDone();
}

DrainState
DmaReadFifo::drain()
{
    return pendingRequests.empty() ?
        DrainState::Drained : DrainState::Draining;
}


DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent, size_t max_size)
    : parent(_parent), _data(max_size, 0)
{
}

void
DmaReadFifo::DmaDoneEvent::kill()
{
    parent = nullptr;
    setFlags(AutoDelete);
}

void
DmaReadFifo::DmaDoneEvent::cancel()
{
    _canceled = true;
}

void
DmaReadFifo::DmaDoneEvent::reset(size_t size)
{
    assert(size <= _data.size());
    _done = false;
    _canceled = false;
    _requestSize = size;
}

void
DmaReadFifo::DmaDoneEvent::process()
{
    if (!parent)
        return;

    assert(!_done);
    _done = true;
    parent->dmaDone();
}

} // namespace gem5