1545SN/A/* 213930Sgiacomo.travaglini@arm.com * Copyright (c) 2012, 2015, 2017, 2019 ARM Limited 38948SN/A * All rights reserved. 48948SN/A * 58948SN/A * The license below extends only to copyright in the software and shall 68948SN/A * not be construed as granting a license to any other intellectual 78948SN/A * property including but not limited to intellectual property relating 88948SN/A * to a hardware implementation of the functionality of the software 98948SN/A * licensed hereunder. You may use the software subject to the license 108948SN/A * terms below provided that you ensure that this notice is replicated 118948SN/A * unmodified and in its entirety in all distributions of the software, 128948SN/A * modified or unmodified, in source code or in binary form. 138948SN/A * 142512SN/A * Copyright (c) 2006 The Regents of The University of Michigan 15545SN/A * All rights reserved. 16545SN/A * 17545SN/A * Redistribution and use in source and binary forms, with or without 18545SN/A * modification, are permitted provided that the following conditions are 19545SN/A * met: redistributions of source code must retain the above copyright 20545SN/A * notice, this list of conditions and the following disclaimer; 21545SN/A * redistributions in binary form must reproduce the above copyright 22545SN/A * notice, this list of conditions and the following disclaimer in the 23545SN/A * documentation and/or other materials provided with the distribution; 24545SN/A * neither the name of the copyright holders nor the names of its 25545SN/A * contributors may be used to endorse or promote products derived from 26545SN/A * this software without specific prior written permission. 27545SN/A * 28545SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29545SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30545SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31545SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32545SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33545SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34545SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35545SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36545SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37545SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38545SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 412665SN/A * Nathan Binkert 429166Sandreas.hansson@arm.com * Andreas Hansson 4311010Sandreas.sandberg@arm.com * Andreas Sandberg 44545SN/A */ 45545SN/A 4611010Sandreas.sandberg@arm.com#include "dev/dma_device.hh" 4711010Sandreas.sandberg@arm.com 4811010Sandreas.sandberg@arm.com#include <utility> 4911010Sandreas.sandberg@arm.com 503090SN/A#include "base/chunk_generator.hh" 518232SN/A#include "debug/DMA.hh" 529152Satgutier@umich.edu#include "debug/Drain.hh" 5311896Ssudhanshu.jha@arm.com#include "mem/port_proxy.hh" 5413892Sgabeblack@google.com#include "sim/clocked_object.hh" 552901SN/A#include "sim/system.hh" 56545SN/A 5713930Sgiacomo.travaglini@arm.comDmaPort::DmaPort(ClockedObject *dev, System *s, 5813930Sgiacomo.travaglini@arm.com uint32_t sid, uint32_t ssid) 5911010Sandreas.sandberg@arm.com : MasterPort(dev->name() + ".dma", dev), 6012680Sgiacomo.travaglini@arm.com device(dev), sys(s), masterId(s->getMasterId(dev)), 6112087Sspwilson2@wisc.edu sendEvent([this]{ sendDma(); }, dev->name()), 6213930Sgiacomo.travaglini@arm.com pendingCount(0), inRetry(false), 6313930Sgiacomo.travaglini@arm.com defaultSid(sid), 6413930Sgiacomo.travaglini@arm.com defaultSSid(ssid) 652489SN/A{ } 662489SN/A 679166Sandreas.hansson@arm.comvoid 689166Sandreas.hansson@arm.comDmaPort::handleResp(PacketPtr pkt, Tick delay) 699166Sandreas.hansson@arm.com{ 709166Sandreas.hansson@arm.com // should always see a response with a sender state 719166Sandreas.hansson@arm.com assert(pkt->isResponse()); 729166Sandreas.hansson@arm.com 739166Sandreas.hansson@arm.com // get the DMA sender state 749166Sandreas.hansson@arm.com DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState); 759166Sandreas.hansson@arm.com assert(state); 769166Sandreas.hansson@arm.com 779166Sandreas.hansson@arm.com DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \ 789166Sandreas.hansson@arm.com " tot: %d sched %d\n", 799166Sandreas.hansson@arm.com pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(), 809166Sandreas.hansson@arm.com state->numBytes, state->totBytes, 819166Sandreas.hansson@arm.com state->completionEvent ? 829166Sandreas.hansson@arm.com state->completionEvent->scheduled() : 0); 839166Sandreas.hansson@arm.com 849166Sandreas.hansson@arm.com assert(pendingCount != 0); 859166Sandreas.hansson@arm.com pendingCount--; 869166Sandreas.hansson@arm.com 879166Sandreas.hansson@arm.com // update the number of bytes received based on the request rather 889166Sandreas.hansson@arm.com // than the packet as the latter could be rounded up to line sizes 899166Sandreas.hansson@arm.com state->numBytes += pkt->req->getSize(); 909166Sandreas.hansson@arm.com assert(state->totBytes >= state->numBytes); 919166Sandreas.hansson@arm.com 929166Sandreas.hansson@arm.com // if we have reached the total number of bytes for this DMA 939166Sandreas.hansson@arm.com // request, then signal the completion and delete the sate 949166Sandreas.hansson@arm.com if (state->totBytes == state->numBytes) { 959166Sandreas.hansson@arm.com if (state->completionEvent) { 969166Sandreas.hansson@arm.com delay += state->delay; 979452SAndreas.Sandberg@ARM.com device->schedule(state->completionEvent, curTick() + delay); 989166Sandreas.hansson@arm.com } 999166Sandreas.hansson@arm.com delete state; 1009166Sandreas.hansson@arm.com } 1019166Sandreas.hansson@arm.com 10212749Sgiacomo.travaglini@arm.com // delete the packet 1039166Sandreas.hansson@arm.com delete pkt; 1049166Sandreas.hansson@arm.com 1059166Sandreas.hansson@arm.com // we might be drained at this point, if so signal the drain event 10610913Sandreas.sandberg@arm.com if (pendingCount == 0) 10710913Sandreas.sandberg@arm.com signalDrainDone(); 1089166Sandreas.hansson@arm.com} 1099166Sandreas.hansson@arm.com 1102489SN/Abool 1118975SN/ADmaPort::recvTimingResp(PacketPtr pkt) 1122384SN/A{ 11311284Sandreas.hansson@arm.com // We shouldn't ever get a cacheable block in Modified state 11410821Sandreas.hansson@arm.com assert(pkt->req->isUncacheable() || 11511284Sandreas.hansson@arm.com !(pkt->cacheResponding() && !pkt->hasSharers())); 1164435SN/A 1179166Sandreas.hansson@arm.com handleResp(pkt); 1182569SN/A 1192657SN/A return true; 1202384SN/A} 121679SN/A 1224762SN/ADmaDevice::DmaDevice(const Params *p) 12313930Sgiacomo.travaglini@arm.com : PioDevice(p), dmaPort(this, sys, p->sid, p->ssid) 1242565SN/A{ } 1252384SN/A 1268851SN/Avoid 1278851SN/ADmaDevice::init() 1288851SN/A{ 1298851SN/A if (!dmaPort.isConnected()) 1308851SN/A panic("DMA port of %s not connected to anything!", name()); 1318851SN/A PioDevice::init(); 1328851SN/A} 1338851SN/A 13410913Sandreas.sandberg@arm.comDrainState 13510913Sandreas.sandberg@arm.comDmaPort::drain() 1362901SN/A{ 13710913Sandreas.sandberg@arm.com if (pendingCount == 0) { 13810913Sandreas.sandberg@arm.com return DrainState::Drained; 13910913Sandreas.sandberg@arm.com } else { 14010913Sandreas.sandberg@arm.com DPRINTF(Drain, "DmaPort not drained\n"); 14110913Sandreas.sandberg@arm.com return DrainState::Draining; 14210913Sandreas.sandberg@arm.com } 1432901SN/A} 1442901SN/A 1452384SN/Avoid 14610713Sandreas.hansson@arm.comDmaPort::recvReqRetry() 1472489SN/A{ 1484435SN/A assert(transmitList.size()); 1499307Sandreas.hansson@arm.com trySendTimingReq(); 1502489SN/A} 1512641SN/A 15210621SCurtis.Dunham@arm.comRequestPtr 1532641SN/ADmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event, 15413930Sgiacomo.travaglini@arm.com uint8_t *data, uint32_t sid, uint32_t ssid, Tick delay, 15513930Sgiacomo.travaglini@arm.com Request::Flags flag) 1562384SN/A{ 1579166Sandreas.hansson@arm.com // one DMA request sender state for every action, that is then 1589166Sandreas.hansson@arm.com // split into many requests and packets based on the block size, 1599166Sandreas.hansson@arm.com // i.e. cache line size 1609016Sandreas.hansson@arm.com DmaReqState *reqState = new DmaReqState(event, size, delay); 1612384SN/A 16210621SCurtis.Dunham@arm.com // (functionality added for Table Walker statistics) 16310621SCurtis.Dunham@arm.com // We're only interested in this when there will only be one request. 16410621SCurtis.Dunham@arm.com // For simplicity, we return the last request, which would also be 16510621SCurtis.Dunham@arm.com // the only request in that case. 16610621SCurtis.Dunham@arm.com RequestPtr req = NULL; 16710621SCurtis.Dunham@arm.com 1684451SN/A DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size, 1699166Sandreas.hansson@arm.com event ? event->scheduled() : -1); 1709814Sandreas.hansson@arm.com for (ChunkGenerator gen(addr, size, sys->cacheLineSize()); 1712406SN/A !gen.done(); gen.next()) { 17212749Sgiacomo.travaglini@arm.com 17312749Sgiacomo.travaglini@arm.com req = std::make_shared<Request>( 17412749Sgiacomo.travaglini@arm.com gen.addr(), gen.size(), flag, masterId); 17512749Sgiacomo.travaglini@arm.com 17613930Sgiacomo.travaglini@arm.com req->setStreamId(sid); 17713930Sgiacomo.travaglini@arm.com req->setSubStreamId(ssid); 17813930Sgiacomo.travaglini@arm.com 17910024Sdam.sunwoo@arm.com req->taskId(ContextSwitchTaskId::DMA); 1809166Sandreas.hansson@arm.com PacketPtr pkt = new Packet(req, cmd); 1812641SN/A 1829166Sandreas.hansson@arm.com // Increment the data pointer on a write 1839166Sandreas.hansson@arm.com if (data) 1849166Sandreas.hansson@arm.com pkt->dataStatic(data + gen.complete()); 1852641SN/A 1869166Sandreas.hansson@arm.com pkt->senderState = reqState; 1872641SN/A 1889166Sandreas.hansson@arm.com DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(), 1899166Sandreas.hansson@arm.com gen.size()); 1909166Sandreas.hansson@arm.com queueDma(pkt); 1912384SN/A } 1929307Sandreas.hansson@arm.com 1939307Sandreas.hansson@arm.com // in zero time also initiate the sending of the packets we have 1949307Sandreas.hansson@arm.com // just created, for atomic this involves actually completing all 1959307Sandreas.hansson@arm.com // the requests 1969307Sandreas.hansson@arm.com sendDma(); 19710621SCurtis.Dunham@arm.com 19810621SCurtis.Dunham@arm.com return req; 1992384SN/A} 2002384SN/A 20113930Sgiacomo.travaglini@arm.comRequestPtr 20213930Sgiacomo.travaglini@arm.comDmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event, 20313930Sgiacomo.travaglini@arm.com uint8_t *data, Tick delay, Request::Flags flag) 20413930Sgiacomo.travaglini@arm.com{ 20513930Sgiacomo.travaglini@arm.com return dmaAction(cmd, addr, size, event, data, 20613930Sgiacomo.travaglini@arm.com defaultSid, defaultSSid, delay, flag); 20713930Sgiacomo.travaglini@arm.com} 20813930Sgiacomo.travaglini@arm.com 2094435SN/Avoid 2109166Sandreas.hansson@arm.comDmaPort::queueDma(PacketPtr pkt) 2114435SN/A{ 2129166Sandreas.hansson@arm.com transmitList.push_back(pkt); 2134435SN/A 2149166Sandreas.hansson@arm.com // remember that we have another packet pending, this will only be 2159166Sandreas.hansson@arm.com // decremented once a response comes back 2169166Sandreas.hansson@arm.com pendingCount++; 2179307Sandreas.hansson@arm.com} 2189166Sandreas.hansson@arm.com 2199307Sandreas.hansson@arm.comvoid 2209307Sandreas.hansson@arm.comDmaPort::trySendTimingReq() 2219307Sandreas.hansson@arm.com{ 2229307Sandreas.hansson@arm.com // send the first packet on the transmit list and schedule the 2239307Sandreas.hansson@arm.com // following send if it is successful 2249307Sandreas.hansson@arm.com PacketPtr pkt = transmitList.front(); 2259307Sandreas.hansson@arm.com 2269307Sandreas.hansson@arm.com DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(), 2279307Sandreas.hansson@arm.com pkt->getAddr()); 2289307Sandreas.hansson@arm.com 2299307Sandreas.hansson@arm.com inRetry = !sendTimingReq(pkt); 2309307Sandreas.hansson@arm.com if (!inRetry) { 2319307Sandreas.hansson@arm.com transmitList.pop_front(); 2329307Sandreas.hansson@arm.com DPRINTF(DMA, "-- Done\n"); 2339307Sandreas.hansson@arm.com // if there is more to do, then do so 2349307Sandreas.hansson@arm.com if (!transmitList.empty()) 2359307Sandreas.hansson@arm.com // this should ultimately wait for as many cycles as the 2369307Sandreas.hansson@arm.com // device needs to send the packet, but currently the port 2379307Sandreas.hansson@arm.com // does not have any known width so simply wait a single 2389307Sandreas.hansson@arm.com // cycle 2399307Sandreas.hansson@arm.com device->schedule(sendEvent, device->clockEdge(Cycles(1))); 2409307Sandreas.hansson@arm.com } else { 2419307Sandreas.hansson@arm.com DPRINTF(DMA, "-- Failed, waiting for retry\n"); 2429307Sandreas.hansson@arm.com } 2439307Sandreas.hansson@arm.com 2449307Sandreas.hansson@arm.com DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n", 2459307Sandreas.hansson@arm.com transmitList.size(), inRetry); 2464435SN/A} 2474435SN/A 2482384SN/Avoid 2494435SN/ADmaPort::sendDma() 2502384SN/A{ 2519166Sandreas.hansson@arm.com // some kind of selcetion between access methods 2522901SN/A // more work is going to have to be done to make 2532901SN/A // switching actually work 2544435SN/A assert(transmitList.size()); 2552902SN/A 2569524SAndreas.Sandberg@ARM.com if (sys->isTimingMode()) { 2579307Sandreas.hansson@arm.com // if we are either waiting for a retry or are still waiting 2589307Sandreas.hansson@arm.com // after sending the last packet, then do not proceed 2599307Sandreas.hansson@arm.com if (inRetry || sendEvent.scheduled()) { 2609307Sandreas.hansson@arm.com DPRINTF(DMA, "Can't send immediately, waiting to send\n"); 2614435SN/A return; 2624435SN/A } 2634435SN/A 2649307Sandreas.hansson@arm.com trySendTimingReq(); 2659524SAndreas.Sandberg@ARM.com } else if (sys->isAtomicMode()) { 2669307Sandreas.hansson@arm.com // send everything there is to send in zero time 2679307Sandreas.hansson@arm.com while (!transmitList.empty()) { 2689307Sandreas.hansson@arm.com PacketPtr pkt = transmitList.front(); 2699307Sandreas.hansson@arm.com transmitList.pop_front(); 2704435SN/A 2719307Sandreas.hansson@arm.com DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n", 2729307Sandreas.hansson@arm.com pkt->req->getPaddr(), pkt->req->getSize()); 2739307Sandreas.hansson@arm.com Tick lat = sendAtomic(pkt); 2744435SN/A 2759307Sandreas.hansson@arm.com handleResp(pkt, lat); 2769307Sandreas.hansson@arm.com } 2779166Sandreas.hansson@arm.com } else 2789166Sandreas.hansson@arm.com panic("Unknown memory mode."); 279545SN/A} 2808598SN/A 28113784Sgabeblack@google.comPort & 28213784Sgabeblack@google.comDmaDevice::getPort(const std::string &if_name, PortID idx) 2838598SN/A{ 2848598SN/A if (if_name == "dma") { 2858922SN/A return dmaPort; 2868598SN/A } 28713784Sgabeblack@google.com return PioDevice::getPort(if_name, idx); 2888598SN/A} 28911010Sandreas.sandberg@arm.com 29011010Sandreas.sandberg@arm.comDmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size, 29111010Sandreas.sandberg@arm.com unsigned max_req_size, 29211010Sandreas.sandberg@arm.com unsigned max_pending, 29311010Sandreas.sandberg@arm.com Request::Flags flags) 29411010Sandreas.sandberg@arm.com : maxReqSize(max_req_size), fifoSize(size), 29511010Sandreas.sandberg@arm.com reqFlags(flags), port(_port), 29611010Sandreas.sandberg@arm.com buffer(size), 29711010Sandreas.sandberg@arm.com nextAddr(0), endAddr(0) 29811010Sandreas.sandberg@arm.com{ 29911010Sandreas.sandberg@arm.com freeRequests.resize(max_pending); 30011010Sandreas.sandberg@arm.com for (auto &e : freeRequests) 30111010Sandreas.sandberg@arm.com e.reset(new DmaDoneEvent(this, max_req_size)); 30211010Sandreas.sandberg@arm.com 30311010Sandreas.sandberg@arm.com} 30411010Sandreas.sandberg@arm.com 30511010Sandreas.sandberg@arm.comDmaReadFifo::~DmaReadFifo() 30611010Sandreas.sandberg@arm.com{ 30711010Sandreas.sandberg@arm.com for (auto &p : pendingRequests) { 30811010Sandreas.sandberg@arm.com DmaDoneEvent *e(p.release()); 30911010Sandreas.sandberg@arm.com 31011010Sandreas.sandberg@arm.com if (e->done()) { 31111010Sandreas.sandberg@arm.com delete e; 31211010Sandreas.sandberg@arm.com } else { 31311010Sandreas.sandberg@arm.com // We can't kill in-flight DMAs, so we'll just transfer 31411010Sandreas.sandberg@arm.com // ownership to the event queue so that they get freed 31511010Sandreas.sandberg@arm.com // when they are done. 31611010Sandreas.sandberg@arm.com e->kill(); 31711010Sandreas.sandberg@arm.com } 31811010Sandreas.sandberg@arm.com } 31911010Sandreas.sandberg@arm.com} 32011010Sandreas.sandberg@arm.com 32111010Sandreas.sandberg@arm.comvoid 32211010Sandreas.sandberg@arm.comDmaReadFifo::serialize(CheckpointOut &cp) const 32311010Sandreas.sandberg@arm.com{ 32411010Sandreas.sandberg@arm.com assert(pendingRequests.empty()); 32511010Sandreas.sandberg@arm.com 32611010Sandreas.sandberg@arm.com SERIALIZE_CONTAINER(buffer); 32711010Sandreas.sandberg@arm.com SERIALIZE_SCALAR(endAddr); 32811010Sandreas.sandberg@arm.com SERIALIZE_SCALAR(nextAddr); 32911010Sandreas.sandberg@arm.com} 33011010Sandreas.sandberg@arm.com 33111010Sandreas.sandberg@arm.comvoid 33211010Sandreas.sandberg@arm.comDmaReadFifo::unserialize(CheckpointIn &cp) 33311010Sandreas.sandberg@arm.com{ 33411010Sandreas.sandberg@arm.com UNSERIALIZE_CONTAINER(buffer); 33511010Sandreas.sandberg@arm.com UNSERIALIZE_SCALAR(endAddr); 33611010Sandreas.sandberg@arm.com UNSERIALIZE_SCALAR(nextAddr); 33711010Sandreas.sandberg@arm.com} 33811010Sandreas.sandberg@arm.com 33911010Sandreas.sandberg@arm.combool 34011010Sandreas.sandberg@arm.comDmaReadFifo::tryGet(uint8_t *dst, size_t len) 34111010Sandreas.sandberg@arm.com{ 34211010Sandreas.sandberg@arm.com if (buffer.size() >= len) { 34311010Sandreas.sandberg@arm.com buffer.read(dst, len); 34411010Sandreas.sandberg@arm.com resumeFill(); 34511010Sandreas.sandberg@arm.com return true; 34611010Sandreas.sandberg@arm.com } else { 34711010Sandreas.sandberg@arm.com return false; 34811010Sandreas.sandberg@arm.com } 34911010Sandreas.sandberg@arm.com} 35011010Sandreas.sandberg@arm.com 35111010Sandreas.sandberg@arm.comvoid 35211010Sandreas.sandberg@arm.comDmaReadFifo::get(uint8_t *dst, size_t len) 35311010Sandreas.sandberg@arm.com{ 35411010Sandreas.sandberg@arm.com const bool success(tryGet(dst, len)); 35511010Sandreas.sandberg@arm.com panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n"); 35611010Sandreas.sandberg@arm.com} 35711010Sandreas.sandberg@arm.com 35811010Sandreas.sandberg@arm.comvoid 35911010Sandreas.sandberg@arm.comDmaReadFifo::startFill(Addr start, size_t size) 36011010Sandreas.sandberg@arm.com{ 36111010Sandreas.sandberg@arm.com assert(atEndOfBlock()); 36211010Sandreas.sandberg@arm.com 36311010Sandreas.sandberg@arm.com nextAddr = start; 36411010Sandreas.sandberg@arm.com endAddr = start + size; 36511010Sandreas.sandberg@arm.com resumeFill(); 36611010Sandreas.sandberg@arm.com} 36711010Sandreas.sandberg@arm.com 36811010Sandreas.sandberg@arm.comvoid 36911010Sandreas.sandberg@arm.comDmaReadFifo::stopFill() 37011010Sandreas.sandberg@arm.com{ 37111010Sandreas.sandberg@arm.com // Prevent new DMA requests by setting the next address to the end 37211010Sandreas.sandberg@arm.com // address. Pending requests will still complete. 37311010Sandreas.sandberg@arm.com nextAddr = endAddr; 37411010Sandreas.sandberg@arm.com 37511010Sandreas.sandberg@arm.com // Flag in-flight accesses as canceled. This prevents their data 37611010Sandreas.sandberg@arm.com // from being written to the FIFO. 37711010Sandreas.sandberg@arm.com for (auto &p : pendingRequests) 37811010Sandreas.sandberg@arm.com p->cancel(); 37911010Sandreas.sandberg@arm.com} 38011010Sandreas.sandberg@arm.com 38111010Sandreas.sandberg@arm.comvoid 38211010Sandreas.sandberg@arm.comDmaReadFifo::resumeFill() 38311010Sandreas.sandberg@arm.com{ 38411010Sandreas.sandberg@arm.com // Don't try to fetch more data if we are draining. This ensures 38511010Sandreas.sandberg@arm.com // that the DMA engine settles down before we checkpoint it. 38611010Sandreas.sandberg@arm.com if (drainState() == DrainState::Draining) 38711010Sandreas.sandberg@arm.com return; 38811010Sandreas.sandberg@arm.com 38911010Sandreas.sandberg@arm.com const bool old_eob(atEndOfBlock()); 39011896Ssudhanshu.jha@arm.com 39111896Ssudhanshu.jha@arm.com if (port.sys->bypassCaches()) 39211896Ssudhanshu.jha@arm.com resumeFillFunctional(); 39311896Ssudhanshu.jha@arm.com else 39411896Ssudhanshu.jha@arm.com resumeFillTiming(); 39511896Ssudhanshu.jha@arm.com 39611896Ssudhanshu.jha@arm.com if (!old_eob && atEndOfBlock()) 39711896Ssudhanshu.jha@arm.com onEndOfBlock(); 39811896Ssudhanshu.jha@arm.com} 39911896Ssudhanshu.jha@arm.com 40011896Ssudhanshu.jha@arm.comvoid 40111896Ssudhanshu.jha@arm.comDmaReadFifo::resumeFillFunctional() 40211896Ssudhanshu.jha@arm.com{ 40311896Ssudhanshu.jha@arm.com const size_t fifo_space = buffer.capacity() - buffer.size(); 40411896Ssudhanshu.jha@arm.com const size_t kvm_watermark = port.sys->cacheLineSize(); 40511896Ssudhanshu.jha@arm.com if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) { 40611896Ssudhanshu.jha@arm.com const size_t block_remaining = endAddr - nextAddr; 40711896Ssudhanshu.jha@arm.com const size_t xfer_size = std::min(fifo_space, block_remaining); 40811896Ssudhanshu.jha@arm.com std::vector<uint8_t> tmp_buffer(xfer_size); 40911896Ssudhanshu.jha@arm.com 41011896Ssudhanshu.jha@arm.com assert(pendingRequests.empty()); 41111896Ssudhanshu.jha@arm.com DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \ 41211896Ssudhanshu.jha@arm.com "fifo_space=%#x block_remaining=%#x\n", 41311896Ssudhanshu.jha@arm.com nextAddr, xfer_size, fifo_space, block_remaining); 41411896Ssudhanshu.jha@arm.com 41511896Ssudhanshu.jha@arm.com port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size); 41611896Ssudhanshu.jha@arm.com buffer.write(tmp_buffer.begin(), xfer_size); 41711896Ssudhanshu.jha@arm.com nextAddr += xfer_size; 41811896Ssudhanshu.jha@arm.com } 41911896Ssudhanshu.jha@arm.com} 42011896Ssudhanshu.jha@arm.com 42111896Ssudhanshu.jha@arm.comvoid 42211896Ssudhanshu.jha@arm.comDmaReadFifo::resumeFillTiming() 42311896Ssudhanshu.jha@arm.com{ 42411010Sandreas.sandberg@arm.com size_t size_pending(0); 42511010Sandreas.sandberg@arm.com for (auto &e : pendingRequests) 42611010Sandreas.sandberg@arm.com size_pending += e->requestSize(); 42711010Sandreas.sandberg@arm.com 42811010Sandreas.sandberg@arm.com while (!freeRequests.empty() && !atEndOfBlock()) { 42911010Sandreas.sandberg@arm.com const size_t req_size(std::min(maxReqSize, endAddr - nextAddr)); 43011010Sandreas.sandberg@arm.com if (buffer.size() + size_pending + req_size > fifoSize) 43111010Sandreas.sandberg@arm.com break; 43211010Sandreas.sandberg@arm.com 43311010Sandreas.sandberg@arm.com DmaDoneEventUPtr event(std::move(freeRequests.front())); 43411010Sandreas.sandberg@arm.com freeRequests.pop_front(); 43511010Sandreas.sandberg@arm.com assert(event); 43611010Sandreas.sandberg@arm.com 43711010Sandreas.sandberg@arm.com event->reset(req_size); 43811010Sandreas.sandberg@arm.com port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(), 43911010Sandreas.sandberg@arm.com event->data(), 0, reqFlags); 44011010Sandreas.sandberg@arm.com nextAddr += req_size; 44111010Sandreas.sandberg@arm.com size_pending += req_size; 44211010Sandreas.sandberg@arm.com 44311010Sandreas.sandberg@arm.com pendingRequests.emplace_back(std::move(event)); 44411010Sandreas.sandberg@arm.com } 44511010Sandreas.sandberg@arm.com} 44611010Sandreas.sandberg@arm.com 44711010Sandreas.sandberg@arm.comvoid 44811010Sandreas.sandberg@arm.comDmaReadFifo::dmaDone() 44911010Sandreas.sandberg@arm.com{ 45011010Sandreas.sandberg@arm.com const bool old_active(isActive()); 45111010Sandreas.sandberg@arm.com 45211010Sandreas.sandberg@arm.com handlePending(); 45311010Sandreas.sandberg@arm.com resumeFill(); 45411010Sandreas.sandberg@arm.com 45512115Srohit.kurup@arm.com if (old_active && !isActive()) 45611010Sandreas.sandberg@arm.com onIdle(); 45711010Sandreas.sandberg@arm.com} 45811010Sandreas.sandberg@arm.com 45911010Sandreas.sandberg@arm.comvoid 46011010Sandreas.sandberg@arm.comDmaReadFifo::handlePending() 46111010Sandreas.sandberg@arm.com{ 46211010Sandreas.sandberg@arm.com while (!pendingRequests.empty() && pendingRequests.front()->done()) { 46311010Sandreas.sandberg@arm.com // Get the first finished pending request 46411010Sandreas.sandberg@arm.com DmaDoneEventUPtr event(std::move(pendingRequests.front())); 46511010Sandreas.sandberg@arm.com pendingRequests.pop_front(); 46611010Sandreas.sandberg@arm.com 46711010Sandreas.sandberg@arm.com if (!event->canceled()) 46811010Sandreas.sandberg@arm.com buffer.write(event->data(), event->requestSize()); 46911010Sandreas.sandberg@arm.com 47011010Sandreas.sandberg@arm.com // Move the event to the list of free requests 47111010Sandreas.sandberg@arm.com freeRequests.emplace_back(std::move(event)); 47211010Sandreas.sandberg@arm.com } 47311010Sandreas.sandberg@arm.com 47411010Sandreas.sandberg@arm.com if (pendingRequests.empty()) 47511010Sandreas.sandberg@arm.com signalDrainDone(); 47611010Sandreas.sandberg@arm.com} 47711010Sandreas.sandberg@arm.com 47811010Sandreas.sandberg@arm.comDrainState 47911010Sandreas.sandberg@arm.comDmaReadFifo::drain() 48011010Sandreas.sandberg@arm.com{ 48111010Sandreas.sandberg@arm.com return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining; 48211010Sandreas.sandberg@arm.com} 48311010Sandreas.sandberg@arm.com 48411010Sandreas.sandberg@arm.com 48511010Sandreas.sandberg@arm.comDmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent, 48611010Sandreas.sandberg@arm.com size_t max_size) 48711010Sandreas.sandberg@arm.com : parent(_parent), _done(false), _canceled(false), _data(max_size, 0) 48811010Sandreas.sandberg@arm.com{ 48911010Sandreas.sandberg@arm.com} 49011010Sandreas.sandberg@arm.com 49111010Sandreas.sandberg@arm.comvoid 49211010Sandreas.sandberg@arm.comDmaReadFifo::DmaDoneEvent::kill() 49311010Sandreas.sandberg@arm.com{ 49411010Sandreas.sandberg@arm.com parent = nullptr; 49511010Sandreas.sandberg@arm.com setFlags(AutoDelete); 49611010Sandreas.sandberg@arm.com} 49711010Sandreas.sandberg@arm.com 49811010Sandreas.sandberg@arm.comvoid 49911010Sandreas.sandberg@arm.comDmaReadFifo::DmaDoneEvent::cancel() 50011010Sandreas.sandberg@arm.com{ 50111010Sandreas.sandberg@arm.com _canceled = true; 50211010Sandreas.sandberg@arm.com} 50311010Sandreas.sandberg@arm.com 50411010Sandreas.sandberg@arm.comvoid 50511010Sandreas.sandberg@arm.comDmaReadFifo::DmaDoneEvent::reset(size_t size) 50611010Sandreas.sandberg@arm.com{ 50711010Sandreas.sandberg@arm.com assert(size <= _data.size()); 50811010Sandreas.sandberg@arm.com _done = false; 50911010Sandreas.sandberg@arm.com _canceled = false; 51011010Sandreas.sandberg@arm.com _requestSize = size; 51111010Sandreas.sandberg@arm.com} 51211010Sandreas.sandberg@arm.com 51311010Sandreas.sandberg@arm.comvoid 51411010Sandreas.sandberg@arm.comDmaReadFifo::DmaDoneEvent::process() 51511010Sandreas.sandberg@arm.com{ 51611010Sandreas.sandberg@arm.com if (!parent) 51711010Sandreas.sandberg@arm.com return; 51811010Sandreas.sandberg@arm.com 51911010Sandreas.sandberg@arm.com assert(!_done); 52011010Sandreas.sandberg@arm.com _done = true; 52111010Sandreas.sandberg@arm.com parent->dmaDone(); 52211010Sandreas.sandberg@arm.com} 523