1/* 2 * Copyright (c) 2012, 2015, 2017 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2006 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Ali Saidi 41 * Nathan Binkert 42 * Andreas Hansson 43 * Andreas Sandberg 44 */ 45 46#include "dev/dma_device.hh" 47 48#include <utility> 49 50#include "base/chunk_generator.hh" 51#include "debug/DMA.hh" 52#include "debug/Drain.hh" 53#include "mem/port_proxy.hh" 54#include "sim/system.hh" 55 56DmaPort::DmaPort(MemObject *dev, System *s) 57 : MasterPort(dev->name() + ".dma", dev), 58 device(dev), sys(s), masterId(s->getMasterId(dev)), 59 sendEvent([this]{ sendDma(); }, dev->name()), 60 pendingCount(0), inRetry(false) 61{ } 62 63void 64DmaPort::handleResp(PacketPtr pkt, Tick delay) 65{ 66 // should always see a response with a sender state 67 assert(pkt->isResponse()); 68 69 // get the DMA sender state 70 DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState); 71 assert(state); 72 73 DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \ 74 " tot: %d sched %d\n", 75 pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(), 76 state->numBytes, state->totBytes, 77 state->completionEvent ? 78 state->completionEvent->scheduled() : 0); 79 80 assert(pendingCount != 0); 81 pendingCount--; 82 83 // update the number of bytes received based on the request rather 84 // than the packet as the latter could be rounded up to line sizes 85 state->numBytes += pkt->req->getSize(); 86 assert(state->totBytes >= state->numBytes); 87 88 // if we have reached the total number of bytes for this DMA 89 // request, then signal the completion and delete the sate 90 if (state->totBytes == state->numBytes) { 91 if (state->completionEvent) { 92 delay += state->delay; 93 device->schedule(state->completionEvent, curTick() + delay); 94 } 95 delete state; 96 } 97
| 1/* 2 * Copyright (c) 2012, 2015, 2017 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2006 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Ali Saidi 41 * Nathan Binkert 42 * Andreas Hansson 43 * Andreas Sandberg 44 */ 45 46#include "dev/dma_device.hh" 47 48#include <utility> 49 50#include "base/chunk_generator.hh" 51#include "debug/DMA.hh" 52#include "debug/Drain.hh" 53#include "mem/port_proxy.hh" 54#include "sim/system.hh" 55 56DmaPort::DmaPort(MemObject *dev, System *s) 57 : MasterPort(dev->name() + ".dma", dev), 58 device(dev), sys(s), masterId(s->getMasterId(dev)), 59 sendEvent([this]{ sendDma(); }, dev->name()), 60 pendingCount(0), inRetry(false) 61{ } 62 63void 64DmaPort::handleResp(PacketPtr pkt, Tick delay) 65{ 66 // should always see a response with a sender state 67 assert(pkt->isResponse()); 68 69 // get the DMA sender state 70 DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState); 71 assert(state); 72 73 DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \ 74 " tot: %d sched %d\n", 75 pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(), 76 state->numBytes, state->totBytes, 77 state->completionEvent ? 78 state->completionEvent->scheduled() : 0); 79 80 assert(pendingCount != 0); 81 pendingCount--; 82 83 // update the number of bytes received based on the request rather 84 // than the packet as the latter could be rounded up to line sizes 85 state->numBytes += pkt->req->getSize(); 86 assert(state->totBytes >= state->numBytes); 87 88 // if we have reached the total number of bytes for this DMA 89 // request, then signal the completion and delete the sate 90 if (state->totBytes == state->numBytes) { 91 if (state->completionEvent) { 92 delay += state->delay; 93 device->schedule(state->completionEvent, curTick() + delay); 94 } 95 delete state; 96 } 97
|
98 // delete the request that we created and also the packet 99 delete pkt->req;
| 98 // delete the packet
|
100 delete pkt; 101 102 // we might be drained at this point, if so signal the drain event 103 if (pendingCount == 0) 104 signalDrainDone(); 105} 106 107bool 108DmaPort::recvTimingResp(PacketPtr pkt) 109{ 110 // We shouldn't ever get a cacheable block in Modified state 111 assert(pkt->req->isUncacheable() || 112 !(pkt->cacheResponding() && !pkt->hasSharers())); 113 114 handleResp(pkt); 115 116 return true; 117} 118 119DmaDevice::DmaDevice(const Params *p) 120 : PioDevice(p), dmaPort(this, sys) 121{ } 122 123void 124DmaDevice::init() 125{ 126 if (!dmaPort.isConnected()) 127 panic("DMA port of %s not connected to anything!", name()); 128 PioDevice::init(); 129} 130 131DrainState 132DmaPort::drain() 133{ 134 if (pendingCount == 0) { 135 return DrainState::Drained; 136 } else { 137 DPRINTF(Drain, "DmaPort not drained\n"); 138 return DrainState::Draining; 139 } 140} 141 142void 143DmaPort::recvReqRetry() 144{ 145 assert(transmitList.size()); 146 trySendTimingReq(); 147} 148 149RequestPtr 150DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event, 151 uint8_t *data, Tick delay, Request::Flags flag) 152{ 153 // one DMA request sender state for every action, that is then 154 // split into many requests and packets based on the block size, 155 // i.e. cache line size 156 DmaReqState *reqState = new DmaReqState(event, size, delay); 157 158 // (functionality added for Table Walker statistics) 159 // We're only interested in this when there will only be one request. 160 // For simplicity, we return the last request, which would also be 161 // the only request in that case. 162 RequestPtr req = NULL; 163 164 DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size, 165 event ? event->scheduled() : -1); 166 for (ChunkGenerator gen(addr, size, sys->cacheLineSize()); 167 !gen.done(); gen.next()) {
| 99 delete pkt; 100 101 // we might be drained at this point, if so signal the drain event 102 if (pendingCount == 0) 103 signalDrainDone(); 104} 105 106bool 107DmaPort::recvTimingResp(PacketPtr pkt) 108{ 109 // We shouldn't ever get a cacheable block in Modified state 110 assert(pkt->req->isUncacheable() || 111 !(pkt->cacheResponding() && !pkt->hasSharers())); 112 113 handleResp(pkt); 114 115 return true; 116} 117 118DmaDevice::DmaDevice(const Params *p) 119 : PioDevice(p), dmaPort(this, sys) 120{ } 121 122void 123DmaDevice::init() 124{ 125 if (!dmaPort.isConnected()) 126 panic("DMA port of %s not connected to anything!", name()); 127 PioDevice::init(); 128} 129 130DrainState 131DmaPort::drain() 132{ 133 if (pendingCount == 0) { 134 return DrainState::Drained; 135 } else { 136 DPRINTF(Drain, "DmaPort not drained\n"); 137 return DrainState::Draining; 138 } 139} 140 141void 142DmaPort::recvReqRetry() 143{ 144 assert(transmitList.size()); 145 trySendTimingReq(); 146} 147 148RequestPtr 149DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event, 150 uint8_t *data, Tick delay, Request::Flags flag) 151{ 152 // one DMA request sender state for every action, that is then 153 // split into many requests and packets based on the block size, 154 // i.e. cache line size 155 DmaReqState *reqState = new DmaReqState(event, size, delay); 156 157 // (functionality added for Table Walker statistics) 158 // We're only interested in this when there will only be one request. 159 // For simplicity, we return the last request, which would also be 160 // the only request in that case. 161 RequestPtr req = NULL; 162 163 DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size, 164 event ? event->scheduled() : -1); 165 for (ChunkGenerator gen(addr, size, sys->cacheLineSize()); 166 !gen.done(); gen.next()) {
|
168 req = new Request(gen.addr(), gen.size(), flag, masterId);
| 167 168 req = std::make_shared<Request>( 169 gen.addr(), gen.size(), flag, masterId); 170
|
169 req->taskId(ContextSwitchTaskId::DMA); 170 PacketPtr pkt = new Packet(req, cmd); 171 172 // Increment the data pointer on a write 173 if (data) 174 pkt->dataStatic(data + gen.complete()); 175 176 pkt->senderState = reqState; 177 178 DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(), 179 gen.size()); 180 queueDma(pkt); 181 } 182 183 // in zero time also initiate the sending of the packets we have 184 // just created, for atomic this involves actually completing all 185 // the requests 186 sendDma(); 187 188 return req; 189} 190 191void 192DmaPort::queueDma(PacketPtr pkt) 193{ 194 transmitList.push_back(pkt); 195 196 // remember that we have another packet pending, this will only be 197 // decremented once a response comes back 198 pendingCount++; 199} 200 201void 202DmaPort::trySendTimingReq() 203{ 204 // send the first packet on the transmit list and schedule the 205 // following send if it is successful 206 PacketPtr pkt = transmitList.front(); 207 208 DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(), 209 pkt->getAddr()); 210 211 inRetry = !sendTimingReq(pkt); 212 if (!inRetry) { 213 transmitList.pop_front(); 214 DPRINTF(DMA, "-- Done\n"); 215 // if there is more to do, then do so 216 if (!transmitList.empty()) 217 // this should ultimately wait for as many cycles as the 218 // device needs to send the packet, but currently the port 219 // does not have any known width so simply wait a single 220 // cycle 221 device->schedule(sendEvent, device->clockEdge(Cycles(1))); 222 } else { 223 DPRINTF(DMA, "-- Failed, waiting for retry\n"); 224 } 225 226 DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n", 227 transmitList.size(), inRetry); 228} 229 230void 231DmaPort::sendDma() 232{ 233 // some kind of selcetion between access methods 234 // more work is going to have to be done to make 235 // switching actually work 236 assert(transmitList.size()); 237 238 if (sys->isTimingMode()) { 239 // if we are either waiting for a retry or are still waiting 240 // after sending the last packet, then do not proceed 241 if (inRetry || sendEvent.scheduled()) { 242 DPRINTF(DMA, "Can't send immediately, waiting to send\n"); 243 return; 244 } 245 246 trySendTimingReq(); 247 } else if (sys->isAtomicMode()) { 248 // send everything there is to send in zero time 249 while (!transmitList.empty()) { 250 PacketPtr pkt = transmitList.front(); 251 transmitList.pop_front(); 252 253 DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n", 254 pkt->req->getPaddr(), pkt->req->getSize()); 255 Tick lat = sendAtomic(pkt); 256 257 handleResp(pkt, lat); 258 } 259 } else 260 panic("Unknown memory mode."); 261} 262 263BaseMasterPort & 264DmaDevice::getMasterPort(const std::string &if_name, PortID idx) 265{ 266 if (if_name == "dma") { 267 return dmaPort; 268 } 269 return PioDevice::getMasterPort(if_name, idx); 270} 271 272 273 274 275 276DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size, 277 unsigned max_req_size, 278 unsigned max_pending, 279 Request::Flags flags) 280 : maxReqSize(max_req_size), fifoSize(size), 281 reqFlags(flags), port(_port), 282 buffer(size), 283 nextAddr(0), endAddr(0) 284{ 285 freeRequests.resize(max_pending); 286 for (auto &e : freeRequests) 287 e.reset(new DmaDoneEvent(this, max_req_size)); 288 289} 290 291DmaReadFifo::~DmaReadFifo() 292{ 293 for (auto &p : pendingRequests) { 294 DmaDoneEvent *e(p.release()); 295 296 if (e->done()) { 297 delete e; 298 } else { 299 // We can't kill in-flight DMAs, so we'll just transfer 300 // ownership to the event queue so that they get freed 301 // when they are done. 302 e->kill(); 303 } 304 } 305} 306 307void 308DmaReadFifo::serialize(CheckpointOut &cp) const 309{ 310 assert(pendingRequests.empty()); 311 312 SERIALIZE_CONTAINER(buffer); 313 SERIALIZE_SCALAR(endAddr); 314 SERIALIZE_SCALAR(nextAddr); 315} 316 317void 318DmaReadFifo::unserialize(CheckpointIn &cp) 319{ 320 UNSERIALIZE_CONTAINER(buffer); 321 UNSERIALIZE_SCALAR(endAddr); 322 UNSERIALIZE_SCALAR(nextAddr); 323} 324 325bool 326DmaReadFifo::tryGet(uint8_t *dst, size_t len) 327{ 328 if (buffer.size() >= len) { 329 buffer.read(dst, len); 330 resumeFill(); 331 return true; 332 } else { 333 return false; 334 } 335} 336 337void 338DmaReadFifo::get(uint8_t *dst, size_t len) 339{ 340 const bool success(tryGet(dst, len)); 341 panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n"); 342} 343 344void 345DmaReadFifo::startFill(Addr start, size_t size) 346{ 347 assert(atEndOfBlock()); 348 349 nextAddr = start; 350 endAddr = start + size; 351 resumeFill(); 352} 353 354void 355DmaReadFifo::stopFill() 356{ 357 // Prevent new DMA requests by setting the next address to the end 358 // address. Pending requests will still complete. 359 nextAddr = endAddr; 360 361 // Flag in-flight accesses as canceled. This prevents their data 362 // from being written to the FIFO. 363 for (auto &p : pendingRequests) 364 p->cancel(); 365} 366 367void 368DmaReadFifo::resumeFill() 369{ 370 // Don't try to fetch more data if we are draining. This ensures 371 // that the DMA engine settles down before we checkpoint it. 372 if (drainState() == DrainState::Draining) 373 return; 374 375 const bool old_eob(atEndOfBlock()); 376 377 if (port.sys->bypassCaches()) 378 resumeFillFunctional(); 379 else 380 resumeFillTiming(); 381 382 if (!old_eob && atEndOfBlock()) 383 onEndOfBlock(); 384} 385 386void 387DmaReadFifo::resumeFillFunctional() 388{ 389 const size_t fifo_space = buffer.capacity() - buffer.size(); 390 const size_t kvm_watermark = port.sys->cacheLineSize(); 391 if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) { 392 const size_t block_remaining = endAddr - nextAddr; 393 const size_t xfer_size = std::min(fifo_space, block_remaining); 394 std::vector<uint8_t> tmp_buffer(xfer_size); 395 396 assert(pendingRequests.empty()); 397 DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \ 398 "fifo_space=%#x block_remaining=%#x\n", 399 nextAddr, xfer_size, fifo_space, block_remaining); 400 401 port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size); 402 buffer.write(tmp_buffer.begin(), xfer_size); 403 nextAddr += xfer_size; 404 } 405} 406 407void 408DmaReadFifo::resumeFillTiming() 409{ 410 size_t size_pending(0); 411 for (auto &e : pendingRequests) 412 size_pending += e->requestSize(); 413 414 while (!freeRequests.empty() && !atEndOfBlock()) { 415 const size_t req_size(std::min(maxReqSize, endAddr - nextAddr)); 416 if (buffer.size() + size_pending + req_size > fifoSize) 417 break; 418 419 DmaDoneEventUPtr event(std::move(freeRequests.front())); 420 freeRequests.pop_front(); 421 assert(event); 422 423 event->reset(req_size); 424 port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(), 425 event->data(), 0, reqFlags); 426 nextAddr += req_size; 427 size_pending += req_size; 428 429 pendingRequests.emplace_back(std::move(event)); 430 } 431} 432 433void 434DmaReadFifo::dmaDone() 435{ 436 const bool old_active(isActive()); 437 438 handlePending(); 439 resumeFill(); 440 441 if (old_active && !isActive()) 442 onIdle(); 443} 444 445void 446DmaReadFifo::handlePending() 447{ 448 while (!pendingRequests.empty() && pendingRequests.front()->done()) { 449 // Get the first finished pending request 450 DmaDoneEventUPtr event(std::move(pendingRequests.front())); 451 pendingRequests.pop_front(); 452 453 if (!event->canceled()) 454 buffer.write(event->data(), event->requestSize()); 455 456 // Move the event to the list of free requests 457 freeRequests.emplace_back(std::move(event)); 458 } 459 460 if (pendingRequests.empty()) 461 signalDrainDone(); 462} 463 464DrainState 465DmaReadFifo::drain() 466{ 467 return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining; 468} 469 470 471DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent, 472 size_t max_size) 473 : parent(_parent), _done(false), _canceled(false), _data(max_size, 0) 474{ 475} 476 477void 478DmaReadFifo::DmaDoneEvent::kill() 479{ 480 parent = nullptr; 481 setFlags(AutoDelete); 482} 483 484void 485DmaReadFifo::DmaDoneEvent::cancel() 486{ 487 _canceled = true; 488} 489 490void 491DmaReadFifo::DmaDoneEvent::reset(size_t size) 492{ 493 assert(size <= _data.size()); 494 _done = false; 495 _canceled = false; 496 _requestSize = size; 497} 498 499void 500DmaReadFifo::DmaDoneEvent::process() 501{ 502 if (!parent) 503 return; 504 505 assert(!_done); 506 _done = true; 507 parent->dmaDone(); 508}
| 171 req->taskId(ContextSwitchTaskId::DMA); 172 PacketPtr pkt = new Packet(req, cmd); 173 174 // Increment the data pointer on a write 175 if (data) 176 pkt->dataStatic(data + gen.complete()); 177 178 pkt->senderState = reqState; 179 180 DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(), 181 gen.size()); 182 queueDma(pkt); 183 } 184 185 // in zero time also initiate the sending of the packets we have 186 // just created, for atomic this involves actually completing all 187 // the requests 188 sendDma(); 189 190 return req; 191} 192 193void 194DmaPort::queueDma(PacketPtr pkt) 195{ 196 transmitList.push_back(pkt); 197 198 // remember that we have another packet pending, this will only be 199 // decremented once a response comes back 200 pendingCount++; 201} 202 203void 204DmaPort::trySendTimingReq() 205{ 206 // send the first packet on the transmit list and schedule the 207 // following send if it is successful 208 PacketPtr pkt = transmitList.front(); 209 210 DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(), 211 pkt->getAddr()); 212 213 inRetry = !sendTimingReq(pkt); 214 if (!inRetry) { 215 transmitList.pop_front(); 216 DPRINTF(DMA, "-- Done\n"); 217 // if there is more to do, then do so 218 if (!transmitList.empty()) 219 // this should ultimately wait for as many cycles as the 220 // device needs to send the packet, but currently the port 221 // does not have any known width so simply wait a single 222 // cycle 223 device->schedule(sendEvent, device->clockEdge(Cycles(1))); 224 } else { 225 DPRINTF(DMA, "-- Failed, waiting for retry\n"); 226 } 227 228 DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n", 229 transmitList.size(), inRetry); 230} 231 232void 233DmaPort::sendDma() 234{ 235 // some kind of selcetion between access methods 236 // more work is going to have to be done to make 237 // switching actually work 238 assert(transmitList.size()); 239 240 if (sys->isTimingMode()) { 241 // if we are either waiting for a retry or are still waiting 242 // after sending the last packet, then do not proceed 243 if (inRetry || sendEvent.scheduled()) { 244 DPRINTF(DMA, "Can't send immediately, waiting to send\n"); 245 return; 246 } 247 248 trySendTimingReq(); 249 } else if (sys->isAtomicMode()) { 250 // send everything there is to send in zero time 251 while (!transmitList.empty()) { 252 PacketPtr pkt = transmitList.front(); 253 transmitList.pop_front(); 254 255 DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n", 256 pkt->req->getPaddr(), pkt->req->getSize()); 257 Tick lat = sendAtomic(pkt); 258 259 handleResp(pkt, lat); 260 } 261 } else 262 panic("Unknown memory mode."); 263} 264 265BaseMasterPort & 266DmaDevice::getMasterPort(const std::string &if_name, PortID idx) 267{ 268 if (if_name == "dma") { 269 return dmaPort; 270 } 271 return PioDevice::getMasterPort(if_name, idx); 272} 273 274 275 276 277 278DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size, 279 unsigned max_req_size, 280 unsigned max_pending, 281 Request::Flags flags) 282 : maxReqSize(max_req_size), fifoSize(size), 283 reqFlags(flags), port(_port), 284 buffer(size), 285 nextAddr(0), endAddr(0) 286{ 287 freeRequests.resize(max_pending); 288 for (auto &e : freeRequests) 289 e.reset(new DmaDoneEvent(this, max_req_size)); 290 291} 292 293DmaReadFifo::~DmaReadFifo() 294{ 295 for (auto &p : pendingRequests) { 296 DmaDoneEvent *e(p.release()); 297 298 if (e->done()) { 299 delete e; 300 } else { 301 // We can't kill in-flight DMAs, so we'll just transfer 302 // ownership to the event queue so that they get freed 303 // when they are done. 304 e->kill(); 305 } 306 } 307} 308 309void 310DmaReadFifo::serialize(CheckpointOut &cp) const 311{ 312 assert(pendingRequests.empty()); 313 314 SERIALIZE_CONTAINER(buffer); 315 SERIALIZE_SCALAR(endAddr); 316 SERIALIZE_SCALAR(nextAddr); 317} 318 319void 320DmaReadFifo::unserialize(CheckpointIn &cp) 321{ 322 UNSERIALIZE_CONTAINER(buffer); 323 UNSERIALIZE_SCALAR(endAddr); 324 UNSERIALIZE_SCALAR(nextAddr); 325} 326 327bool 328DmaReadFifo::tryGet(uint8_t *dst, size_t len) 329{ 330 if (buffer.size() >= len) { 331 buffer.read(dst, len); 332 resumeFill(); 333 return true; 334 } else { 335 return false; 336 } 337} 338 339void 340DmaReadFifo::get(uint8_t *dst, size_t len) 341{ 342 const bool success(tryGet(dst, len)); 343 panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n"); 344} 345 346void 347DmaReadFifo::startFill(Addr start, size_t size) 348{ 349 assert(atEndOfBlock()); 350 351 nextAddr = start; 352 endAddr = start + size; 353 resumeFill(); 354} 355 356void 357DmaReadFifo::stopFill() 358{ 359 // Prevent new DMA requests by setting the next address to the end 360 // address. Pending requests will still complete. 361 nextAddr = endAddr; 362 363 // Flag in-flight accesses as canceled. This prevents their data 364 // from being written to the FIFO. 365 for (auto &p : pendingRequests) 366 p->cancel(); 367} 368 369void 370DmaReadFifo::resumeFill() 371{ 372 // Don't try to fetch more data if we are draining. This ensures 373 // that the DMA engine settles down before we checkpoint it. 374 if (drainState() == DrainState::Draining) 375 return; 376 377 const bool old_eob(atEndOfBlock()); 378 379 if (port.sys->bypassCaches()) 380 resumeFillFunctional(); 381 else 382 resumeFillTiming(); 383 384 if (!old_eob && atEndOfBlock()) 385 onEndOfBlock(); 386} 387 388void 389DmaReadFifo::resumeFillFunctional() 390{ 391 const size_t fifo_space = buffer.capacity() - buffer.size(); 392 const size_t kvm_watermark = port.sys->cacheLineSize(); 393 if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) { 394 const size_t block_remaining = endAddr - nextAddr; 395 const size_t xfer_size = std::min(fifo_space, block_remaining); 396 std::vector<uint8_t> tmp_buffer(xfer_size); 397 398 assert(pendingRequests.empty()); 399 DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \ 400 "fifo_space=%#x block_remaining=%#x\n", 401 nextAddr, xfer_size, fifo_space, block_remaining); 402 403 port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size); 404 buffer.write(tmp_buffer.begin(), xfer_size); 405 nextAddr += xfer_size; 406 } 407} 408 409void 410DmaReadFifo::resumeFillTiming() 411{ 412 size_t size_pending(0); 413 for (auto &e : pendingRequests) 414 size_pending += e->requestSize(); 415 416 while (!freeRequests.empty() && !atEndOfBlock()) { 417 const size_t req_size(std::min(maxReqSize, endAddr - nextAddr)); 418 if (buffer.size() + size_pending + req_size > fifoSize) 419 break; 420 421 DmaDoneEventUPtr event(std::move(freeRequests.front())); 422 freeRequests.pop_front(); 423 assert(event); 424 425 event->reset(req_size); 426 port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(), 427 event->data(), 0, reqFlags); 428 nextAddr += req_size; 429 size_pending += req_size; 430 431 pendingRequests.emplace_back(std::move(event)); 432 } 433} 434 435void 436DmaReadFifo::dmaDone() 437{ 438 const bool old_active(isActive()); 439 440 handlePending(); 441 resumeFill(); 442 443 if (old_active && !isActive()) 444 onIdle(); 445} 446 447void 448DmaReadFifo::handlePending() 449{ 450 while (!pendingRequests.empty() && pendingRequests.front()->done()) { 451 // Get the first finished pending request 452 DmaDoneEventUPtr event(std::move(pendingRequests.front())); 453 pendingRequests.pop_front(); 454 455 if (!event->canceled()) 456 buffer.write(event->data(), event->requestSize()); 457 458 // Move the event to the list of free requests 459 freeRequests.emplace_back(std::move(event)); 460 } 461 462 if (pendingRequests.empty()) 463 signalDrainDone(); 464} 465 466DrainState 467DmaReadFifo::drain() 468{ 469 return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining; 470} 471 472 473DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent, 474 size_t max_size) 475 : parent(_parent), _done(false), _canceled(false), _data(max_size, 0) 476{ 477} 478 479void 480DmaReadFifo::DmaDoneEvent::kill() 481{ 482 parent = nullptr; 483 setFlags(AutoDelete); 484} 485 486void 487DmaReadFifo::DmaDoneEvent::cancel() 488{ 489 _canceled = true; 490} 491 492void 493DmaReadFifo::DmaDoneEvent::reset(size_t size) 494{ 495 assert(size <= _data.size()); 496 _done = false; 497 _canceled = false; 498 _requestSize = size; 499} 500 501void 502DmaReadFifo::DmaDoneEvent::process() 503{ 504 if (!parent) 505 return; 506 507 assert(!_done); 508 _done = true; 509 parent->dmaDone(); 510}
|