dma_device.cc revision 11896
1/* 2 * Copyright (c) 2012, 2015, 2017 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2006 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Ali Saidi 41 * Nathan Binkert 42 * Andreas Hansson 43 * Andreas Sandberg 44 */ 45 46#include "dev/dma_device.hh" 47 48#include <utility> 49 50#include "base/chunk_generator.hh" 51#include "debug/DMA.hh" 52#include "debug/Drain.hh" 53#include "mem/port_proxy.hh" 54#include "sim/system.hh" 55 56DmaPort::DmaPort(MemObject *dev, System *s) 57 : MasterPort(dev->name() + ".dma", dev), 58 device(dev), sys(s), masterId(s->getMasterId(dev->name())), 59 sendEvent(this), pendingCount(0), inRetry(false) 60{ } 61 62void 63DmaPort::handleResp(PacketPtr pkt, Tick delay) 64{ 65 // should always see a response with a sender state 66 assert(pkt->isResponse()); 67 68 // get the DMA sender state 69 DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState); 70 assert(state); 71 72 DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \ 73 " tot: %d sched %d\n", 74 pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(), 75 state->numBytes, state->totBytes, 76 state->completionEvent ? 77 state->completionEvent->scheduled() : 0); 78 79 assert(pendingCount != 0); 80 pendingCount--; 81 82 // update the number of bytes received based on the request rather 83 // than the packet as the latter could be rounded up to line sizes 84 state->numBytes += pkt->req->getSize(); 85 assert(state->totBytes >= state->numBytes); 86 87 // if we have reached the total number of bytes for this DMA 88 // request, then signal the completion and delete the sate 89 if (state->totBytes == state->numBytes) { 90 if (state->completionEvent) { 91 delay += state->delay; 92 device->schedule(state->completionEvent, curTick() + delay); 93 } 94 delete state; 95 } 96 97 // delete the request that we created and also the packet 98 delete pkt->req; 99 delete pkt; 100 101 // we might be drained at this point, if so signal the drain event 102 if (pendingCount == 0) 103 signalDrainDone(); 104} 105 106bool 107DmaPort::recvTimingResp(PacketPtr pkt) 108{ 109 // We shouldn't ever get a cacheable block in Modified state 110 assert(pkt->req->isUncacheable() || 111 !(pkt->cacheResponding() && !pkt->hasSharers())); 112 113 handleResp(pkt); 114 115 return true; 116} 117 118DmaDevice::DmaDevice(const Params *p) 119 : PioDevice(p), dmaPort(this, sys) 120{ } 121 122void 123DmaDevice::init() 124{ 125 if (!dmaPort.isConnected()) 126 panic("DMA port of %s not connected to anything!", name()); 127 PioDevice::init(); 128} 129 130DrainState 131DmaPort::drain() 132{ 133 if (pendingCount == 0) { 134 return DrainState::Drained; 135 } else { 136 DPRINTF(Drain, "DmaPort not drained\n"); 137 return DrainState::Draining; 138 } 139} 140 141void 142DmaPort::recvReqRetry() 143{ 144 assert(transmitList.size()); 145 trySendTimingReq(); 146} 147 148RequestPtr 149DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event, 150 uint8_t *data, Tick delay, Request::Flags flag) 151{ 152 // one DMA request sender state for every action, that is then 153 // split into many requests and packets based on the block size, 154 // i.e. cache line size 155 DmaReqState *reqState = new DmaReqState(event, size, delay); 156 157 // (functionality added for Table Walker statistics) 158 // We're only interested in this when there will only be one request. 159 // For simplicity, we return the last request, which would also be 160 // the only request in that case. 161 RequestPtr req = NULL; 162 163 DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size, 164 event ? event->scheduled() : -1); 165 for (ChunkGenerator gen(addr, size, sys->cacheLineSize()); 166 !gen.done(); gen.next()) { 167 req = new Request(gen.addr(), gen.size(), flag, masterId); 168 req->taskId(ContextSwitchTaskId::DMA); 169 PacketPtr pkt = new Packet(req, cmd); 170 171 // Increment the data pointer on a write 172 if (data) 173 pkt->dataStatic(data + gen.complete()); 174 175 pkt->senderState = reqState; 176 177 DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(), 178 gen.size()); 179 queueDma(pkt); 180 } 181 182 // in zero time also initiate the sending of the packets we have 183 // just created, for atomic this involves actually completing all 184 // the requests 185 sendDma(); 186 187 return req; 188} 189 190void 191DmaPort::queueDma(PacketPtr pkt) 192{ 193 transmitList.push_back(pkt); 194 195 // remember that we have another packet pending, this will only be 196 // decremented once a response comes back 197 pendingCount++; 198} 199 200void 201DmaPort::trySendTimingReq() 202{ 203 // send the first packet on the transmit list and schedule the 204 // following send if it is successful 205 PacketPtr pkt = transmitList.front(); 206 207 DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(), 208 pkt->getAddr()); 209 210 inRetry = !sendTimingReq(pkt); 211 if (!inRetry) { 212 transmitList.pop_front(); 213 DPRINTF(DMA, "-- Done\n"); 214 // if there is more to do, then do so 215 if (!transmitList.empty()) 216 // this should ultimately wait for as many cycles as the 217 // device needs to send the packet, but currently the port 218 // does not have any known width so simply wait a single 219 // cycle 220 device->schedule(sendEvent, device->clockEdge(Cycles(1))); 221 } else { 222 DPRINTF(DMA, "-- Failed, waiting for retry\n"); 223 } 224 225 DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n", 226 transmitList.size(), inRetry); 227} 228 229void 230DmaPort::sendDma() 231{ 232 // some kind of selcetion between access methods 233 // more work is going to have to be done to make 234 // switching actually work 235 assert(transmitList.size()); 236 237 if (sys->isTimingMode()) { 238 // if we are either waiting for a retry or are still waiting 239 // after sending the last packet, then do not proceed 240 if (inRetry || sendEvent.scheduled()) { 241 DPRINTF(DMA, "Can't send immediately, waiting to send\n"); 242 return; 243 } 244 245 trySendTimingReq(); 246 } else if (sys->isAtomicMode()) { 247 // send everything there is to send in zero time 248 while (!transmitList.empty()) { 249 PacketPtr pkt = transmitList.front(); 250 transmitList.pop_front(); 251 252 DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n", 253 pkt->req->getPaddr(), pkt->req->getSize()); 254 Tick lat = sendAtomic(pkt); 255 256 handleResp(pkt, lat); 257 } 258 } else 259 panic("Unknown memory mode."); 260} 261 262BaseMasterPort & 263DmaDevice::getMasterPort(const std::string &if_name, PortID idx) 264{ 265 if (if_name == "dma") { 266 return dmaPort; 267 } 268 return PioDevice::getMasterPort(if_name, idx); 269} 270 271 272 273 274 275DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size, 276 unsigned max_req_size, 277 unsigned max_pending, 278 Request::Flags flags) 279 : maxReqSize(max_req_size), fifoSize(size), 280 reqFlags(flags), port(_port), 281 buffer(size), 282 nextAddr(0), endAddr(0) 283{ 284 freeRequests.resize(max_pending); 285 for (auto &e : freeRequests) 286 e.reset(new DmaDoneEvent(this, max_req_size)); 287 288} 289 290DmaReadFifo::~DmaReadFifo() 291{ 292 for (auto &p : pendingRequests) { 293 DmaDoneEvent *e(p.release()); 294 295 if (e->done()) { 296 delete e; 297 } else { 298 // We can't kill in-flight DMAs, so we'll just transfer 299 // ownership to the event queue so that they get freed 300 // when they are done. 301 e->kill(); 302 } 303 } 304} 305 306void 307DmaReadFifo::serialize(CheckpointOut &cp) const 308{ 309 assert(pendingRequests.empty()); 310 311 SERIALIZE_CONTAINER(buffer); 312 SERIALIZE_SCALAR(endAddr); 313 SERIALIZE_SCALAR(nextAddr); 314} 315 316void 317DmaReadFifo::unserialize(CheckpointIn &cp) 318{ 319 UNSERIALIZE_CONTAINER(buffer); 320 UNSERIALIZE_SCALAR(endAddr); 321 UNSERIALIZE_SCALAR(nextAddr); 322} 323 324bool 325DmaReadFifo::tryGet(uint8_t *dst, size_t len) 326{ 327 if (buffer.size() >= len) { 328 buffer.read(dst, len); 329 resumeFill(); 330 return true; 331 } else { 332 return false; 333 } 334} 335 336void 337DmaReadFifo::get(uint8_t *dst, size_t len) 338{ 339 const bool success(tryGet(dst, len)); 340 panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n"); 341} 342 343void 344DmaReadFifo::startFill(Addr start, size_t size) 345{ 346 assert(atEndOfBlock()); 347 348 nextAddr = start; 349 endAddr = start + size; 350 resumeFill(); 351} 352 353void 354DmaReadFifo::stopFill() 355{ 356 // Prevent new DMA requests by setting the next address to the end 357 // address. Pending requests will still complete. 358 nextAddr = endAddr; 359 360 // Flag in-flight accesses as canceled. This prevents their data 361 // from being written to the FIFO. 362 for (auto &p : pendingRequests) 363 p->cancel(); 364} 365 366void 367DmaReadFifo::resumeFill() 368{ 369 // Don't try to fetch more data if we are draining. This ensures 370 // that the DMA engine settles down before we checkpoint it. 371 if (drainState() == DrainState::Draining) 372 return; 373 374 const bool old_eob(atEndOfBlock()); 375 376 if (port.sys->bypassCaches()) 377 resumeFillFunctional(); 378 else 379 resumeFillTiming(); 380 381 if (!old_eob && atEndOfBlock()) 382 onEndOfBlock(); 383} 384 385void 386DmaReadFifo::resumeFillFunctional() 387{ 388 const size_t fifo_space = buffer.capacity() - buffer.size(); 389 const size_t kvm_watermark = port.sys->cacheLineSize(); 390 if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) { 391 const size_t block_remaining = endAddr - nextAddr; 392 const size_t xfer_size = std::min(fifo_space, block_remaining); 393 std::vector<uint8_t> tmp_buffer(xfer_size); 394 395 assert(pendingRequests.empty()); 396 DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \ 397 "fifo_space=%#x block_remaining=%#x\n", 398 nextAddr, xfer_size, fifo_space, block_remaining); 399 400 port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size); 401 buffer.write(tmp_buffer.begin(), xfer_size); 402 nextAddr += xfer_size; 403 } 404} 405 406void 407DmaReadFifo::resumeFillTiming() 408{ 409 size_t size_pending(0); 410 for (auto &e : pendingRequests) 411 size_pending += e->requestSize(); 412 413 while (!freeRequests.empty() && !atEndOfBlock()) { 414 const size_t req_size(std::min(maxReqSize, endAddr - nextAddr)); 415 if (buffer.size() + size_pending + req_size > fifoSize) 416 break; 417 418 DmaDoneEventUPtr event(std::move(freeRequests.front())); 419 freeRequests.pop_front(); 420 assert(event); 421 422 event->reset(req_size); 423 port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(), 424 event->data(), 0, reqFlags); 425 nextAddr += req_size; 426 size_pending += req_size; 427 428 pendingRequests.emplace_back(std::move(event)); 429 } 430} 431 432void 433DmaReadFifo::dmaDone() 434{ 435 const bool old_active(isActive()); 436 437 handlePending(); 438 resumeFill(); 439 440 if (!old_active && isActive()) 441 onIdle(); 442} 443 444void 445DmaReadFifo::handlePending() 446{ 447 while (!pendingRequests.empty() && pendingRequests.front()->done()) { 448 // Get the first finished pending request 449 DmaDoneEventUPtr event(std::move(pendingRequests.front())); 450 pendingRequests.pop_front(); 451 452 if (!event->canceled()) 453 buffer.write(event->data(), event->requestSize()); 454 455 // Move the event to the list of free requests 456 freeRequests.emplace_back(std::move(event)); 457 } 458 459 if (pendingRequests.empty()) 460 signalDrainDone(); 461} 462 463DrainState 464DmaReadFifo::drain() 465{ 466 return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining; 467} 468 469 470DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent, 471 size_t max_size) 472 : parent(_parent), _done(false), _canceled(false), _data(max_size, 0) 473{ 474} 475 476void 477DmaReadFifo::DmaDoneEvent::kill() 478{ 479 parent = nullptr; 480 setFlags(AutoDelete); 481} 482 483void 484DmaReadFifo::DmaDoneEvent::cancel() 485{ 486 _canceled = true; 487} 488 489void 490DmaReadFifo::DmaDoneEvent::reset(size_t size) 491{ 492 assert(size <= _data.size()); 493 _done = false; 494 _canceled = false; 495 _requestSize = size; 496} 497 498void 499DmaReadFifo::DmaDoneEvent::process() 500{ 501 if (!parent) 502 return; 503 504 assert(!_done); 505 _done = true; 506 parent->dmaDone(); 507} 508