1/*
2 * Copyright (c) 2012, 2015, 2017 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2006 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Ali Saidi
41 * Nathan Binkert
42 * Andreas Hansson
43 * Andreas Sandberg
44 */
45
46#include "dev/dma_device.hh"
47
48#include <utility>
49
50#include "base/chunk_generator.hh"
51#include "debug/DMA.hh"
52#include "debug/Drain.hh"
53#include "mem/port_proxy.hh"
54#include "sim/system.hh"
55
56DmaPort::DmaPort(MemObject *dev, System *s)
57 : MasterPort(dev->name() + ".dma", dev),
| 1/*
2 * Copyright (c) 2012, 2015, 2017 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2006 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Ali Saidi
41 * Nathan Binkert
42 * Andreas Hansson
43 * Andreas Sandberg
44 */
45
46#include "dev/dma_device.hh"
47
48#include <utility>
49
50#include "base/chunk_generator.hh"
51#include "debug/DMA.hh"
52#include "debug/Drain.hh"
53#include "mem/port_proxy.hh"
54#include "sim/system.hh"
55
56DmaPort::DmaPort(MemObject *dev, System *s)
57 : MasterPort(dev->name() + ".dma", dev),
|
59 sendEvent([this]{ sendDma(); }, dev->name()),
60 pendingCount(0), inRetry(false)
61{ }
62
63void
64DmaPort::handleResp(PacketPtr pkt, Tick delay)
65{
66 // should always see a response with a sender state
67 assert(pkt->isResponse());
68
69 // get the DMA sender state
70 DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
71 assert(state);
72
73 DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \
74 " tot: %d sched %d\n",
75 pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
76 state->numBytes, state->totBytes,
77 state->completionEvent ?
78 state->completionEvent->scheduled() : 0);
79
80 assert(pendingCount != 0);
81 pendingCount--;
82
83 // update the number of bytes received based on the request rather
84 // than the packet as the latter could be rounded up to line sizes
85 state->numBytes += pkt->req->getSize();
86 assert(state->totBytes >= state->numBytes);
87
88 // if we have reached the total number of bytes for this DMA
89 // request, then signal the completion and delete the sate
90 if (state->totBytes == state->numBytes) {
91 if (state->completionEvent) {
92 delay += state->delay;
93 device->schedule(state->completionEvent, curTick() + delay);
94 }
95 delete state;
96 }
97
98 // delete the request that we created and also the packet
99 delete pkt->req;
100 delete pkt;
101
102 // we might be drained at this point, if so signal the drain event
103 if (pendingCount == 0)
104 signalDrainDone();
105}
106
107bool
108DmaPort::recvTimingResp(PacketPtr pkt)
109{
110 // We shouldn't ever get a cacheable block in Modified state
111 assert(pkt->req->isUncacheable() ||
112 !(pkt->cacheResponding() && !pkt->hasSharers()));
113
114 handleResp(pkt);
115
116 return true;
117}
118
119DmaDevice::DmaDevice(const Params *p)
120 : PioDevice(p), dmaPort(this, sys)
121{ }
122
123void
124DmaDevice::init()
125{
126 if (!dmaPort.isConnected())
127 panic("DMA port of %s not connected to anything!", name());
128 PioDevice::init();
129}
130
131DrainState
132DmaPort::drain()
133{
134 if (pendingCount == 0) {
135 return DrainState::Drained;
136 } else {
137 DPRINTF(Drain, "DmaPort not drained\n");
138 return DrainState::Draining;
139 }
140}
141
142void
143DmaPort::recvReqRetry()
144{
145 assert(transmitList.size());
146 trySendTimingReq();
147}
148
149RequestPtr
150DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
151 uint8_t *data, Tick delay, Request::Flags flag)
152{
153 // one DMA request sender state for every action, that is then
154 // split into many requests and packets based on the block size,
155 // i.e. cache line size
156 DmaReqState *reqState = new DmaReqState(event, size, delay);
157
158 // (functionality added for Table Walker statistics)
159 // We're only interested in this when there will only be one request.
160 // For simplicity, we return the last request, which would also be
161 // the only request in that case.
162 RequestPtr req = NULL;
163
164 DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
165 event ? event->scheduled() : -1);
166 for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
167 !gen.done(); gen.next()) {
168 req = new Request(gen.addr(), gen.size(), flag, masterId);
169 req->taskId(ContextSwitchTaskId::DMA);
170 PacketPtr pkt = new Packet(req, cmd);
171
172 // Increment the data pointer on a write
173 if (data)
174 pkt->dataStatic(data + gen.complete());
175
176 pkt->senderState = reqState;
177
178 DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
179 gen.size());
180 queueDma(pkt);
181 }
182
183 // in zero time also initiate the sending of the packets we have
184 // just created, for atomic this involves actually completing all
185 // the requests
186 sendDma();
187
188 return req;
189}
190
191void
192DmaPort::queueDma(PacketPtr pkt)
193{
194 transmitList.push_back(pkt);
195
196 // remember that we have another packet pending, this will only be
197 // decremented once a response comes back
198 pendingCount++;
199}
200
201void
202DmaPort::trySendTimingReq()
203{
204 // send the first packet on the transmit list and schedule the
205 // following send if it is successful
206 PacketPtr pkt = transmitList.front();
207
208 DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
209 pkt->getAddr());
210
211 inRetry = !sendTimingReq(pkt);
212 if (!inRetry) {
213 transmitList.pop_front();
214 DPRINTF(DMA, "-- Done\n");
215 // if there is more to do, then do so
216 if (!transmitList.empty())
217 // this should ultimately wait for as many cycles as the
218 // device needs to send the packet, but currently the port
219 // does not have any known width so simply wait a single
220 // cycle
221 device->schedule(sendEvent, device->clockEdge(Cycles(1)));
222 } else {
223 DPRINTF(DMA, "-- Failed, waiting for retry\n");
224 }
225
226 DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
227 transmitList.size(), inRetry);
228}
229
230void
231DmaPort::sendDma()
232{
233 // some kind of selcetion between access methods
234 // more work is going to have to be done to make
235 // switching actually work
236 assert(transmitList.size());
237
238 if (sys->isTimingMode()) {
239 // if we are either waiting for a retry or are still waiting
240 // after sending the last packet, then do not proceed
241 if (inRetry || sendEvent.scheduled()) {
242 DPRINTF(DMA, "Can't send immediately, waiting to send\n");
243 return;
244 }
245
246 trySendTimingReq();
247 } else if (sys->isAtomicMode()) {
248 // send everything there is to send in zero time
249 while (!transmitList.empty()) {
250 PacketPtr pkt = transmitList.front();
251 transmitList.pop_front();
252
253 DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
254 pkt->req->getPaddr(), pkt->req->getSize());
255 Tick lat = sendAtomic(pkt);
256
257 handleResp(pkt, lat);
258 }
259 } else
260 panic("Unknown memory mode.");
261}
262
263BaseMasterPort &
264DmaDevice::getMasterPort(const std::string &if_name, PortID idx)
265{
266 if (if_name == "dma") {
267 return dmaPort;
268 }
269 return PioDevice::getMasterPort(if_name, idx);
270}
271
272
273
274
275
276DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
277 unsigned max_req_size,
278 unsigned max_pending,
279 Request::Flags flags)
280 : maxReqSize(max_req_size), fifoSize(size),
281 reqFlags(flags), port(_port),
282 buffer(size),
283 nextAddr(0), endAddr(0)
284{
285 freeRequests.resize(max_pending);
286 for (auto &e : freeRequests)
287 e.reset(new DmaDoneEvent(this, max_req_size));
288
289}
290
291DmaReadFifo::~DmaReadFifo()
292{
293 for (auto &p : pendingRequests) {
294 DmaDoneEvent *e(p.release());
295
296 if (e->done()) {
297 delete e;
298 } else {
299 // We can't kill in-flight DMAs, so we'll just transfer
300 // ownership to the event queue so that they get freed
301 // when they are done.
302 e->kill();
303 }
304 }
305}
306
307void
308DmaReadFifo::serialize(CheckpointOut &cp) const
309{
310 assert(pendingRequests.empty());
311
312 SERIALIZE_CONTAINER(buffer);
313 SERIALIZE_SCALAR(endAddr);
314 SERIALIZE_SCALAR(nextAddr);
315}
316
317void
318DmaReadFifo::unserialize(CheckpointIn &cp)
319{
320 UNSERIALIZE_CONTAINER(buffer);
321 UNSERIALIZE_SCALAR(endAddr);
322 UNSERIALIZE_SCALAR(nextAddr);
323}
324
325bool
326DmaReadFifo::tryGet(uint8_t *dst, size_t len)
327{
328 if (buffer.size() >= len) {
329 buffer.read(dst, len);
330 resumeFill();
331 return true;
332 } else {
333 return false;
334 }
335}
336
337void
338DmaReadFifo::get(uint8_t *dst, size_t len)
339{
340 const bool success(tryGet(dst, len));
341 panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
342}
343
344void
345DmaReadFifo::startFill(Addr start, size_t size)
346{
347 assert(atEndOfBlock());
348
349 nextAddr = start;
350 endAddr = start + size;
351 resumeFill();
352}
353
354void
355DmaReadFifo::stopFill()
356{
357 // Prevent new DMA requests by setting the next address to the end
358 // address. Pending requests will still complete.
359 nextAddr = endAddr;
360
361 // Flag in-flight accesses as canceled. This prevents their data
362 // from being written to the FIFO.
363 for (auto &p : pendingRequests)
364 p->cancel();
365}
366
367void
368DmaReadFifo::resumeFill()
369{
370 // Don't try to fetch more data if we are draining. This ensures
371 // that the DMA engine settles down before we checkpoint it.
372 if (drainState() == DrainState::Draining)
373 return;
374
375 const bool old_eob(atEndOfBlock());
376
377 if (port.sys->bypassCaches())
378 resumeFillFunctional();
379 else
380 resumeFillTiming();
381
382 if (!old_eob && atEndOfBlock())
383 onEndOfBlock();
384}
385
386void
387DmaReadFifo::resumeFillFunctional()
388{
389 const size_t fifo_space = buffer.capacity() - buffer.size();
390 const size_t kvm_watermark = port.sys->cacheLineSize();
391 if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
392 const size_t block_remaining = endAddr - nextAddr;
393 const size_t xfer_size = std::min(fifo_space, block_remaining);
394 std::vector<uint8_t> tmp_buffer(xfer_size);
395
396 assert(pendingRequests.empty());
397 DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
398 "fifo_space=%#x block_remaining=%#x\n",
399 nextAddr, xfer_size, fifo_space, block_remaining);
400
401 port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
402 buffer.write(tmp_buffer.begin(), xfer_size);
403 nextAddr += xfer_size;
404 }
405}
406
407void
408DmaReadFifo::resumeFillTiming()
409{
410 size_t size_pending(0);
411 for (auto &e : pendingRequests)
412 size_pending += e->requestSize();
413
414 while (!freeRequests.empty() && !atEndOfBlock()) {
415 const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
416 if (buffer.size() + size_pending + req_size > fifoSize)
417 break;
418
419 DmaDoneEventUPtr event(std::move(freeRequests.front()));
420 freeRequests.pop_front();
421 assert(event);
422
423 event->reset(req_size);
424 port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
425 event->data(), 0, reqFlags);
426 nextAddr += req_size;
427 size_pending += req_size;
428
429 pendingRequests.emplace_back(std::move(event));
430 }
431}
432
433void
434DmaReadFifo::dmaDone()
435{
436 const bool old_active(isActive());
437
438 handlePending();
439 resumeFill();
440
441 if (old_active && !isActive())
442 onIdle();
443}
444
445void
446DmaReadFifo::handlePending()
447{
448 while (!pendingRequests.empty() && pendingRequests.front()->done()) {
449 // Get the first finished pending request
450 DmaDoneEventUPtr event(std::move(pendingRequests.front()));
451 pendingRequests.pop_front();
452
453 if (!event->canceled())
454 buffer.write(event->data(), event->requestSize());
455
456 // Move the event to the list of free requests
457 freeRequests.emplace_back(std::move(event));
458 }
459
460 if (pendingRequests.empty())
461 signalDrainDone();
462}
463
464DrainState
465DmaReadFifo::drain()
466{
467 return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
468}
469
470
471DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
472 size_t max_size)
473 : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
474{
475}
476
477void
478DmaReadFifo::DmaDoneEvent::kill()
479{
480 parent = nullptr;
481 setFlags(AutoDelete);
482}
483
484void
485DmaReadFifo::DmaDoneEvent::cancel()
486{
487 _canceled = true;
488}
489
490void
491DmaReadFifo::DmaDoneEvent::reset(size_t size)
492{
493 assert(size <= _data.size());
494 _done = false;
495 _canceled = false;
496 _requestSize = size;
497}
498
499void
500DmaReadFifo::DmaDoneEvent::process()
501{
502 if (!parent)
503 return;
504
505 assert(!_done);
506 _done = true;
507 parent->dmaDone();
508}
| 59 sendEvent([this]{ sendDma(); }, dev->name()),
60 pendingCount(0), inRetry(false)
61{ }
62
63void
64DmaPort::handleResp(PacketPtr pkt, Tick delay)
65{
66 // should always see a response with a sender state
67 assert(pkt->isResponse());
68
69 // get the DMA sender state
70 DmaReqState *state = dynamic_cast<DmaReqState*>(pkt->senderState);
71 assert(state);
72
73 DPRINTF(DMA, "Received response %s for addr: %#x size: %d nb: %d," \
74 " tot: %d sched %d\n",
75 pkt->cmdString(), pkt->getAddr(), pkt->req->getSize(),
76 state->numBytes, state->totBytes,
77 state->completionEvent ?
78 state->completionEvent->scheduled() : 0);
79
80 assert(pendingCount != 0);
81 pendingCount--;
82
83 // update the number of bytes received based on the request rather
84 // than the packet as the latter could be rounded up to line sizes
85 state->numBytes += pkt->req->getSize();
86 assert(state->totBytes >= state->numBytes);
87
88 // if we have reached the total number of bytes for this DMA
89 // request, then signal the completion and delete the sate
90 if (state->totBytes == state->numBytes) {
91 if (state->completionEvent) {
92 delay += state->delay;
93 device->schedule(state->completionEvent, curTick() + delay);
94 }
95 delete state;
96 }
97
98 // delete the request that we created and also the packet
99 delete pkt->req;
100 delete pkt;
101
102 // we might be drained at this point, if so signal the drain event
103 if (pendingCount == 0)
104 signalDrainDone();
105}
106
107bool
108DmaPort::recvTimingResp(PacketPtr pkt)
109{
110 // We shouldn't ever get a cacheable block in Modified state
111 assert(pkt->req->isUncacheable() ||
112 !(pkt->cacheResponding() && !pkt->hasSharers()));
113
114 handleResp(pkt);
115
116 return true;
117}
118
119DmaDevice::DmaDevice(const Params *p)
120 : PioDevice(p), dmaPort(this, sys)
121{ }
122
123void
124DmaDevice::init()
125{
126 if (!dmaPort.isConnected())
127 panic("DMA port of %s not connected to anything!", name());
128 PioDevice::init();
129}
130
131DrainState
132DmaPort::drain()
133{
134 if (pendingCount == 0) {
135 return DrainState::Drained;
136 } else {
137 DPRINTF(Drain, "DmaPort not drained\n");
138 return DrainState::Draining;
139 }
140}
141
142void
143DmaPort::recvReqRetry()
144{
145 assert(transmitList.size());
146 trySendTimingReq();
147}
148
149RequestPtr
150DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
151 uint8_t *data, Tick delay, Request::Flags flag)
152{
153 // one DMA request sender state for every action, that is then
154 // split into many requests and packets based on the block size,
155 // i.e. cache line size
156 DmaReqState *reqState = new DmaReqState(event, size, delay);
157
158 // (functionality added for Table Walker statistics)
159 // We're only interested in this when there will only be one request.
160 // For simplicity, we return the last request, which would also be
161 // the only request in that case.
162 RequestPtr req = NULL;
163
164 DPRINTF(DMA, "Starting DMA for addr: %#x size: %d sched: %d\n", addr, size,
165 event ? event->scheduled() : -1);
166 for (ChunkGenerator gen(addr, size, sys->cacheLineSize());
167 !gen.done(); gen.next()) {
168 req = new Request(gen.addr(), gen.size(), flag, masterId);
169 req->taskId(ContextSwitchTaskId::DMA);
170 PacketPtr pkt = new Packet(req, cmd);
171
172 // Increment the data pointer on a write
173 if (data)
174 pkt->dataStatic(data + gen.complete());
175
176 pkt->senderState = reqState;
177
178 DPRINTF(DMA, "--Queuing DMA for addr: %#x size: %d\n", gen.addr(),
179 gen.size());
180 queueDma(pkt);
181 }
182
183 // in zero time also initiate the sending of the packets we have
184 // just created, for atomic this involves actually completing all
185 // the requests
186 sendDma();
187
188 return req;
189}
190
191void
192DmaPort::queueDma(PacketPtr pkt)
193{
194 transmitList.push_back(pkt);
195
196 // remember that we have another packet pending, this will only be
197 // decremented once a response comes back
198 pendingCount++;
199}
200
201void
202DmaPort::trySendTimingReq()
203{
204 // send the first packet on the transmit list and schedule the
205 // following send if it is successful
206 PacketPtr pkt = transmitList.front();
207
208 DPRINTF(DMA, "Trying to send %s addr %#x\n", pkt->cmdString(),
209 pkt->getAddr());
210
211 inRetry = !sendTimingReq(pkt);
212 if (!inRetry) {
213 transmitList.pop_front();
214 DPRINTF(DMA, "-- Done\n");
215 // if there is more to do, then do so
216 if (!transmitList.empty())
217 // this should ultimately wait for as many cycles as the
218 // device needs to send the packet, but currently the port
219 // does not have any known width so simply wait a single
220 // cycle
221 device->schedule(sendEvent, device->clockEdge(Cycles(1)));
222 } else {
223 DPRINTF(DMA, "-- Failed, waiting for retry\n");
224 }
225
226 DPRINTF(DMA, "TransmitList: %d, inRetry: %d\n",
227 transmitList.size(), inRetry);
228}
229
230void
231DmaPort::sendDma()
232{
233 // some kind of selcetion between access methods
234 // more work is going to have to be done to make
235 // switching actually work
236 assert(transmitList.size());
237
238 if (sys->isTimingMode()) {
239 // if we are either waiting for a retry or are still waiting
240 // after sending the last packet, then do not proceed
241 if (inRetry || sendEvent.scheduled()) {
242 DPRINTF(DMA, "Can't send immediately, waiting to send\n");
243 return;
244 }
245
246 trySendTimingReq();
247 } else if (sys->isAtomicMode()) {
248 // send everything there is to send in zero time
249 while (!transmitList.empty()) {
250 PacketPtr pkt = transmitList.front();
251 transmitList.pop_front();
252
253 DPRINTF(DMA, "Sending DMA for addr: %#x size: %d\n",
254 pkt->req->getPaddr(), pkt->req->getSize());
255 Tick lat = sendAtomic(pkt);
256
257 handleResp(pkt, lat);
258 }
259 } else
260 panic("Unknown memory mode.");
261}
262
263BaseMasterPort &
264DmaDevice::getMasterPort(const std::string &if_name, PortID idx)
265{
266 if (if_name == "dma") {
267 return dmaPort;
268 }
269 return PioDevice::getMasterPort(if_name, idx);
270}
271
272
273
274
275
276DmaReadFifo::DmaReadFifo(DmaPort &_port, size_t size,
277 unsigned max_req_size,
278 unsigned max_pending,
279 Request::Flags flags)
280 : maxReqSize(max_req_size), fifoSize(size),
281 reqFlags(flags), port(_port),
282 buffer(size),
283 nextAddr(0), endAddr(0)
284{
285 freeRequests.resize(max_pending);
286 for (auto &e : freeRequests)
287 e.reset(new DmaDoneEvent(this, max_req_size));
288
289}
290
291DmaReadFifo::~DmaReadFifo()
292{
293 for (auto &p : pendingRequests) {
294 DmaDoneEvent *e(p.release());
295
296 if (e->done()) {
297 delete e;
298 } else {
299 // We can't kill in-flight DMAs, so we'll just transfer
300 // ownership to the event queue so that they get freed
301 // when they are done.
302 e->kill();
303 }
304 }
305}
306
307void
308DmaReadFifo::serialize(CheckpointOut &cp) const
309{
310 assert(pendingRequests.empty());
311
312 SERIALIZE_CONTAINER(buffer);
313 SERIALIZE_SCALAR(endAddr);
314 SERIALIZE_SCALAR(nextAddr);
315}
316
317void
318DmaReadFifo::unserialize(CheckpointIn &cp)
319{
320 UNSERIALIZE_CONTAINER(buffer);
321 UNSERIALIZE_SCALAR(endAddr);
322 UNSERIALIZE_SCALAR(nextAddr);
323}
324
325bool
326DmaReadFifo::tryGet(uint8_t *dst, size_t len)
327{
328 if (buffer.size() >= len) {
329 buffer.read(dst, len);
330 resumeFill();
331 return true;
332 } else {
333 return false;
334 }
335}
336
337void
338DmaReadFifo::get(uint8_t *dst, size_t len)
339{
340 const bool success(tryGet(dst, len));
341 panic_if(!success, "Buffer underrun in DmaReadFifo::get()\n");
342}
343
344void
345DmaReadFifo::startFill(Addr start, size_t size)
346{
347 assert(atEndOfBlock());
348
349 nextAddr = start;
350 endAddr = start + size;
351 resumeFill();
352}
353
354void
355DmaReadFifo::stopFill()
356{
357 // Prevent new DMA requests by setting the next address to the end
358 // address. Pending requests will still complete.
359 nextAddr = endAddr;
360
361 // Flag in-flight accesses as canceled. This prevents their data
362 // from being written to the FIFO.
363 for (auto &p : pendingRequests)
364 p->cancel();
365}
366
367void
368DmaReadFifo::resumeFill()
369{
370 // Don't try to fetch more data if we are draining. This ensures
371 // that the DMA engine settles down before we checkpoint it.
372 if (drainState() == DrainState::Draining)
373 return;
374
375 const bool old_eob(atEndOfBlock());
376
377 if (port.sys->bypassCaches())
378 resumeFillFunctional();
379 else
380 resumeFillTiming();
381
382 if (!old_eob && atEndOfBlock())
383 onEndOfBlock();
384}
385
386void
387DmaReadFifo::resumeFillFunctional()
388{
389 const size_t fifo_space = buffer.capacity() - buffer.size();
390 const size_t kvm_watermark = port.sys->cacheLineSize();
391 if (fifo_space >= kvm_watermark || buffer.capacity() < kvm_watermark) {
392 const size_t block_remaining = endAddr - nextAddr;
393 const size_t xfer_size = std::min(fifo_space, block_remaining);
394 std::vector<uint8_t> tmp_buffer(xfer_size);
395
396 assert(pendingRequests.empty());
397 DPRINTF(DMA, "KVM Bypassing startAddr=%#x xfer_size=%#x " \
398 "fifo_space=%#x block_remaining=%#x\n",
399 nextAddr, xfer_size, fifo_space, block_remaining);
400
401 port.sys->physProxy.readBlob(nextAddr, tmp_buffer.data(), xfer_size);
402 buffer.write(tmp_buffer.begin(), xfer_size);
403 nextAddr += xfer_size;
404 }
405}
406
407void
408DmaReadFifo::resumeFillTiming()
409{
410 size_t size_pending(0);
411 for (auto &e : pendingRequests)
412 size_pending += e->requestSize();
413
414 while (!freeRequests.empty() && !atEndOfBlock()) {
415 const size_t req_size(std::min(maxReqSize, endAddr - nextAddr));
416 if (buffer.size() + size_pending + req_size > fifoSize)
417 break;
418
419 DmaDoneEventUPtr event(std::move(freeRequests.front()));
420 freeRequests.pop_front();
421 assert(event);
422
423 event->reset(req_size);
424 port.dmaAction(MemCmd::ReadReq, nextAddr, req_size, event.get(),
425 event->data(), 0, reqFlags);
426 nextAddr += req_size;
427 size_pending += req_size;
428
429 pendingRequests.emplace_back(std::move(event));
430 }
431}
432
433void
434DmaReadFifo::dmaDone()
435{
436 const bool old_active(isActive());
437
438 handlePending();
439 resumeFill();
440
441 if (old_active && !isActive())
442 onIdle();
443}
444
445void
446DmaReadFifo::handlePending()
447{
448 while (!pendingRequests.empty() && pendingRequests.front()->done()) {
449 // Get the first finished pending request
450 DmaDoneEventUPtr event(std::move(pendingRequests.front()));
451 pendingRequests.pop_front();
452
453 if (!event->canceled())
454 buffer.write(event->data(), event->requestSize());
455
456 // Move the event to the list of free requests
457 freeRequests.emplace_back(std::move(event));
458 }
459
460 if (pendingRequests.empty())
461 signalDrainDone();
462}
463
464DrainState
465DmaReadFifo::drain()
466{
467 return pendingRequests.empty() ? DrainState::Drained : DrainState::Draining;
468}
469
470
471DmaReadFifo::DmaDoneEvent::DmaDoneEvent(DmaReadFifo *_parent,
472 size_t max_size)
473 : parent(_parent), _done(false), _canceled(false), _data(max_size, 0)
474{
475}
476
477void
478DmaReadFifo::DmaDoneEvent::kill()
479{
480 parent = nullptr;
481 setFlags(AutoDelete);
482}
483
484void
485DmaReadFifo::DmaDoneEvent::cancel()
486{
487 _canceled = true;
488}
489
490void
491DmaReadFifo::DmaDoneEvent::reset(size_t size)
492{
493 assert(size <= _data.size());
494 _done = false;
495 _canceled = false;
496 _requestSize = size;
497}
498
499void
500DmaReadFifo::DmaDoneEvent::process()
501{
502 if (!parent)
503 return;
504
505 assert(!_done);
506 _done = true;
507 parent->dmaDone();
508}
|