1/*
2 * Copyright (c) 2013-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include <iomanip>
41#include <sstream>
42
43#include "arch/locked_mem.hh"
44#include "arch/mmapped_ipr.hh"
45#include "cpu/minor/cpu.hh"
46#include "cpu/minor/exec_context.hh"
47#include "cpu/minor/execute.hh"
48#include "cpu/minor/lsq.hh"
49#include "cpu/minor/pipeline.hh"
50#include "debug/Activity.hh"
51#include "debug/MinorMem.hh"
52
53namespace Minor
54{
55
56/** Returns the offset of addr into an aligned a block of size block_size */
57static Addr
58addrBlockOffset(Addr addr, unsigned int block_size)
59{
60 return addr & (block_size - 1);
61}
62
63/** Returns true if the given [addr .. addr+size-1] transfer needs to be
64 * fragmented across a block size of block_size */
65static bool
66transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
67{
68 return (addrBlockOffset(addr, block_size) + size) > block_size;
69}
70
71LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
72 PacketDataPtr data_, uint64_t *res_) :
73 SenderState(),
74 port(port_),
75 inst(inst_),
76 isLoad(isLoad_),
77 data(data_),
78 packet(NULL),
79 request(),
80 fault(NoFault),
81 res(res_),
82 skipped(false),
83 issuedToMemory(false),
84 state(NotIssued)
85{ }
86
87LSQ::AddrRangeCoverage
88LSQ::LSQRequest::containsAddrRangeOf(
89 Addr req1_addr, unsigned int req1_size,
90 Addr req2_addr, unsigned int req2_size)
91{
92 /* 'end' here means the address of the byte just past the request
93 * blocks */
94 Addr req2_end_addr = req2_addr + req2_size;
95 Addr req1_end_addr = req1_addr + req1_size;
96
97 AddrRangeCoverage ret;
98
99 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
100 ret = NoAddrRangeCoverage;
101 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
102 ret = FullAddrRangeCoverage;
103 else
104 ret = PartialAddrRangeCoverage;
105
106 return ret;
107}
108
109LSQ::AddrRangeCoverage
110LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
111{
112 return containsAddrRangeOf(request.getPaddr(), request.getSize(),
113 other_request->request.getPaddr(), other_request->request.getSize());
114}
115
116bool
117LSQ::LSQRequest::isBarrier()
118{
119 return inst->isInst() && inst->staticInst->isMemBarrier();
120}
121
122bool
123LSQ::LSQRequest::needsToBeSentToStoreBuffer()
124{
125 return state == StoreToStoreBuffer;
126}
127
128void
129LSQ::LSQRequest::setState(LSQRequestState new_state)
130{
131 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
132 " %s\n", state, new_state, *inst);
133 state = new_state;
134}
135
136bool
137LSQ::LSQRequest::isComplete() const
138{
139 /* @todo, There is currently only one 'completed' state. This
140 * may not be a good choice */
141 return state == Complete;
142}
143
144void
145LSQ::LSQRequest::reportData(std::ostream &os) const
146{
147 os << (isLoad ? 'R' : 'W') << ';';
148 inst->reportData(os);
149 os << ';' << state;
150}
151
152std::ostream &
153operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
154{
155 switch (coverage) {
156 case LSQ::PartialAddrRangeCoverage:
157 os << "PartialAddrRangeCoverage";
158 break;
159 case LSQ::FullAddrRangeCoverage:
160 os << "FullAddrRangeCoverage";
161 break;
162 case LSQ::NoAddrRangeCoverage:
163 os << "NoAddrRangeCoverage";
164 break;
165 default:
166 os << "AddrRangeCoverage-" << static_cast<int>(coverage);
167 break;
168 }
169 return os;
170}
171
172std::ostream &
173operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
174{
175 switch (state) {
176 case LSQ::LSQRequest::NotIssued:
177 os << "NotIssued";
178 break;
179 case LSQ::LSQRequest::InTranslation:
180 os << "InTranslation";
181 break;
182 case LSQ::LSQRequest::Translated:
183 os << "Translated";
184 break;
185 case LSQ::LSQRequest::Failed:
186 os << "Failed";
187 break;
188 case LSQ::LSQRequest::RequestIssuing:
189 os << "RequestIssuing";
190 break;
191 case LSQ::LSQRequest::StoreToStoreBuffer:
192 os << "StoreToStoreBuffer";
193 break;
194 case LSQ::LSQRequest::StoreInStoreBuffer:
195 os << "StoreInStoreBuffer";
196 break;
197 case LSQ::LSQRequest::StoreBufferIssuing:
198 os << "StoreBufferIssuing";
199 break;
200 case LSQ::LSQRequest::RequestNeedsRetry:
201 os << "RequestNeedsRetry";
202 break;
203 case LSQ::LSQRequest::StoreBufferNeedsRetry:
204 os << "StoreBufferNeedsRetry";
205 break;
206 case LSQ::LSQRequest::Complete:
207 os << "Complete";
208 break;
209 default:
210 os << "LSQRequestState-" << static_cast<int>(state);
211 break;
212 }
213 return os;
214}
215
216void
217LSQ::clearMemBarrier(MinorDynInstPtr inst)
218{
219 bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier;
220
221 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
222 (is_last_barrier ? "last" : "a"), *inst);
223
224 if (is_last_barrier)
225 lastMemBarrier = 0;
226}
227
228void
229LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_,
230 ThreadContext *tc, BaseTLB::Mode mode)
231{
232 fault = fault_;
233
234 port.numAccessesInDTLB--;
235
236 DPRINTFS(MinorMem, (&port), "Received translation response for"
237 " request: %s\n", *inst);
238
239 makePacket();
240
241 setState(Translated);
242 port.tryToSendToTransfers(this);
243
244 /* Let's try and wake up the processor for the next cycle */
245 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
246}
247
248void
249LSQ::SingleDataRequest::startAddrTranslation()
250{
251 ThreadContext *thread = port.cpu.getContext(
252 inst->id.threadId);
253
254 port.numAccessesInDTLB++;
255
256 setState(LSQ::LSQRequest::InTranslation);
257
258 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
259 /* Submit the translation request. The response will come through
260 * finish/markDelayed on the LSQRequest as it bears the Translation
261 * interface */
262 thread->getDTBPtr()->translateTiming(
263 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
264}
265
266void
267LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
268{
269 DPRINTFS(MinorMem, (&port), "Retiring packet\n");
270 packet = packet_;
271 packetInFlight = false;
272 setState(Complete);
273}
274
275void
276LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_,
277 ThreadContext *tc, BaseTLB::Mode mode)
278{
279 fault = fault_;
280
281 port.numAccessesInDTLB--;
282
283 unsigned int M5_VAR_USED expected_fragment_index =
284 numTranslatedFragments;
285
286 numInTranslationFragments--;
287 numTranslatedFragments++;
288
289 DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
290 " %d of request: %s\n", expected_fragment_index, *inst);
291
292 assert(request_ == fragmentRequests[expected_fragment_index]);
293
294 /* Wake up next cycle to get things going again in case the
295 * tryToSendToTransfers does take */
296 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
297
298 if (fault != NoFault) {
299 /* tryToSendToTransfers will handle the fault */
300
301 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
302 " %d of request: %s\n",
303 expected_fragment_index, *inst);
304
305 setState(Translated);
306 port.tryToSendToTransfers(this);
307 } else if (numTranslatedFragments == numFragments) {
308 makeFragmentPackets();
309
310 setState(Translated);
311 port.tryToSendToTransfers(this);
312 } else {
313 /* Avoid calling translateTiming from within ::finish */
314 assert(!translationEvent.scheduled());
315 port.cpu.schedule(translationEvent, curTick());
316 }
317}
318
319LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
320 bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
321 LSQRequest(port_, inst_, isLoad_, data_, res_),
322 translationEvent(*this),
323 numFragments(0),
324 numInTranslationFragments(0),
325 numTranslatedFragments(0),
326 numIssuedFragments(0),
327 numRetiredFragments(0),
328 fragmentRequests(),
329 fragmentPackets()
330{
331 /* Don't know how many elements are needed until the request is
332 * populated by the caller. */
333}
334
335LSQ::SplitDataRequest::~SplitDataRequest()
336{
337 for (auto i = fragmentRequests.begin();
338 i != fragmentRequests.end(); i++)
339 {
340 delete *i;
341 }
342
343 for (auto i = fragmentPackets.begin();
344 i != fragmentPackets.end(); i++)
345 {
346 delete *i;
347 }
348}
349
350void
351LSQ::SplitDataRequest::makeFragmentRequests()
352{
353 Addr base_addr = request.getVaddr();
354 unsigned int whole_size = request.getSize();
355 unsigned int line_width = port.lineWidth;
356
357 unsigned int fragment_size;
358 Addr fragment_addr;
359
360 /* Assume that this transfer is across potentially many block snap
361 * boundaries:
362 *
363 * | _|________|________|________|___ |
364 * | |0| 1 | 2 | 3 | 4 | |
365 * | |_|________|________|________|___| |
366 * | | | | | |
367 *
368 * The first transfer (0) can be up to lineWidth in size.
369 * All the middle transfers (1-3) are lineWidth in size
370 * The last transfer (4) can be from zero to lineWidth - 1 in size
371 */
372 unsigned int first_fragment_offset =
373 addrBlockOffset(base_addr, line_width);
374 unsigned int last_fragment_size =
375 addrBlockOffset(base_addr + whole_size, line_width);
376 unsigned int first_fragment_size =
377 line_width - first_fragment_offset;
378
379 unsigned int middle_fragments_total_size =
380 whole_size - (first_fragment_size + last_fragment_size);
381
382 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
383
384 unsigned int middle_fragment_count =
385 middle_fragments_total_size / line_width;
386
387 numFragments = 1 /* first */ + middle_fragment_count +
388 (last_fragment_size == 0 ? 0 : 1);
389
390 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
391 " First fragment size: %d Last fragment size: %d\n",
392 numFragments, first_fragment_size,
393 (last_fragment_size == 0 ? line_width : last_fragment_size));
394
395 assert(((middle_fragment_count * line_width) +
396 first_fragment_size + last_fragment_size) == whole_size);
397
398 fragment_addr = base_addr;
399 fragment_size = first_fragment_size;
400
401 /* Just past the last address in the request */
402 Addr end_addr = base_addr + whole_size;
403
404 for (unsigned int fragment_index = 0; fragment_index < numFragments;
405 fragment_index++)
406 {
407 bool M5_VAR_USED is_last_fragment = false;
408
409 if (fragment_addr == base_addr) {
410 /* First fragment */
411 fragment_size = first_fragment_size;
412 } else {
413 if ((fragment_addr + line_width) > end_addr) {
414 /* Adjust size of last fragment */
415 fragment_size = end_addr - fragment_addr;
416 is_last_fragment = true;
417 } else {
418 /* Middle fragments */
419 fragment_size = line_width;
420 }
421 }
422
423 Request *fragment = new Request();
424
425 fragment->setThreadContext(request.contextId(), /* thread id */ 0);
426 fragment->setVirt(0 /* asid */,
427 fragment_addr, fragment_size, request.getFlags(),
428 request.masterId(),
429 request.getPC());
430
431 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
432 " (whole request addr: 0x%x size: %d) %s\n",
433 fragment_addr, fragment_size, base_addr, whole_size,
434 (is_last_fragment ? "last fragment" : ""));
435
436 fragment_addr += fragment_size;
437
438 fragmentRequests.push_back(fragment);
439 }
440}
441
442void
443LSQ::SplitDataRequest::makeFragmentPackets()
444{
445 Addr base_addr = request.getVaddr();
446
447 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
448
449 for (unsigned int fragment_index = 0; fragment_index < numFragments;
450 fragment_index++)
451 {
452 Request *fragment = fragmentRequests[fragment_index];
453
454 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
455 " (%d, 0x%x)\n",
456 fragment_index, *inst,
457 (fragment->hasPaddr() ? "has paddr" : "no paddr"),
458 (fragment->hasPaddr() ? fragment->getPaddr() : 0));
459
460 Addr fragment_addr = fragment->getVaddr();
461 unsigned int fragment_size = fragment->getSize();
462
463 uint8_t *request_data = NULL;
464
465 if (!isLoad) {
466 /* Split data for Packets. Will become the property of the
467 * outgoing Packets */
468 request_data = new uint8_t[fragment_size];
469 std::memcpy(request_data, data + (fragment_addr - base_addr),
470 fragment_size);
471 }
472
473 assert(fragment->hasPaddr());
474
475 PacketPtr fragment_packet =
476 makePacketForRequest(*fragment, isLoad, this, request_data);
477
478 fragmentPackets.push_back(fragment_packet);
479 /* Accumulate flags in parent request */
480 request.setFlags(fragment->getFlags());
481 }
482
483 /* Might as well make the overall/response packet here */
484 /* Get the physical address for the whole request/packet from the first
485 * fragment */
486 request.setPaddr(fragmentRequests[0]->getPaddr());
487 makePacket();
488}
489
490void
491LSQ::SplitDataRequest::startAddrTranslation()
492{
493 setState(LSQ::LSQRequest::InTranslation);
494
495 makeFragmentRequests();
496
497 numInTranslationFragments = 0;
498 numTranslatedFragments = 0;
499
500 /* @todo, just do these in sequence for now with
501 * a loop of:
502 * do {
503 * sendNextFragmentToTranslation ; translateTiming ; finish
504 * } while (numTranslatedFragments != numFragments);
505 */
506
507 /* Do first translation */
508 sendNextFragmentToTranslation();
509}
510
511PacketPtr
512LSQ::SplitDataRequest::getHeadPacket()
513{
514 assert(numIssuedFragments < numFragments);
515
516 return fragmentPackets[numIssuedFragments];
517}
518
519void
520LSQ::SplitDataRequest::stepToNextPacket()
521{
522 assert(numIssuedFragments < numFragments);
523
524 numIssuedFragments++;
525}
526
527void
528LSQ::SplitDataRequest::retireResponse(PacketPtr response)
529{
530 assert(numRetiredFragments < numFragments);
531
532 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
533 " offset: 0x%x (retired fragment num: %d) %s\n",
534 response->req->getVaddr(), response->req->getSize(),
535 request.getVaddr() - response->req->getVaddr(),
536 numRetiredFragments,
537 (fault == NoFault ? "" : fault->name()));
538
539 numRetiredFragments++;
540
541 if (skipped) {
542 /* Skip because we already knew the request had faulted or been
543 * skipped */
544 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
545 } else if (response->isError()) {
546 /* Mark up the error and leave to execute to handle it */
547 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
548 setSkipped();
549 packet->copyError(response);
550 } else {
551 if (isLoad) {
552 if (!data) {
553 /* For a split transfer, a Packet must be constructed
554 * to contain all returning data. This is that packet's
555 * data */
556 data = new uint8_t[request.getSize()];
557 }
558
559 /* Populate the portion of the overall response data represented
560 * by the response fragment */
561 std::memcpy(
562 data + (response->req->getVaddr() - request.getVaddr()),
563 response->getPtr<uint8_t>(),
564 response->req->getSize());
565 }
566 }
567
568 /* Complete early if we're skipping are no more in-flight accesses */
569 if (skipped && !hasPacketsInMemSystem()) {
570 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
571 setState(Complete);
572 if (packet->needsResponse())
573 packet->makeResponse();
574 }
575
576 if (numRetiredFragments == numFragments)
577 setState(Complete);
578
579 if (!skipped && isComplete()) {
580 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
581
582 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
583 " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
584 " %s\n", packet->isRead(), packet->isWrite(),
585 packet->needsResponse(), packet->getSize(), request.getSize(),
586 response->getSize());
587
588 /* A request can become complete by several paths, this is a sanity
589 * check to make sure the packet's data is created */
590 if (!data) {
591 data = new uint8_t[request.getSize()];
592 }
593
594 if (isLoad) {
595 DPRINTFS(MinorMem, (&port), "Copying read data\n");
596 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
597 }
598 packet->makeResponse();
599 }
600
601 /* Packets are all deallocated together in ~SplitLSQRequest */
602}
603
604void
605LSQ::SplitDataRequest::sendNextFragmentToTranslation()
606{
607 unsigned int fragment_index = numTranslatedFragments;
608
609 ThreadContext *thread = port.cpu.getContext(
610 inst->id.threadId);
611
612 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
613 fragment_index);
614
615 port.numAccessesInDTLB++;
616 numInTranslationFragments++;
617
618 thread->getDTBPtr()->translateTiming(
619 fragmentRequests[fragment_index], thread, this, (isLoad ?
620 BaseTLB::Read : BaseTLB::Write));
621}
622
623bool
624LSQ::StoreBuffer::canInsert() const
625{
626 /* @todo, support store amalgamation */
627 return slots.size() < numSlots;
628}
629
630void
631LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
632{
633 auto found = std::find(slots.begin(), slots.end(), request);
634
635 if (found != slots.end()) {
636 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
637 request, *found, *(request->inst));
638 slots.erase(found);
639
640 delete request;
641 }
642}
643
644void
645LSQ::StoreBuffer::insert(LSQRequestPtr request)
646{
647 if (!canInsert()) {
648 warn("%s: store buffer insertion without space to insert from"
649 " inst: %s\n", name(), *(request->inst));
650 }
651
652 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
653
654 numUnissuedAccesses++;
655
656 if (request->state != LSQRequest::Complete)
657 request->setState(LSQRequest::StoreInStoreBuffer);
658
659 slots.push_back(request);
660
661 /* Let's try and wake up the processor for the next cycle to step
662 * the store buffer */
663 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
664}
665
666LSQ::AddrRangeCoverage
667LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
668 unsigned int &found_slot)
669{
670 unsigned int slot_index = slots.size() - 1;
671 auto i = slots.rbegin();
672 AddrRangeCoverage ret = NoAddrRangeCoverage;
673
674 /* Traverse the store buffer in reverse order (most to least recent)
675 * and try to find a slot whose address range overlaps this request */
676 while (ret == NoAddrRangeCoverage && i != slots.rend()) {
677 LSQRequestPtr slot = *i;
678
679 if (slot->packet) {
680 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
681
682 if (coverage != NoAddrRangeCoverage) {
683 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
684 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
685 slot_index, coverage,
686 request->request.getPaddr(), request->request.getSize(),
687 slot->request.getPaddr(), slot->request.getSize());
688
689 found_slot = slot_index;
690 ret = coverage;
691 }
692 }
693
694 i++;
695 slot_index--;
696 }
697
698 return ret;
699}
700
701/** Fill the given packet with appropriate date from slot slot_number */
702void
703LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
704 unsigned int slot_number)
705{
706 assert(slot_number < slots.size());
707 assert(load->packet);
708 assert(load->isLoad);
709
710 LSQRequestPtr store = slots[slot_number];
711
712 assert(store->packet);
713 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
714
715 Addr load_addr = load->request.getPaddr();
716 Addr store_addr = store->request.getPaddr();
717 Addr addr_offset = load_addr - store_addr;
718
719 unsigned int load_size = load->request.getSize();
720
721 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
722 " slot: %d addr: 0x%x addressOffset: 0x%x\n",
723 load_size, load_addr, slot_number,
724 store_addr, addr_offset);
725
726 void *load_packet_data = load->packet->getPtr<void>();
727 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
728
729 std::memcpy(load_packet_data, store_packet_data, load_size);
730}
731
732void
733LSQ::StoreBuffer::step()
734{
735 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
736 numUnissuedAccesses);
737
738 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
739 /* Clear all the leading barriers */
740 while (!slots.empty() &&
741 slots.front()->isComplete() && slots.front()->isBarrier())
742 {
743 LSQRequestPtr barrier = slots.front();
744
745 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
746 *(barrier->inst));
747
748 numUnissuedAccesses--;
749 lsq.clearMemBarrier(barrier->inst);
750 slots.pop_front();
751
752 delete barrier;
753 }
754
755 auto i = slots.begin();
756 bool issued = true;
757 unsigned int issue_count = 0;
758
759 /* Skip trying if the memory system is busy */
760 if (lsq.state == LSQ::MemoryNeedsRetry)
761 issued = false;
762
763 /* Try to issue all stores in order starting from the head
764 * of the queue. Responses are allowed to be retired
765 * out of order */
766 while (issued &&
767 issue_count < storeLimitPerCycle &&
768 lsq.canSendToMemorySystem() &&
769 i != slots.end())
770 {
771 LSQRequestPtr request = *i;
772
773 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
774 " state: %s\n",
775 *(request->inst), request->sentAllPackets(),
776 request->state);
777
778 if (request->isBarrier() && request->isComplete()) {
779 /* Give up at barriers */
780 issued = false;
781 } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
782 request->sentAllPackets()))
783 {
784 DPRINTF(MinorMem, "Trying to send request: %s to memory"
785 " system\n", *(request->inst));
786
787 if (lsq.tryToSend(request)) {
788 /* Barrier are accounted for as they are cleared from
789 * the queue, not after their transfers are complete */
790 if (!request->isBarrier())
791 numUnissuedAccesses--;
792 issue_count++;
793 } else {
794 /* Don't step on to the next store buffer entry if this
795 * one hasn't issued all its packets as the store
796 * buffer must still enforce ordering */
797 issued = false;
798 }
799 }
800 i++;
801 }
802 }
803}
804
805void
806LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
807 bool committed)
808{
809 if (committed) {
810 /* Not already sent to the store buffer as a store request? */
811 if (!inst->inStoreBuffer) {
812 /* Insert an entry into the store buffer to tick off barriers
813 * until there are none in flight */
814 storeBuffer.insert(new BarrierDataRequest(*this, inst));
815 }
816 } else {
817 /* Clear the barrier anyway if it wasn't actually committed */
818 clearMemBarrier(inst);
819 }
820}
821
822void
823LSQ::StoreBuffer::minorTrace() const
824{
825 unsigned int size = slots.size();
826 unsigned int i = 0;
827 std::ostringstream os;
828
829 while (i < size) {
830 LSQRequestPtr request = slots[i];
831
832 request->reportData(os);
833
834 i++;
835 if (i < numSlots)
836 os << ',';
837 }
838
839 while (i < numSlots) {
840 os << '-';
841
842 i++;
843 if (i < numSlots)
844 os << ',';
845 }
846
847 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
848 numUnissuedAccesses);
849}
850
851void
852LSQ::tryToSendToTransfers(LSQRequestPtr request)
853{
854 if (state == MemoryNeedsRetry) {
855 DPRINTF(MinorMem, "Request needs retry, not issuing to"
856 " memory until retry arrives\n");
857 return;
858 }
859
860 if (request->state == LSQRequest::InTranslation) {
861 DPRINTF(MinorMem, "Request still in translation, not issuing to"
862 " memory\n");
863 return;
864 }
865
866 assert(request->state == LSQRequest::Translated ||
867 request->state == LSQRequest::RequestIssuing ||
868 request->state == LSQRequest::Failed ||
869 request->state == LSQRequest::Complete);
870
871 if (requests.empty() || requests.front() != request) {
872 DPRINTF(MinorMem, "Request not at front of requests queue, can't"
873 " issue to memory\n");
874 return;
875 }
876
877 if (transfers.unreservedRemainingSpace() == 0) {
878 DPRINTF(MinorMem, "No space to insert request into transfers"
879 " queue\n");
880 return;
881 }
882
883 if (request->isComplete() || request->state == LSQRequest::Failed) {
884 DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
885 " queue\n", (request->isComplete() ? "completed" : "failed"));
886 request->setState(LSQRequest::Complete);
887 request->setSkipped();
888 moveFromRequestsToTransfers(request);
889 return;
890 }
891
892 if (!execute.instIsRightStream(request->inst)) {
893 /* Wrong stream, try to abort the transfer but only do so if
894 * there are no packets in flight */
895 if (request->hasPacketsInMemSystem()) {
896 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
897 " waiting for responses before aborting request\n");
898 } else {
899 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
900 " aborting request\n");
901 request->setState(LSQRequest::Complete);
902 request->setSkipped();
903 moveFromRequestsToTransfers(request);
904 }
905 return;
906 }
907
908 if (request->fault != NoFault) {
909 if (request->inst->staticInst->isPrefetch()) {
910 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
911 }
912 DPRINTF(MinorMem, "Moving faulting request into the transfers"
913 " queue\n");
914 request->setState(LSQRequest::Complete);
915 request->setSkipped();
916 moveFromRequestsToTransfers(request);
917 return;
918 }
919
920 bool is_load = request->isLoad;
921 bool is_llsc = request->request.isLLSC();
922 bool is_swap = request->request.isSwap();
923 bool bufferable = !(request->request.isUncacheable() ||
924 is_llsc || is_swap);
925
926 if (is_load) {
927 if (numStoresInTransfers != 0) {
928 DPRINTF(MinorMem, "Load request with stores still in transfers"
929 " queue, stalling\n");
930 return;
931 }
932 } else {
933 /* Store. Can it be sent to the store buffer? */
934 if (bufferable && !request->request.isMmappedIpr()) {
935 request->setState(LSQRequest::StoreToStoreBuffer);
936 moveFromRequestsToTransfers(request);
937 DPRINTF(MinorMem, "Moving store into transfers queue\n");
938 return;
939 }
940 }
941
942 /* Check if this is the head instruction (and so must be executable as
943 * its stream sequence number was checked above) for loads which must
944 * not be speculatively issued and stores which must be issued here */
945 if (!bufferable) {
946 if (!execute.instIsHeadInst(request->inst)) {
947 DPRINTF(MinorMem, "Memory access not the head inst., can't be"
948 " sure it can be performed, not issuing\n");
949 return;
950 }
951
952 unsigned int forwarding_slot = 0;
953
954 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
955 NoAddrRangeCoverage)
956 {
957 DPRINTF(MinorMem, "Memory access can receive forwarded data"
958 " from the store buffer, need to wait for store buffer to"
959 " drain\n");
960 return;
961 }
962 }
963
964 /* True: submit this packet to the transfers queue to be sent to the
965 * memory system.
966 * False: skip the memory and push a packet for this request onto
967 * requests */
968 bool do_access = true;
969
970 if (!is_llsc) {
971 /* Check for match in the store buffer */
972 if (is_load) {
973 unsigned int forwarding_slot = 0;
974 AddrRangeCoverage forwarding_result =
975 storeBuffer.canForwardDataToLoad(request,
976 forwarding_slot);
977
978 switch (forwarding_result) {
979 case FullAddrRangeCoverage:
980 /* Forward data from the store buffer into this request and
981 * repurpose this request's packet into a response packet */
982 storeBuffer.forwardStoreData(request, forwarding_slot);
983 request->packet->makeResponse();
984
985 /* Just move between queues, no access */
986 do_access = false;
987 break;
988 case PartialAddrRangeCoverage:
989 DPRINTF(MinorMem, "Load partly satisfied by store buffer"
990 " data. Must wait for the store to complete\n");
991 return;
992 break;
993 case NoAddrRangeCoverage:
994 DPRINTF(MinorMem, "No forwardable data from store buffer\n");
995 /* Fall through to try access */
996 break;
997 }
998 }
999 } else {
1000 if (!canSendToMemorySystem()) {
1001 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1002 return;
1003 }
1004
1005 SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1006
1007 TheISA::PCState old_pc = thread.pcState();
1008 ExecContext context(cpu, thread, execute, request->inst);
1009
1010 /* Handle LLSC requests and tests */
1011 if (is_load) {
1012 TheISA::handleLockedRead(&context, &request->request);
1013 } else {
1014 do_access = TheISA::handleLockedWrite(&context,
1015 &request->request, cacheBlockMask);
1016
1017 if (!do_access) {
1018 DPRINTF(MinorMem, "Not perfoming a memory "
1019 "access for store conditional\n");
1020 }
1021 }
1022 thread.pcState(old_pc);
1023 }
1024
1025 /* See the do_access comment above */
1026 if (do_access) {
1027 if (!canSendToMemorySystem()) {
1028 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1029 return;
1030 }
1031
1032 /* Remember if this is an access which can't be idly
1033 * discarded by an interrupt */
1034 if (!bufferable && !request->issuedToMemory) {
1035 numAccessesIssuedToMemory++;
1036 request->issuedToMemory = true;
1037 }
1038
1039 if (tryToSend(request))
1040 moveFromRequestsToTransfers(request);
1041 } else {
1042 request->setState(LSQRequest::Complete);
1043 moveFromRequestsToTransfers(request);
1044 }
1045}
1046
1047bool
1048LSQ::tryToSend(LSQRequestPtr request)
1049{
1050 bool ret = false;
1051
1052 if (!canSendToMemorySystem()) {
1053 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1054 *(request->inst));
1055 } else {
1056 PacketPtr packet = request->getHeadPacket();
1057
1058 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1059 *(request->inst), packet->req->getVaddr());
1060
1061 /* The sender state of the packet *must* be an LSQRequest
1062 * so the response can be correctly handled */
1063 assert(packet->findNextSenderState<LSQRequest>());
1064
1065 if (request->request.isMmappedIpr()) {
1066 ThreadContext *thread =
1067 cpu.getContext(request->request.threadId());
1068
1069 if (request->isLoad) {
1070 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1071 TheISA::handleIprRead(thread, packet);
1072 } else {
1073 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1074 TheISA::handleIprWrite(thread, packet);
1075 }
1076
1077 request->stepToNextPacket();
1078 ret = request->sentAllPackets();
1079
1080 if (!ret) {
1081 DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1082 *(request->inst));
1083 }
1084
1085 if (ret)
1086 request->setState(LSQRequest::Complete);
1087 else
1088 request->setState(LSQRequest::RequestIssuing);
1089 } else if (dcachePort.sendTimingReq(packet)) {
1090 DPRINTF(MinorMem, "Sent data memory request\n");
1091
1092 numAccessesInMemorySystem++;
1093
1094 request->stepToNextPacket();
1095
1096 ret = request->sentAllPackets();
1097
1098 switch (request->state) {
1099 case LSQRequest::Translated:
1100 case LSQRequest::RequestIssuing:
1101 /* Fully or partially issued a request in the transfers
1102 * queue */
1103 request->setState(LSQRequest::RequestIssuing);
1104 break;
1105 case LSQRequest::StoreInStoreBuffer:
1106 case LSQRequest::StoreBufferIssuing:
1107 /* Fully or partially issued a request in the store
1108 * buffer */
1109 request->setState(LSQRequest::StoreBufferIssuing);
1110 break;
1111 default:
1112 assert(false);
1113 break;
1114 }
1115
1116 state = MemoryRunning;
1117 } else {
1118 DPRINTF(MinorMem,
1119 "Sending data memory request - needs retry\n");
1120
1121 /* Needs to be resent, wait for that */
1122 state = MemoryNeedsRetry;
1123 retryRequest = request;
1124
1125 switch (request->state) {
1126 case LSQRequest::Translated:
1127 case LSQRequest::RequestIssuing:
1128 request->setState(LSQRequest::RequestNeedsRetry);
1129 break;
1130 case LSQRequest::StoreInStoreBuffer:
1131 case LSQRequest::StoreBufferIssuing:
1132 request->setState(LSQRequest::StoreBufferNeedsRetry);
1133 break;
1134 default:
1135 assert(false);
1136 break;
1137 }
1138 }
1139 }
1140
1141 return ret;
1142}
1143
1144void
1145LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1146{
1147 assert(!requests.empty() && requests.front() == request);
1148 assert(transfers.unreservedRemainingSpace() != 0);
1149
1150 /* Need to count the number of stores in the transfers
1151 * queue so that loads know when their store buffer forwarding
1152 * results will be correct (only when all those stores
1153 * have reached the store buffer) */
1154 if (!request->isLoad)
1155 numStoresInTransfers++;
1156
1157 requests.pop();
1158 transfers.push(request);
1159}
1160
1161bool
1162LSQ::canSendToMemorySystem()
1163{
1164 return state == MemoryRunning &&
1165 numAccessesInMemorySystem < inMemorySystemLimit;
1166}
1167
1168bool
1169LSQ::recvTimingResp(PacketPtr response)
1170{
1171 LSQRequestPtr request =
1172 safe_cast<LSQRequestPtr>(response->popSenderState());
1173
1174 DPRINTF(MinorMem, "Received response packet inst: %s"
1175 " addr: 0x%x cmd: %s\n",
1176 *(request->inst), response->getAddr(),
1177 response->cmd.toString());
1178
1179 numAccessesInMemorySystem--;
1180
1181 if (response->isError()) {
1182 DPRINTF(MinorMem, "Received error response packet: %s\n",
1183 *request->inst);
1184 }
1185
1186 switch (request->state) {
1187 case LSQRequest::RequestIssuing:
1188 case LSQRequest::RequestNeedsRetry:
1189 /* Response to a request from the transfers queue */
1190 request->retireResponse(response);
1191
1192 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1193 request->hasPacketsInMemSystem(), request->isComplete());
1194
1195 break;
1196 case LSQRequest::StoreBufferIssuing:
1197 case LSQRequest::StoreBufferNeedsRetry:
1198 /* Response to a request from the store buffer */
1199 request->retireResponse(response);
1200
1201 /* Remove completed requests unless they are barrier (which will
1202 * need to be removed in order */
1203 if (request->isComplete()) {
1204 if (!request->isBarrier()) {
1205 storeBuffer.deleteRequest(request);
1206 } else {
1207 DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1208 " leaving the request as it is also a barrier\n",
1209 *(request->inst));
1210 }
1211 }
1212 break;
1213 default:
1214 /* Shouldn't be allowed to receive a response from another
1215 * state */
1216 assert(false);
1217 break;
1218 }
1219
1220 /* We go to idle even if there are more things in the requests queue
1221 * as it's the job of step to actually step us on to the next
1222 * transaction */
1223
1224 /* Let's try and wake up the processor for the next cycle */
1225 cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1226
1227 /* Never busy */
1228 return true;
1229}
1230
1231void
1232LSQ::recvRetry()
1233{
1234 DPRINTF(MinorMem, "Received retry request\n");
1235
1236 assert(state == MemoryNeedsRetry);
1237
1238 switch (retryRequest->state) {
1239 case LSQRequest::RequestNeedsRetry:
1240 /* Retry in the requests queue */
1241 retryRequest->setState(LSQRequest::Translated);
1242 break;
1243 case LSQRequest::StoreBufferNeedsRetry:
1244 /* Retry in the store buffer */
1245 retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1246 break;
1247 default:
1248 assert(false);
1249 }
1250
1251 /* Set state back to MemoryRunning so that the following
1252 * tryToSend can actually send. Note that this won't
1253 * allow another transfer in as tryToSend should
1254 * issue a memory request and either succeed for this
1255 * request or return the LSQ back to MemoryNeedsRetry */
1256 state = MemoryRunning;
1257
1258 /* Try to resend the request */
1259 if (tryToSend(retryRequest)) {
1260 /* Successfully sent, need to move the request */
1261 switch (retryRequest->state) {
1262 case LSQRequest::RequestIssuing:
1263 /* In the requests queue */
1264 moveFromRequestsToTransfers(retryRequest);
1265 break;
1266 case LSQRequest::StoreBufferIssuing:
1267 /* In the store buffer */
1268 storeBuffer.numUnissuedAccesses--;
1269 break;
1270 default:
1271 assert(false);
1272 break;
1273 }
1274 }
1275
1276 retryRequest = NULL;
1277}
1278
1279LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1280 MinorCPU &cpu_, Execute &execute_,
1281 unsigned int in_memory_system_limit, unsigned int line_width,
1282 unsigned int requests_queue_size, unsigned int transfers_queue_size,
1283 unsigned int store_buffer_size,
1284 unsigned int store_buffer_cycle_store_limit) :
1285 Named(name_),
1286 cpu(cpu_),
1287 execute(execute_),
1288 dcachePort(dcache_port_name_, *this, cpu_),
1289 lastMemBarrier(0),
1290 state(MemoryRunning),
1291 inMemorySystemLimit(in_memory_system_limit),
1292 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1293 requests(name_ + ".requests", "addr", requests_queue_size),
1294 transfers(name_ + ".transfers", "addr", transfers_queue_size),
1295 storeBuffer(name_ + ".storeBuffer",
1296 *this, store_buffer_size, store_buffer_cycle_store_limit),
1297 numAccessesInMemorySystem(0),
1298 numAccessesInDTLB(0),
1299 numStoresInTransfers(0),
1300 numAccessesIssuedToMemory(0),
1301 retryRequest(NULL),
1302 cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1303{
1304 if (in_memory_system_limit < 1) {
1305 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1306 in_memory_system_limit);
1307 }
1308
1309 if (store_buffer_cycle_store_limit < 1) {
1310 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1311 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1312 }
1313
1314 if (requests_queue_size < 1) {
1315 fatal("%s: executeLSQRequestsQueueSize must be"
1316 " >= 1 (%d)\n", name_, requests_queue_size);
1317 }
1318
1319 if (transfers_queue_size < 1) {
1320 fatal("%s: executeLSQTransfersQueueSize must be"
1321 " >= 1 (%d)\n", name_, transfers_queue_size);
1322 }
1323
1324 if (store_buffer_size < 1) {
1325 fatal("%s: executeLSQStoreBufferSize must be"
1326 " >= 1 (%d)\n", name_, store_buffer_size);
1327 }
1328
1329 if ((lineWidth & (lineWidth - 1)) != 0) {
1330 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1331 }
1332}
1333
1334LSQ::~LSQ()
1335{ }
1336
1337LSQ::LSQRequest::~LSQRequest()
1338{
1339 if (packet)
1340 delete packet;
1341 if (data)
1342 delete [] data;
1343}
1344
1345/**
1346 * Step the memory access mechanism on to its next state. In reality, most
1347 * of the stepping is done by the callbacks on the LSQ but this
1348 * function is responsible for issuing memory requests lodged in the
1349 * requests queue.
1350 */
1351void
1352LSQ::step()
1353{
1354 /* Try to move address-translated requests between queues and issue
1355 * them */
1356 if (!requests.empty())
1357 tryToSendToTransfers(requests.front());
1358
1359 storeBuffer.step();
1360}
1361
1362LSQ::LSQRequestPtr
1363LSQ::findResponse(MinorDynInstPtr inst)
1364{
1365 LSQ::LSQRequestPtr ret = NULL;
1366
1367 if (!transfers.empty()) {
1368 LSQRequestPtr request = transfers.front();
1369
1370 /* Same instruction and complete access or a store that's
1371 * capable of being moved to the store buffer */
1372 if (request->inst->id == inst->id) {
| 1/*
2 * Copyright (c) 2013-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Andrew Bardsley
38 */
39
40#include <iomanip>
41#include <sstream>
42
43#include "arch/locked_mem.hh"
44#include "arch/mmapped_ipr.hh"
45#include "cpu/minor/cpu.hh"
46#include "cpu/minor/exec_context.hh"
47#include "cpu/minor/execute.hh"
48#include "cpu/minor/lsq.hh"
49#include "cpu/minor/pipeline.hh"
50#include "debug/Activity.hh"
51#include "debug/MinorMem.hh"
52
53namespace Minor
54{
55
56/** Returns the offset of addr into an aligned a block of size block_size */
57static Addr
58addrBlockOffset(Addr addr, unsigned int block_size)
59{
60 return addr & (block_size - 1);
61}
62
63/** Returns true if the given [addr .. addr+size-1] transfer needs to be
64 * fragmented across a block size of block_size */
65static bool
66transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size)
67{
68 return (addrBlockOffset(addr, block_size) + size) > block_size;
69}
70
71LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_,
72 PacketDataPtr data_, uint64_t *res_) :
73 SenderState(),
74 port(port_),
75 inst(inst_),
76 isLoad(isLoad_),
77 data(data_),
78 packet(NULL),
79 request(),
80 fault(NoFault),
81 res(res_),
82 skipped(false),
83 issuedToMemory(false),
84 state(NotIssued)
85{ }
86
87LSQ::AddrRangeCoverage
88LSQ::LSQRequest::containsAddrRangeOf(
89 Addr req1_addr, unsigned int req1_size,
90 Addr req2_addr, unsigned int req2_size)
91{
92 /* 'end' here means the address of the byte just past the request
93 * blocks */
94 Addr req2_end_addr = req2_addr + req2_size;
95 Addr req1_end_addr = req1_addr + req1_size;
96
97 AddrRangeCoverage ret;
98
99 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr)
100 ret = NoAddrRangeCoverage;
101 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr)
102 ret = FullAddrRangeCoverage;
103 else
104 ret = PartialAddrRangeCoverage;
105
106 return ret;
107}
108
109LSQ::AddrRangeCoverage
110LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request)
111{
112 return containsAddrRangeOf(request.getPaddr(), request.getSize(),
113 other_request->request.getPaddr(), other_request->request.getSize());
114}
115
116bool
117LSQ::LSQRequest::isBarrier()
118{
119 return inst->isInst() && inst->staticInst->isMemBarrier();
120}
121
122bool
123LSQ::LSQRequest::needsToBeSentToStoreBuffer()
124{
125 return state == StoreToStoreBuffer;
126}
127
128void
129LSQ::LSQRequest::setState(LSQRequestState new_state)
130{
131 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:"
132 " %s\n", state, new_state, *inst);
133 state = new_state;
134}
135
136bool
137LSQ::LSQRequest::isComplete() const
138{
139 /* @todo, There is currently only one 'completed' state. This
140 * may not be a good choice */
141 return state == Complete;
142}
143
144void
145LSQ::LSQRequest::reportData(std::ostream &os) const
146{
147 os << (isLoad ? 'R' : 'W') << ';';
148 inst->reportData(os);
149 os << ';' << state;
150}
151
152std::ostream &
153operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage)
154{
155 switch (coverage) {
156 case LSQ::PartialAddrRangeCoverage:
157 os << "PartialAddrRangeCoverage";
158 break;
159 case LSQ::FullAddrRangeCoverage:
160 os << "FullAddrRangeCoverage";
161 break;
162 case LSQ::NoAddrRangeCoverage:
163 os << "NoAddrRangeCoverage";
164 break;
165 default:
166 os << "AddrRangeCoverage-" << static_cast<int>(coverage);
167 break;
168 }
169 return os;
170}
171
172std::ostream &
173operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state)
174{
175 switch (state) {
176 case LSQ::LSQRequest::NotIssued:
177 os << "NotIssued";
178 break;
179 case LSQ::LSQRequest::InTranslation:
180 os << "InTranslation";
181 break;
182 case LSQ::LSQRequest::Translated:
183 os << "Translated";
184 break;
185 case LSQ::LSQRequest::Failed:
186 os << "Failed";
187 break;
188 case LSQ::LSQRequest::RequestIssuing:
189 os << "RequestIssuing";
190 break;
191 case LSQ::LSQRequest::StoreToStoreBuffer:
192 os << "StoreToStoreBuffer";
193 break;
194 case LSQ::LSQRequest::StoreInStoreBuffer:
195 os << "StoreInStoreBuffer";
196 break;
197 case LSQ::LSQRequest::StoreBufferIssuing:
198 os << "StoreBufferIssuing";
199 break;
200 case LSQ::LSQRequest::RequestNeedsRetry:
201 os << "RequestNeedsRetry";
202 break;
203 case LSQ::LSQRequest::StoreBufferNeedsRetry:
204 os << "StoreBufferNeedsRetry";
205 break;
206 case LSQ::LSQRequest::Complete:
207 os << "Complete";
208 break;
209 default:
210 os << "LSQRequestState-" << static_cast<int>(state);
211 break;
212 }
213 return os;
214}
215
216void
217LSQ::clearMemBarrier(MinorDynInstPtr inst)
218{
219 bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier;
220
221 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n",
222 (is_last_barrier ? "last" : "a"), *inst);
223
224 if (is_last_barrier)
225 lastMemBarrier = 0;
226}
227
228void
229LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_,
230 ThreadContext *tc, BaseTLB::Mode mode)
231{
232 fault = fault_;
233
234 port.numAccessesInDTLB--;
235
236 DPRINTFS(MinorMem, (&port), "Received translation response for"
237 " request: %s\n", *inst);
238
239 makePacket();
240
241 setState(Translated);
242 port.tryToSendToTransfers(this);
243
244 /* Let's try and wake up the processor for the next cycle */
245 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
246}
247
248void
249LSQ::SingleDataRequest::startAddrTranslation()
250{
251 ThreadContext *thread = port.cpu.getContext(
252 inst->id.threadId);
253
254 port.numAccessesInDTLB++;
255
256 setState(LSQ::LSQRequest::InTranslation);
257
258 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n");
259 /* Submit the translation request. The response will come through
260 * finish/markDelayed on the LSQRequest as it bears the Translation
261 * interface */
262 thread->getDTBPtr()->translateTiming(
263 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write));
264}
265
266void
267LSQ::SingleDataRequest::retireResponse(PacketPtr packet_)
268{
269 DPRINTFS(MinorMem, (&port), "Retiring packet\n");
270 packet = packet_;
271 packetInFlight = false;
272 setState(Complete);
273}
274
275void
276LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_,
277 ThreadContext *tc, BaseTLB::Mode mode)
278{
279 fault = fault_;
280
281 port.numAccessesInDTLB--;
282
283 unsigned int M5_VAR_USED expected_fragment_index =
284 numTranslatedFragments;
285
286 numInTranslationFragments--;
287 numTranslatedFragments++;
288
289 DPRINTFS(MinorMem, (&port), "Received translation response for fragment"
290 " %d of request: %s\n", expected_fragment_index, *inst);
291
292 assert(request_ == fragmentRequests[expected_fragment_index]);
293
294 /* Wake up next cycle to get things going again in case the
295 * tryToSendToTransfers does take */
296 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
297
298 if (fault != NoFault) {
299 /* tryToSendToTransfers will handle the fault */
300
301 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:"
302 " %d of request: %s\n",
303 expected_fragment_index, *inst);
304
305 setState(Translated);
306 port.tryToSendToTransfers(this);
307 } else if (numTranslatedFragments == numFragments) {
308 makeFragmentPackets();
309
310 setState(Translated);
311 port.tryToSendToTransfers(this);
312 } else {
313 /* Avoid calling translateTiming from within ::finish */
314 assert(!translationEvent.scheduled());
315 port.cpu.schedule(translationEvent, curTick());
316 }
317}
318
319LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_,
320 bool isLoad_, PacketDataPtr data_, uint64_t *res_) :
321 LSQRequest(port_, inst_, isLoad_, data_, res_),
322 translationEvent(*this),
323 numFragments(0),
324 numInTranslationFragments(0),
325 numTranslatedFragments(0),
326 numIssuedFragments(0),
327 numRetiredFragments(0),
328 fragmentRequests(),
329 fragmentPackets()
330{
331 /* Don't know how many elements are needed until the request is
332 * populated by the caller. */
333}
334
335LSQ::SplitDataRequest::~SplitDataRequest()
336{
337 for (auto i = fragmentRequests.begin();
338 i != fragmentRequests.end(); i++)
339 {
340 delete *i;
341 }
342
343 for (auto i = fragmentPackets.begin();
344 i != fragmentPackets.end(); i++)
345 {
346 delete *i;
347 }
348}
349
350void
351LSQ::SplitDataRequest::makeFragmentRequests()
352{
353 Addr base_addr = request.getVaddr();
354 unsigned int whole_size = request.getSize();
355 unsigned int line_width = port.lineWidth;
356
357 unsigned int fragment_size;
358 Addr fragment_addr;
359
360 /* Assume that this transfer is across potentially many block snap
361 * boundaries:
362 *
363 * | _|________|________|________|___ |
364 * | |0| 1 | 2 | 3 | 4 | |
365 * | |_|________|________|________|___| |
366 * | | | | | |
367 *
368 * The first transfer (0) can be up to lineWidth in size.
369 * All the middle transfers (1-3) are lineWidth in size
370 * The last transfer (4) can be from zero to lineWidth - 1 in size
371 */
372 unsigned int first_fragment_offset =
373 addrBlockOffset(base_addr, line_width);
374 unsigned int last_fragment_size =
375 addrBlockOffset(base_addr + whole_size, line_width);
376 unsigned int first_fragment_size =
377 line_width - first_fragment_offset;
378
379 unsigned int middle_fragments_total_size =
380 whole_size - (first_fragment_size + last_fragment_size);
381
382 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0);
383
384 unsigned int middle_fragment_count =
385 middle_fragments_total_size / line_width;
386
387 numFragments = 1 /* first */ + middle_fragment_count +
388 (last_fragment_size == 0 ? 0 : 1);
389
390 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests."
391 " First fragment size: %d Last fragment size: %d\n",
392 numFragments, first_fragment_size,
393 (last_fragment_size == 0 ? line_width : last_fragment_size));
394
395 assert(((middle_fragment_count * line_width) +
396 first_fragment_size + last_fragment_size) == whole_size);
397
398 fragment_addr = base_addr;
399 fragment_size = first_fragment_size;
400
401 /* Just past the last address in the request */
402 Addr end_addr = base_addr + whole_size;
403
404 for (unsigned int fragment_index = 0; fragment_index < numFragments;
405 fragment_index++)
406 {
407 bool M5_VAR_USED is_last_fragment = false;
408
409 if (fragment_addr == base_addr) {
410 /* First fragment */
411 fragment_size = first_fragment_size;
412 } else {
413 if ((fragment_addr + line_width) > end_addr) {
414 /* Adjust size of last fragment */
415 fragment_size = end_addr - fragment_addr;
416 is_last_fragment = true;
417 } else {
418 /* Middle fragments */
419 fragment_size = line_width;
420 }
421 }
422
423 Request *fragment = new Request();
424
425 fragment->setThreadContext(request.contextId(), /* thread id */ 0);
426 fragment->setVirt(0 /* asid */,
427 fragment_addr, fragment_size, request.getFlags(),
428 request.masterId(),
429 request.getPC());
430
431 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d"
432 " (whole request addr: 0x%x size: %d) %s\n",
433 fragment_addr, fragment_size, base_addr, whole_size,
434 (is_last_fragment ? "last fragment" : ""));
435
436 fragment_addr += fragment_size;
437
438 fragmentRequests.push_back(fragment);
439 }
440}
441
442void
443LSQ::SplitDataRequest::makeFragmentPackets()
444{
445 Addr base_addr = request.getVaddr();
446
447 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst);
448
449 for (unsigned int fragment_index = 0; fragment_index < numFragments;
450 fragment_index++)
451 {
452 Request *fragment = fragmentRequests[fragment_index];
453
454 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s"
455 " (%d, 0x%x)\n",
456 fragment_index, *inst,
457 (fragment->hasPaddr() ? "has paddr" : "no paddr"),
458 (fragment->hasPaddr() ? fragment->getPaddr() : 0));
459
460 Addr fragment_addr = fragment->getVaddr();
461 unsigned int fragment_size = fragment->getSize();
462
463 uint8_t *request_data = NULL;
464
465 if (!isLoad) {
466 /* Split data for Packets. Will become the property of the
467 * outgoing Packets */
468 request_data = new uint8_t[fragment_size];
469 std::memcpy(request_data, data + (fragment_addr - base_addr),
470 fragment_size);
471 }
472
473 assert(fragment->hasPaddr());
474
475 PacketPtr fragment_packet =
476 makePacketForRequest(*fragment, isLoad, this, request_data);
477
478 fragmentPackets.push_back(fragment_packet);
479 /* Accumulate flags in parent request */
480 request.setFlags(fragment->getFlags());
481 }
482
483 /* Might as well make the overall/response packet here */
484 /* Get the physical address for the whole request/packet from the first
485 * fragment */
486 request.setPaddr(fragmentRequests[0]->getPaddr());
487 makePacket();
488}
489
490void
491LSQ::SplitDataRequest::startAddrTranslation()
492{
493 setState(LSQ::LSQRequest::InTranslation);
494
495 makeFragmentRequests();
496
497 numInTranslationFragments = 0;
498 numTranslatedFragments = 0;
499
500 /* @todo, just do these in sequence for now with
501 * a loop of:
502 * do {
503 * sendNextFragmentToTranslation ; translateTiming ; finish
504 * } while (numTranslatedFragments != numFragments);
505 */
506
507 /* Do first translation */
508 sendNextFragmentToTranslation();
509}
510
511PacketPtr
512LSQ::SplitDataRequest::getHeadPacket()
513{
514 assert(numIssuedFragments < numFragments);
515
516 return fragmentPackets[numIssuedFragments];
517}
518
519void
520LSQ::SplitDataRequest::stepToNextPacket()
521{
522 assert(numIssuedFragments < numFragments);
523
524 numIssuedFragments++;
525}
526
527void
528LSQ::SplitDataRequest::retireResponse(PacketPtr response)
529{
530 assert(numRetiredFragments < numFragments);
531
532 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d"
533 " offset: 0x%x (retired fragment num: %d) %s\n",
534 response->req->getVaddr(), response->req->getSize(),
535 request.getVaddr() - response->req->getVaddr(),
536 numRetiredFragments,
537 (fault == NoFault ? "" : fault->name()));
538
539 numRetiredFragments++;
540
541 if (skipped) {
542 /* Skip because we already knew the request had faulted or been
543 * skipped */
544 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n");
545 } else if (response->isError()) {
546 /* Mark up the error and leave to execute to handle it */
547 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n");
548 setSkipped();
549 packet->copyError(response);
550 } else {
551 if (isLoad) {
552 if (!data) {
553 /* For a split transfer, a Packet must be constructed
554 * to contain all returning data. This is that packet's
555 * data */
556 data = new uint8_t[request.getSize()];
557 }
558
559 /* Populate the portion of the overall response data represented
560 * by the response fragment */
561 std::memcpy(
562 data + (response->req->getVaddr() - request.getVaddr()),
563 response->getPtr<uint8_t>(),
564 response->req->getSize());
565 }
566 }
567
568 /* Complete early if we're skipping are no more in-flight accesses */
569 if (skipped && !hasPacketsInMemSystem()) {
570 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n");
571 setState(Complete);
572 if (packet->needsResponse())
573 packet->makeResponse();
574 }
575
576 if (numRetiredFragments == numFragments)
577 setState(Complete);
578
579 if (!skipped && isComplete()) {
580 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL);
581
582 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d"
583 " needsResponse: %d packetSize: %s requestSize: %s responseSize:"
584 " %s\n", packet->isRead(), packet->isWrite(),
585 packet->needsResponse(), packet->getSize(), request.getSize(),
586 response->getSize());
587
588 /* A request can become complete by several paths, this is a sanity
589 * check to make sure the packet's data is created */
590 if (!data) {
591 data = new uint8_t[request.getSize()];
592 }
593
594 if (isLoad) {
595 DPRINTFS(MinorMem, (&port), "Copying read data\n");
596 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize());
597 }
598 packet->makeResponse();
599 }
600
601 /* Packets are all deallocated together in ~SplitLSQRequest */
602}
603
604void
605LSQ::SplitDataRequest::sendNextFragmentToTranslation()
606{
607 unsigned int fragment_index = numTranslatedFragments;
608
609 ThreadContext *thread = port.cpu.getContext(
610 inst->id.threadId);
611
612 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n",
613 fragment_index);
614
615 port.numAccessesInDTLB++;
616 numInTranslationFragments++;
617
618 thread->getDTBPtr()->translateTiming(
619 fragmentRequests[fragment_index], thread, this, (isLoad ?
620 BaseTLB::Read : BaseTLB::Write));
621}
622
623bool
624LSQ::StoreBuffer::canInsert() const
625{
626 /* @todo, support store amalgamation */
627 return slots.size() < numSlots;
628}
629
630void
631LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request)
632{
633 auto found = std::find(slots.begin(), slots.end(), request);
634
635 if (found != slots.end()) {
636 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n",
637 request, *found, *(request->inst));
638 slots.erase(found);
639
640 delete request;
641 }
642}
643
644void
645LSQ::StoreBuffer::insert(LSQRequestPtr request)
646{
647 if (!canInsert()) {
648 warn("%s: store buffer insertion without space to insert from"
649 " inst: %s\n", name(), *(request->inst));
650 }
651
652 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request);
653
654 numUnissuedAccesses++;
655
656 if (request->state != LSQRequest::Complete)
657 request->setState(LSQRequest::StoreInStoreBuffer);
658
659 slots.push_back(request);
660
661 /* Let's try and wake up the processor for the next cycle to step
662 * the store buffer */
663 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
664}
665
666LSQ::AddrRangeCoverage
667LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request,
668 unsigned int &found_slot)
669{
670 unsigned int slot_index = slots.size() - 1;
671 auto i = slots.rbegin();
672 AddrRangeCoverage ret = NoAddrRangeCoverage;
673
674 /* Traverse the store buffer in reverse order (most to least recent)
675 * and try to find a slot whose address range overlaps this request */
676 while (ret == NoAddrRangeCoverage && i != slots.rend()) {
677 LSQRequestPtr slot = *i;
678
679 if (slot->packet) {
680 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request);
681
682 if (coverage != NoAddrRangeCoverage) {
683 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:"
684 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n",
685 slot_index, coverage,
686 request->request.getPaddr(), request->request.getSize(),
687 slot->request.getPaddr(), slot->request.getSize());
688
689 found_slot = slot_index;
690 ret = coverage;
691 }
692 }
693
694 i++;
695 slot_index--;
696 }
697
698 return ret;
699}
700
701/** Fill the given packet with appropriate date from slot slot_number */
702void
703LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load,
704 unsigned int slot_number)
705{
706 assert(slot_number < slots.size());
707 assert(load->packet);
708 assert(load->isLoad);
709
710 LSQRequestPtr store = slots[slot_number];
711
712 assert(store->packet);
713 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage);
714
715 Addr load_addr = load->request.getPaddr();
716 Addr store_addr = store->request.getPaddr();
717 Addr addr_offset = load_addr - store_addr;
718
719 unsigned int load_size = load->request.getSize();
720
721 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer"
722 " slot: %d addr: 0x%x addressOffset: 0x%x\n",
723 load_size, load_addr, slot_number,
724 store_addr, addr_offset);
725
726 void *load_packet_data = load->packet->getPtr<void>();
727 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset;
728
729 std::memcpy(load_packet_data, store_packet_data, load_size);
730}
731
732void
733LSQ::StoreBuffer::step()
734{
735 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n",
736 numUnissuedAccesses);
737
738 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) {
739 /* Clear all the leading barriers */
740 while (!slots.empty() &&
741 slots.front()->isComplete() && slots.front()->isBarrier())
742 {
743 LSQRequestPtr barrier = slots.front();
744
745 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n",
746 *(barrier->inst));
747
748 numUnissuedAccesses--;
749 lsq.clearMemBarrier(barrier->inst);
750 slots.pop_front();
751
752 delete barrier;
753 }
754
755 auto i = slots.begin();
756 bool issued = true;
757 unsigned int issue_count = 0;
758
759 /* Skip trying if the memory system is busy */
760 if (lsq.state == LSQ::MemoryNeedsRetry)
761 issued = false;
762
763 /* Try to issue all stores in order starting from the head
764 * of the queue. Responses are allowed to be retired
765 * out of order */
766 while (issued &&
767 issue_count < storeLimitPerCycle &&
768 lsq.canSendToMemorySystem() &&
769 i != slots.end())
770 {
771 LSQRequestPtr request = *i;
772
773 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d"
774 " state: %s\n",
775 *(request->inst), request->sentAllPackets(),
776 request->state);
777
778 if (request->isBarrier() && request->isComplete()) {
779 /* Give up at barriers */
780 issued = false;
781 } else if (!(request->state == LSQRequest::StoreBufferIssuing &&
782 request->sentAllPackets()))
783 {
784 DPRINTF(MinorMem, "Trying to send request: %s to memory"
785 " system\n", *(request->inst));
786
787 if (lsq.tryToSend(request)) {
788 /* Barrier are accounted for as they are cleared from
789 * the queue, not after their transfers are complete */
790 if (!request->isBarrier())
791 numUnissuedAccesses--;
792 issue_count++;
793 } else {
794 /* Don't step on to the next store buffer entry if this
795 * one hasn't issued all its packets as the store
796 * buffer must still enforce ordering */
797 issued = false;
798 }
799 }
800 i++;
801 }
802 }
803}
804
805void
806LSQ::completeMemBarrierInst(MinorDynInstPtr inst,
807 bool committed)
808{
809 if (committed) {
810 /* Not already sent to the store buffer as a store request? */
811 if (!inst->inStoreBuffer) {
812 /* Insert an entry into the store buffer to tick off barriers
813 * until there are none in flight */
814 storeBuffer.insert(new BarrierDataRequest(*this, inst));
815 }
816 } else {
817 /* Clear the barrier anyway if it wasn't actually committed */
818 clearMemBarrier(inst);
819 }
820}
821
822void
823LSQ::StoreBuffer::minorTrace() const
824{
825 unsigned int size = slots.size();
826 unsigned int i = 0;
827 std::ostringstream os;
828
829 while (i < size) {
830 LSQRequestPtr request = slots[i];
831
832 request->reportData(os);
833
834 i++;
835 if (i < numSlots)
836 os << ',';
837 }
838
839 while (i < numSlots) {
840 os << '-';
841
842 i++;
843 if (i < numSlots)
844 os << ',';
845 }
846
847 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(),
848 numUnissuedAccesses);
849}
850
851void
852LSQ::tryToSendToTransfers(LSQRequestPtr request)
853{
854 if (state == MemoryNeedsRetry) {
855 DPRINTF(MinorMem, "Request needs retry, not issuing to"
856 " memory until retry arrives\n");
857 return;
858 }
859
860 if (request->state == LSQRequest::InTranslation) {
861 DPRINTF(MinorMem, "Request still in translation, not issuing to"
862 " memory\n");
863 return;
864 }
865
866 assert(request->state == LSQRequest::Translated ||
867 request->state == LSQRequest::RequestIssuing ||
868 request->state == LSQRequest::Failed ||
869 request->state == LSQRequest::Complete);
870
871 if (requests.empty() || requests.front() != request) {
872 DPRINTF(MinorMem, "Request not at front of requests queue, can't"
873 " issue to memory\n");
874 return;
875 }
876
877 if (transfers.unreservedRemainingSpace() == 0) {
878 DPRINTF(MinorMem, "No space to insert request into transfers"
879 " queue\n");
880 return;
881 }
882
883 if (request->isComplete() || request->state == LSQRequest::Failed) {
884 DPRINTF(MinorMem, "Passing a %s transfer on to transfers"
885 " queue\n", (request->isComplete() ? "completed" : "failed"));
886 request->setState(LSQRequest::Complete);
887 request->setSkipped();
888 moveFromRequestsToTransfers(request);
889 return;
890 }
891
892 if (!execute.instIsRightStream(request->inst)) {
893 /* Wrong stream, try to abort the transfer but only do so if
894 * there are no packets in flight */
895 if (request->hasPacketsInMemSystem()) {
896 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
897 " waiting for responses before aborting request\n");
898 } else {
899 DPRINTF(MinorMem, "Request's inst. is from the wrong stream,"
900 " aborting request\n");
901 request->setState(LSQRequest::Complete);
902 request->setSkipped();
903 moveFromRequestsToTransfers(request);
904 }
905 return;
906 }
907
908 if (request->fault != NoFault) {
909 if (request->inst->staticInst->isPrefetch()) {
910 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n");
911 }
912 DPRINTF(MinorMem, "Moving faulting request into the transfers"
913 " queue\n");
914 request->setState(LSQRequest::Complete);
915 request->setSkipped();
916 moveFromRequestsToTransfers(request);
917 return;
918 }
919
920 bool is_load = request->isLoad;
921 bool is_llsc = request->request.isLLSC();
922 bool is_swap = request->request.isSwap();
923 bool bufferable = !(request->request.isUncacheable() ||
924 is_llsc || is_swap);
925
926 if (is_load) {
927 if (numStoresInTransfers != 0) {
928 DPRINTF(MinorMem, "Load request with stores still in transfers"
929 " queue, stalling\n");
930 return;
931 }
932 } else {
933 /* Store. Can it be sent to the store buffer? */
934 if (bufferable && !request->request.isMmappedIpr()) {
935 request->setState(LSQRequest::StoreToStoreBuffer);
936 moveFromRequestsToTransfers(request);
937 DPRINTF(MinorMem, "Moving store into transfers queue\n");
938 return;
939 }
940 }
941
942 /* Check if this is the head instruction (and so must be executable as
943 * its stream sequence number was checked above) for loads which must
944 * not be speculatively issued and stores which must be issued here */
945 if (!bufferable) {
946 if (!execute.instIsHeadInst(request->inst)) {
947 DPRINTF(MinorMem, "Memory access not the head inst., can't be"
948 " sure it can be performed, not issuing\n");
949 return;
950 }
951
952 unsigned int forwarding_slot = 0;
953
954 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) !=
955 NoAddrRangeCoverage)
956 {
957 DPRINTF(MinorMem, "Memory access can receive forwarded data"
958 " from the store buffer, need to wait for store buffer to"
959 " drain\n");
960 return;
961 }
962 }
963
964 /* True: submit this packet to the transfers queue to be sent to the
965 * memory system.
966 * False: skip the memory and push a packet for this request onto
967 * requests */
968 bool do_access = true;
969
970 if (!is_llsc) {
971 /* Check for match in the store buffer */
972 if (is_load) {
973 unsigned int forwarding_slot = 0;
974 AddrRangeCoverage forwarding_result =
975 storeBuffer.canForwardDataToLoad(request,
976 forwarding_slot);
977
978 switch (forwarding_result) {
979 case FullAddrRangeCoverage:
980 /* Forward data from the store buffer into this request and
981 * repurpose this request's packet into a response packet */
982 storeBuffer.forwardStoreData(request, forwarding_slot);
983 request->packet->makeResponse();
984
985 /* Just move between queues, no access */
986 do_access = false;
987 break;
988 case PartialAddrRangeCoverage:
989 DPRINTF(MinorMem, "Load partly satisfied by store buffer"
990 " data. Must wait for the store to complete\n");
991 return;
992 break;
993 case NoAddrRangeCoverage:
994 DPRINTF(MinorMem, "No forwardable data from store buffer\n");
995 /* Fall through to try access */
996 break;
997 }
998 }
999 } else {
1000 if (!canSendToMemorySystem()) {
1001 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1002 return;
1003 }
1004
1005 SimpleThread &thread = *cpu.threads[request->inst->id.threadId];
1006
1007 TheISA::PCState old_pc = thread.pcState();
1008 ExecContext context(cpu, thread, execute, request->inst);
1009
1010 /* Handle LLSC requests and tests */
1011 if (is_load) {
1012 TheISA::handleLockedRead(&context, &request->request);
1013 } else {
1014 do_access = TheISA::handleLockedWrite(&context,
1015 &request->request, cacheBlockMask);
1016
1017 if (!do_access) {
1018 DPRINTF(MinorMem, "Not perfoming a memory "
1019 "access for store conditional\n");
1020 }
1021 }
1022 thread.pcState(old_pc);
1023 }
1024
1025 /* See the do_access comment above */
1026 if (do_access) {
1027 if (!canSendToMemorySystem()) {
1028 DPRINTF(MinorMem, "Can't send request to memory system yet\n");
1029 return;
1030 }
1031
1032 /* Remember if this is an access which can't be idly
1033 * discarded by an interrupt */
1034 if (!bufferable && !request->issuedToMemory) {
1035 numAccessesIssuedToMemory++;
1036 request->issuedToMemory = true;
1037 }
1038
1039 if (tryToSend(request))
1040 moveFromRequestsToTransfers(request);
1041 } else {
1042 request->setState(LSQRequest::Complete);
1043 moveFromRequestsToTransfers(request);
1044 }
1045}
1046
1047bool
1048LSQ::tryToSend(LSQRequestPtr request)
1049{
1050 bool ret = false;
1051
1052 if (!canSendToMemorySystem()) {
1053 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n",
1054 *(request->inst));
1055 } else {
1056 PacketPtr packet = request->getHeadPacket();
1057
1058 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n",
1059 *(request->inst), packet->req->getVaddr());
1060
1061 /* The sender state of the packet *must* be an LSQRequest
1062 * so the response can be correctly handled */
1063 assert(packet->findNextSenderState<LSQRequest>());
1064
1065 if (request->request.isMmappedIpr()) {
1066 ThreadContext *thread =
1067 cpu.getContext(request->request.threadId());
1068
1069 if (request->isLoad) {
1070 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst));
1071 TheISA::handleIprRead(thread, packet);
1072 } else {
1073 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst));
1074 TheISA::handleIprWrite(thread, packet);
1075 }
1076
1077 request->stepToNextPacket();
1078 ret = request->sentAllPackets();
1079
1080 if (!ret) {
1081 DPRINTF(MinorMem, "IPR access has another packet: %s\n",
1082 *(request->inst));
1083 }
1084
1085 if (ret)
1086 request->setState(LSQRequest::Complete);
1087 else
1088 request->setState(LSQRequest::RequestIssuing);
1089 } else if (dcachePort.sendTimingReq(packet)) {
1090 DPRINTF(MinorMem, "Sent data memory request\n");
1091
1092 numAccessesInMemorySystem++;
1093
1094 request->stepToNextPacket();
1095
1096 ret = request->sentAllPackets();
1097
1098 switch (request->state) {
1099 case LSQRequest::Translated:
1100 case LSQRequest::RequestIssuing:
1101 /* Fully or partially issued a request in the transfers
1102 * queue */
1103 request->setState(LSQRequest::RequestIssuing);
1104 break;
1105 case LSQRequest::StoreInStoreBuffer:
1106 case LSQRequest::StoreBufferIssuing:
1107 /* Fully or partially issued a request in the store
1108 * buffer */
1109 request->setState(LSQRequest::StoreBufferIssuing);
1110 break;
1111 default:
1112 assert(false);
1113 break;
1114 }
1115
1116 state = MemoryRunning;
1117 } else {
1118 DPRINTF(MinorMem,
1119 "Sending data memory request - needs retry\n");
1120
1121 /* Needs to be resent, wait for that */
1122 state = MemoryNeedsRetry;
1123 retryRequest = request;
1124
1125 switch (request->state) {
1126 case LSQRequest::Translated:
1127 case LSQRequest::RequestIssuing:
1128 request->setState(LSQRequest::RequestNeedsRetry);
1129 break;
1130 case LSQRequest::StoreInStoreBuffer:
1131 case LSQRequest::StoreBufferIssuing:
1132 request->setState(LSQRequest::StoreBufferNeedsRetry);
1133 break;
1134 default:
1135 assert(false);
1136 break;
1137 }
1138 }
1139 }
1140
1141 return ret;
1142}
1143
1144void
1145LSQ::moveFromRequestsToTransfers(LSQRequestPtr request)
1146{
1147 assert(!requests.empty() && requests.front() == request);
1148 assert(transfers.unreservedRemainingSpace() != 0);
1149
1150 /* Need to count the number of stores in the transfers
1151 * queue so that loads know when their store buffer forwarding
1152 * results will be correct (only when all those stores
1153 * have reached the store buffer) */
1154 if (!request->isLoad)
1155 numStoresInTransfers++;
1156
1157 requests.pop();
1158 transfers.push(request);
1159}
1160
1161bool
1162LSQ::canSendToMemorySystem()
1163{
1164 return state == MemoryRunning &&
1165 numAccessesInMemorySystem < inMemorySystemLimit;
1166}
1167
1168bool
1169LSQ::recvTimingResp(PacketPtr response)
1170{
1171 LSQRequestPtr request =
1172 safe_cast<LSQRequestPtr>(response->popSenderState());
1173
1174 DPRINTF(MinorMem, "Received response packet inst: %s"
1175 " addr: 0x%x cmd: %s\n",
1176 *(request->inst), response->getAddr(),
1177 response->cmd.toString());
1178
1179 numAccessesInMemorySystem--;
1180
1181 if (response->isError()) {
1182 DPRINTF(MinorMem, "Received error response packet: %s\n",
1183 *request->inst);
1184 }
1185
1186 switch (request->state) {
1187 case LSQRequest::RequestIssuing:
1188 case LSQRequest::RequestNeedsRetry:
1189 /* Response to a request from the transfers queue */
1190 request->retireResponse(response);
1191
1192 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n",
1193 request->hasPacketsInMemSystem(), request->isComplete());
1194
1195 break;
1196 case LSQRequest::StoreBufferIssuing:
1197 case LSQRequest::StoreBufferNeedsRetry:
1198 /* Response to a request from the store buffer */
1199 request->retireResponse(response);
1200
1201 /* Remove completed requests unless they are barrier (which will
1202 * need to be removed in order */
1203 if (request->isComplete()) {
1204 if (!request->isBarrier()) {
1205 storeBuffer.deleteRequest(request);
1206 } else {
1207 DPRINTF(MinorMem, "Completed transfer for barrier: %s"
1208 " leaving the request as it is also a barrier\n",
1209 *(request->inst));
1210 }
1211 }
1212 break;
1213 default:
1214 /* Shouldn't be allowed to receive a response from another
1215 * state */
1216 assert(false);
1217 break;
1218 }
1219
1220 /* We go to idle even if there are more things in the requests queue
1221 * as it's the job of step to actually step us on to the next
1222 * transaction */
1223
1224 /* Let's try and wake up the processor for the next cycle */
1225 cpu.wakeupOnEvent(Pipeline::ExecuteStageId);
1226
1227 /* Never busy */
1228 return true;
1229}
1230
1231void
1232LSQ::recvRetry()
1233{
1234 DPRINTF(MinorMem, "Received retry request\n");
1235
1236 assert(state == MemoryNeedsRetry);
1237
1238 switch (retryRequest->state) {
1239 case LSQRequest::RequestNeedsRetry:
1240 /* Retry in the requests queue */
1241 retryRequest->setState(LSQRequest::Translated);
1242 break;
1243 case LSQRequest::StoreBufferNeedsRetry:
1244 /* Retry in the store buffer */
1245 retryRequest->setState(LSQRequest::StoreInStoreBuffer);
1246 break;
1247 default:
1248 assert(false);
1249 }
1250
1251 /* Set state back to MemoryRunning so that the following
1252 * tryToSend can actually send. Note that this won't
1253 * allow another transfer in as tryToSend should
1254 * issue a memory request and either succeed for this
1255 * request or return the LSQ back to MemoryNeedsRetry */
1256 state = MemoryRunning;
1257
1258 /* Try to resend the request */
1259 if (tryToSend(retryRequest)) {
1260 /* Successfully sent, need to move the request */
1261 switch (retryRequest->state) {
1262 case LSQRequest::RequestIssuing:
1263 /* In the requests queue */
1264 moveFromRequestsToTransfers(retryRequest);
1265 break;
1266 case LSQRequest::StoreBufferIssuing:
1267 /* In the store buffer */
1268 storeBuffer.numUnissuedAccesses--;
1269 break;
1270 default:
1271 assert(false);
1272 break;
1273 }
1274 }
1275
1276 retryRequest = NULL;
1277}
1278
1279LSQ::LSQ(std::string name_, std::string dcache_port_name_,
1280 MinorCPU &cpu_, Execute &execute_,
1281 unsigned int in_memory_system_limit, unsigned int line_width,
1282 unsigned int requests_queue_size, unsigned int transfers_queue_size,
1283 unsigned int store_buffer_size,
1284 unsigned int store_buffer_cycle_store_limit) :
1285 Named(name_),
1286 cpu(cpu_),
1287 execute(execute_),
1288 dcachePort(dcache_port_name_, *this, cpu_),
1289 lastMemBarrier(0),
1290 state(MemoryRunning),
1291 inMemorySystemLimit(in_memory_system_limit),
1292 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)),
1293 requests(name_ + ".requests", "addr", requests_queue_size),
1294 transfers(name_ + ".transfers", "addr", transfers_queue_size),
1295 storeBuffer(name_ + ".storeBuffer",
1296 *this, store_buffer_size, store_buffer_cycle_store_limit),
1297 numAccessesInMemorySystem(0),
1298 numAccessesInDTLB(0),
1299 numStoresInTransfers(0),
1300 numAccessesIssuedToMemory(0),
1301 retryRequest(NULL),
1302 cacheBlockMask(~(cpu_.cacheLineSize() - 1))
1303{
1304 if (in_memory_system_limit < 1) {
1305 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_,
1306 in_memory_system_limit);
1307 }
1308
1309 if (store_buffer_cycle_store_limit < 1) {
1310 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be"
1311 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit);
1312 }
1313
1314 if (requests_queue_size < 1) {
1315 fatal("%s: executeLSQRequestsQueueSize must be"
1316 " >= 1 (%d)\n", name_, requests_queue_size);
1317 }
1318
1319 if (transfers_queue_size < 1) {
1320 fatal("%s: executeLSQTransfersQueueSize must be"
1321 " >= 1 (%d)\n", name_, transfers_queue_size);
1322 }
1323
1324 if (store_buffer_size < 1) {
1325 fatal("%s: executeLSQStoreBufferSize must be"
1326 " >= 1 (%d)\n", name_, store_buffer_size);
1327 }
1328
1329 if ((lineWidth & (lineWidth - 1)) != 0) {
1330 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth);
1331 }
1332}
1333
1334LSQ::~LSQ()
1335{ }
1336
1337LSQ::LSQRequest::~LSQRequest()
1338{
1339 if (packet)
1340 delete packet;
1341 if (data)
1342 delete [] data;
1343}
1344
1345/**
1346 * Step the memory access mechanism on to its next state. In reality, most
1347 * of the stepping is done by the callbacks on the LSQ but this
1348 * function is responsible for issuing memory requests lodged in the
1349 * requests queue.
1350 */
1351void
1352LSQ::step()
1353{
1354 /* Try to move address-translated requests between queues and issue
1355 * them */
1356 if (!requests.empty())
1357 tryToSendToTransfers(requests.front());
1358
1359 storeBuffer.step();
1360}
1361
1362LSQ::LSQRequestPtr
1363LSQ::findResponse(MinorDynInstPtr inst)
1364{
1365 LSQ::LSQRequestPtr ret = NULL;
1366
1367 if (!transfers.empty()) {
1368 LSQRequestPtr request = transfers.front();
1369
1370 /* Same instruction and complete access or a store that's
1371 * capable of being moved to the store buffer */
1372 if (request->inst->id == inst->id) {
|
1376 {
1377 ret = request;
1378 }
1379 }
1380 }
1381
1382 if (ret) {
1383 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1384 *inst);
1385 } else {
1386 DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1387 *inst);
1388 }
1389
1390 return ret;
1391}
1392
1393void
1394LSQ::popResponse(LSQ::LSQRequestPtr response)
1395{
1396 assert(!transfers.empty() && transfers.front() == response);
1397
1398 transfers.pop();
1399
1400 if (!response->isLoad)
1401 numStoresInTransfers--;
1402
1403 if (response->issuedToMemory)
1404 numAccessesIssuedToMemory--;
1405
1406 if (response->state != LSQRequest::StoreInStoreBuffer) {
1407 DPRINTF(MinorMem, "Deleting %s request: %s\n",
1408 (response->isLoad ? "load" : "store"),
1409 *(response->inst));
1410
1411 delete response;
1412 }
1413}
1414
1415void
1416LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1417{
1418 assert(request->state == LSQRequest::StoreToStoreBuffer);
1419
1420 DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1421 *(request->inst));
1422
1423 request->inst->inStoreBuffer = true;
1424
1425 storeBuffer.insert(request);
1426}
1427
1428bool
1429LSQ::isDrained()
1430{
1431 return requests.empty() && transfers.empty() &&
1432 storeBuffer.isDrained();
1433}
1434
1435bool
1436LSQ::needsToTick()
1437{
1438 bool ret = false;
1439
1440 if (canSendToMemorySystem()) {
1441 bool have_translated_requests = !requests.empty() &&
1442 requests.front()->state != LSQRequest::InTranslation &&
1443 transfers.unreservedRemainingSpace() != 0;
1444
1445 ret = have_translated_requests ||
1446 storeBuffer.numUnissuedStores() != 0;
1447 }
1448
1449 if (ret)
1450 DPRINTF(Activity, "Need to tick\n");
1451
1452 return ret;
1453}
1454
1455void
1456LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1457 unsigned int size, Addr addr, unsigned int flags, uint64_t *res)
1458{
1459 bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1460 LSQRequestPtr request;
1461
1462 /* Copy given data into the request. The request will pass this to the
1463 * packet and then it will own the data */
1464 uint8_t *request_data = NULL;
1465
1466 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1467 " 0x%x%s lineWidth : 0x%x\n",
1468 (isLoad ? "load" : "store"), addr, size, flags,
1469 (needs_burst ? " (needs burst)" : ""), lineWidth);
1470
1471 if (!isLoad) {
1472 /* request_data becomes the property of a ...DataRequest (see below)
1473 * and destroyed by its destructor */
1474 request_data = new uint8_t[size];
1475 if (flags & Request::CACHE_BLOCK_ZERO) {
1476 /* For cache zeroing, just use zeroed data */
1477 std::memset(request_data, 0, size);
1478 } else {
1479 std::memcpy(request_data, data, size);
1480 }
1481 }
1482
1483 if (needs_burst) {
1484 request = new SplitDataRequest(
1485 *this, inst, isLoad, request_data, res);
1486 } else {
1487 request = new SingleDataRequest(
1488 *this, inst, isLoad, request_data, res);
1489 }
1490
1491 if (inst->traceData)
1492 inst->traceData->setAddr(addr);
1493
1494 request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0);
1495 request->request.setVirt(0 /* asid */,
1496 addr, size, flags, cpu.instMasterId(),
1497 /* I've no idea why we need the PC, but give it */
1498 inst->pc.instAddr());
1499
1500 requests.push(request);
1501 request->startAddrTranslation();
1502}
1503
1504void
1505LSQ::pushFailedRequest(MinorDynInstPtr inst)
1506{
1507 LSQRequestPtr request = new FailedDataRequest(*this, inst);
1508 requests.push(request);
1509}
1510
1511void
1512LSQ::minorTrace() const
1513{
1514 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1515 " lastMemBarrier=%d\n",
1516 state, numAccessesInDTLB, numAccessesInMemorySystem,
1517 numStoresInTransfers, lastMemBarrier);
1518 requests.minorTrace();
1519 transfers.minorTrace();
1520 storeBuffer.minorTrace();
1521}
1522
1523LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1524 unsigned int store_buffer_size,
1525 unsigned int store_limit_per_cycle) :
1526 Named(name_), lsq(lsq_),
1527 numSlots(store_buffer_size),
1528 storeLimitPerCycle(store_limit_per_cycle),
1529 slots(),
1530 numUnissuedAccesses(0)
1531{
1532}
1533
1534PacketPtr
1535makePacketForRequest(Request &request, bool isLoad,
1536 Packet::SenderState *sender_state, PacketDataPtr data)
1537{
1538 MemCmd command;
1539
1540 /* Make a ret with the right command type to match the request */
1541 if (request.isLLSC()) {
1542 command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq);
1543 } else if (request.isSwap()) {
1544 command = MemCmd::SwapReq;
1545 } else {
1546 command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq);
1547 }
1548
1549 PacketPtr ret = new Packet(&request, command);
1550
1551 if (sender_state)
1552 ret->pushSenderState(sender_state);
1553
1554 if (isLoad)
1555 ret->allocate();
1556 else
1557 ret->dataDynamicArray(data);
1558
1559 return ret;
1560}
1561
1562void
1563LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1564{
1565 assert(inst->isInst() && inst->staticInst->isMemBarrier());
1566 assert(inst->id.execSeqNum > lastMemBarrier);
1567
1568 /* Remember the barrier. We only have a notion of one
1569 * barrier so this may result in some mem refs being
1570 * delayed if they are between barriers */
1571 lastMemBarrier = inst->id.execSeqNum;
1572}
1573
1574void
1575LSQ::LSQRequest::makePacket()
1576{
1577 /* Make the function idempotent */
1578 if (packet)
1579 return;
1580
1581 packet = makePacketForRequest(request, isLoad, this, data);
1582 /* Null the ret data so we know not to deallocate it when the
1583 * ret is destroyed. The data now belongs to the ret and
1584 * the ret is responsible for its destruction */
1585 data = NULL;
1586}
1587
1588std::ostream &
1589operator <<(std::ostream &os, LSQ::MemoryState state)
1590{
1591 switch (state) {
1592 case LSQ::MemoryRunning:
1593 os << "MemoryRunning";
1594 break;
1595 case LSQ::MemoryNeedsRetry:
1596 os << "MemoryNeedsRetry";
1597 break;
1598 default:
1599 os << "MemoryState-" << static_cast<int>(state);
1600 break;
1601 }
1602 return os;
1603}
1604
1605void
1606LSQ::recvTimingSnoopReq(PacketPtr pkt)
1607{
1608 /* LLSC operations in Minor can't be speculative and are executed from
1609 * the head of the requests queue. We shouldn't need to do more than
1610 * this action on snoops. */
1611
1612 /* THREAD */
1613 TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask);
1614}
1615
1616}
| 1380 {
1381 ret = request;
1382 }
1383 }
1384 }
1385
1386 if (ret) {
1387 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n",
1388 *inst);
1389 } else {
1390 DPRINTF(MinorMem, "No matching memory response for inst: %s\n",
1391 *inst);
1392 }
1393
1394 return ret;
1395}
1396
1397void
1398LSQ::popResponse(LSQ::LSQRequestPtr response)
1399{
1400 assert(!transfers.empty() && transfers.front() == response);
1401
1402 transfers.pop();
1403
1404 if (!response->isLoad)
1405 numStoresInTransfers--;
1406
1407 if (response->issuedToMemory)
1408 numAccessesIssuedToMemory--;
1409
1410 if (response->state != LSQRequest::StoreInStoreBuffer) {
1411 DPRINTF(MinorMem, "Deleting %s request: %s\n",
1412 (response->isLoad ? "load" : "store"),
1413 *(response->inst));
1414
1415 delete response;
1416 }
1417}
1418
1419void
1420LSQ::sendStoreToStoreBuffer(LSQRequestPtr request)
1421{
1422 assert(request->state == LSQRequest::StoreToStoreBuffer);
1423
1424 DPRINTF(MinorMem, "Sending store: %s to store buffer\n",
1425 *(request->inst));
1426
1427 request->inst->inStoreBuffer = true;
1428
1429 storeBuffer.insert(request);
1430}
1431
1432bool
1433LSQ::isDrained()
1434{
1435 return requests.empty() && transfers.empty() &&
1436 storeBuffer.isDrained();
1437}
1438
1439bool
1440LSQ::needsToTick()
1441{
1442 bool ret = false;
1443
1444 if (canSendToMemorySystem()) {
1445 bool have_translated_requests = !requests.empty() &&
1446 requests.front()->state != LSQRequest::InTranslation &&
1447 transfers.unreservedRemainingSpace() != 0;
1448
1449 ret = have_translated_requests ||
1450 storeBuffer.numUnissuedStores() != 0;
1451 }
1452
1453 if (ret)
1454 DPRINTF(Activity, "Need to tick\n");
1455
1456 return ret;
1457}
1458
1459void
1460LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data,
1461 unsigned int size, Addr addr, unsigned int flags, uint64_t *res)
1462{
1463 bool needs_burst = transferNeedsBurst(addr, size, lineWidth);
1464 LSQRequestPtr request;
1465
1466 /* Copy given data into the request. The request will pass this to the
1467 * packet and then it will own the data */
1468 uint8_t *request_data = NULL;
1469
1470 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:"
1471 " 0x%x%s lineWidth : 0x%x\n",
1472 (isLoad ? "load" : "store"), addr, size, flags,
1473 (needs_burst ? " (needs burst)" : ""), lineWidth);
1474
1475 if (!isLoad) {
1476 /* request_data becomes the property of a ...DataRequest (see below)
1477 * and destroyed by its destructor */
1478 request_data = new uint8_t[size];
1479 if (flags & Request::CACHE_BLOCK_ZERO) {
1480 /* For cache zeroing, just use zeroed data */
1481 std::memset(request_data, 0, size);
1482 } else {
1483 std::memcpy(request_data, data, size);
1484 }
1485 }
1486
1487 if (needs_burst) {
1488 request = new SplitDataRequest(
1489 *this, inst, isLoad, request_data, res);
1490 } else {
1491 request = new SingleDataRequest(
1492 *this, inst, isLoad, request_data, res);
1493 }
1494
1495 if (inst->traceData)
1496 inst->traceData->setAddr(addr);
1497
1498 request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0);
1499 request->request.setVirt(0 /* asid */,
1500 addr, size, flags, cpu.instMasterId(),
1501 /* I've no idea why we need the PC, but give it */
1502 inst->pc.instAddr());
1503
1504 requests.push(request);
1505 request->startAddrTranslation();
1506}
1507
1508void
1509LSQ::pushFailedRequest(MinorDynInstPtr inst)
1510{
1511 LSQRequestPtr request = new FailedDataRequest(*this, inst);
1512 requests.push(request);
1513}
1514
1515void
1516LSQ::minorTrace() const
1517{
1518 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d"
1519 " lastMemBarrier=%d\n",
1520 state, numAccessesInDTLB, numAccessesInMemorySystem,
1521 numStoresInTransfers, lastMemBarrier);
1522 requests.minorTrace();
1523 transfers.minorTrace();
1524 storeBuffer.minorTrace();
1525}
1526
1527LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_,
1528 unsigned int store_buffer_size,
1529 unsigned int store_limit_per_cycle) :
1530 Named(name_), lsq(lsq_),
1531 numSlots(store_buffer_size),
1532 storeLimitPerCycle(store_limit_per_cycle),
1533 slots(),
1534 numUnissuedAccesses(0)
1535{
1536}
1537
1538PacketPtr
1539makePacketForRequest(Request &request, bool isLoad,
1540 Packet::SenderState *sender_state, PacketDataPtr data)
1541{
1542 MemCmd command;
1543
1544 /* Make a ret with the right command type to match the request */
1545 if (request.isLLSC()) {
1546 command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq);
1547 } else if (request.isSwap()) {
1548 command = MemCmd::SwapReq;
1549 } else {
1550 command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq);
1551 }
1552
1553 PacketPtr ret = new Packet(&request, command);
1554
1555 if (sender_state)
1556 ret->pushSenderState(sender_state);
1557
1558 if (isLoad)
1559 ret->allocate();
1560 else
1561 ret->dataDynamicArray(data);
1562
1563 return ret;
1564}
1565
1566void
1567LSQ::issuedMemBarrierInst(MinorDynInstPtr inst)
1568{
1569 assert(inst->isInst() && inst->staticInst->isMemBarrier());
1570 assert(inst->id.execSeqNum > lastMemBarrier);
1571
1572 /* Remember the barrier. We only have a notion of one
1573 * barrier so this may result in some mem refs being
1574 * delayed if they are between barriers */
1575 lastMemBarrier = inst->id.execSeqNum;
1576}
1577
1578void
1579LSQ::LSQRequest::makePacket()
1580{
1581 /* Make the function idempotent */
1582 if (packet)
1583 return;
1584
1585 packet = makePacketForRequest(request, isLoad, this, data);
1586 /* Null the ret data so we know not to deallocate it when the
1587 * ret is destroyed. The data now belongs to the ret and
1588 * the ret is responsible for its destruction */
1589 data = NULL;
1590}
1591
1592std::ostream &
1593operator <<(std::ostream &os, LSQ::MemoryState state)
1594{
1595 switch (state) {
1596 case LSQ::MemoryRunning:
1597 os << "MemoryRunning";
1598 break;
1599 case LSQ::MemoryNeedsRetry:
1600 os << "MemoryNeedsRetry";
1601 break;
1602 default:
1603 os << "MemoryState-" << static_cast<int>(state);
1604 break;
1605 }
1606 return os;
1607}
1608
1609void
1610LSQ::recvTimingSnoopReq(PacketPtr pkt)
1611{
1612 /* LLSC operations in Minor can't be speculative and are executed from
1613 * the head of the requests queue. We shouldn't need to do more than
1614 * this action on snoops. */
1615
1616 /* THREAD */
1617 TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask);
1618}
1619
1620}
|