1
2/*
3 * Copyright (c) 2010-2014, 2017-2018 ARM Limited
4 * Copyright (c) 2013 Advanced Micro Devices, Inc.
5 * All rights reserved
6 *
7 * The license below extends only to copyright in the software and shall
8 * not be construed as granting a license to any other intellectual
9 * property including but not limited to intellectual property relating
10 * to a hardware implementation of the functionality of the software
11 * licensed hereunder. You may use the software subject to the license
12 * terms below provided that you ensure that this notice is replicated
13 * unmodified and in its entirety in all distributions of the software,
14 * modified or unmodified, in source code or in binary form.
15 *
16 * Copyright (c) 2004-2005 The Regents of The University of Michigan
17 * All rights reserved.
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions are
21 * met: redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer;
23 * redistributions in binary form must reproduce the above copyright
24 * notice, this list of conditions and the following disclaimer in the
25 * documentation and/or other materials provided with the distribution;
26 * neither the name of the copyright holders nor the names of its
27 * contributors may be used to endorse or promote products derived from
28 * this software without specific prior written permission.
29 *
30 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
31 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
32 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
33 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
34 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
36 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
37 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
38 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
39 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
40 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 *
42 * Authors: Kevin Lim
43 * Korey Sewell
44 */
45
46#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__
47#define __CPU_O3_LSQ_UNIT_IMPL_HH__
48
49#include "arch/generic/debugfaults.hh"
50#include "arch/locked_mem.hh"
51#include "base/str.hh"
52#include "config/the_isa.hh"
53#include "cpu/checker/cpu.hh"
54#include "cpu/o3/lsq.hh"
55#include "cpu/o3/lsq_unit.hh"
56#include "debug/Activity.hh"
57#include "debug/IEW.hh"
58#include "debug/LSQUnit.hh"
59#include "debug/O3PipeView.hh"
60#include "mem/packet.hh"
61#include "mem/request.hh"
62
63template<class Impl>
64LSQUnit<Impl>::WritebackEvent::WritebackEvent(const DynInstPtr &_inst,
65 PacketPtr _pkt, LSQUnit *lsq_ptr)
66 : Event(Default_Pri, AutoDelete),
67 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
68{
69 assert(_inst->savedReq);
70 _inst->savedReq->writebackScheduled();
71}
72
73template<class Impl>
74void
75LSQUnit<Impl>::WritebackEvent::process()
76{
77 assert(!lsqPtr->cpu->switchedOut());
78
79 lsqPtr->writeback(inst, pkt);
80
81 assert(inst->savedReq);
82 inst->savedReq->writebackDone();
83 delete pkt;
84}
85
86template<class Impl>
87const char *
88LSQUnit<Impl>::WritebackEvent::description() const
89{
90 return "Store writeback";
91}
92
93template <class Impl>
94bool
95LSQUnit<Impl>::recvTimingResp(PacketPtr pkt)
96{
97 auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState);
98 LSQRequest* req = senderState->request();
99 assert(req != nullptr);
100 bool ret = true;
101 /* Check that the request is still alive before any further action. */
102 if (senderState->alive()) {
103 ret = req->recvTimingResp(pkt);
104 } else {
105 senderState->outstanding--;
106 }
107 return ret;
108
109}
110
111template<class Impl>
112void
113LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
114{
115 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
116 DynInstPtr inst = state->inst;
117
118 cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
119
120 /* Notify the sender state that the access is complete (for ownership
121 * tracking). */
122 state->complete();
123
124 assert(!cpu->switchedOut());
125 if (!inst->isSquashed()) {
126 if (state->needWB) {
127 // Only loads, store conditionals and atomics perform the writeback
128 // after receving the response from the memory
129 assert(inst->isLoad() || inst->isStoreConditional() ||
130 inst->isAtomic());
131 writeback(inst, state->request()->mainPacket());
132 if (inst->isStore() || inst->isAtomic()) {
133 auto ss = dynamic_cast<SQSenderState*>(state);
134 ss->writebackDone();
135 completeStore(ss->idx);
136 }
137 } else if (inst->isStore()) {
138 // This is a regular store (i.e., not store conditionals and
139 // atomics), so it can complete without writing back
140 completeStore(dynamic_cast<SQSenderState*>(state)->idx);
141 }
142 }
143}
144
145template <class Impl>
146LSQUnit<Impl>::LSQUnit(uint32_t lqEntries, uint32_t sqEntries)
147 : lsqID(-1), storeQueue(sqEntries+1), loadQueue(lqEntries+1),
148 loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
149 isStoreBlocked(false), storeInFlight(false), hasPendingRequest(false),
150 pendingRequest(nullptr)
151{
152}
153
154template<class Impl>
155void
156LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
157 LSQ *lsq_ptr, unsigned id)
158{
159 lsqID = id;
160
161 cpu = cpu_ptr;
162 iewStage = iew_ptr;
163
164 lsq = lsq_ptr;
165
166 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",lsqID);
167
168 depCheckShift = params->LSQDepCheckShift;
169 checkLoads = params->LSQCheckLoads;
170 needsTSO = params->needsTSO;
171
172 resetState();
173}
174
175
176template<class Impl>
177void
178LSQUnit<Impl>::resetState()
179{
180 loads = stores = storesToWB = 0;
181
182
183 storeWBIt = storeQueue.begin();
184
185 retryPkt = NULL;
186 memDepViolator = NULL;
187
188 stalled = false;
189
190 cacheBlockMask = ~(cpu->cacheLineSize() - 1);
191}
192
193template<class Impl>
194std::string
195LSQUnit<Impl>::name() const
196{
197 if (Impl::MaxThreads == 1) {
198 return iewStage->name() + ".lsq";
199 } else {
200 return iewStage->name() + ".lsq.thread" + std::to_string(lsqID);
201 }
202}
203
204template<class Impl>
205void
206LSQUnit<Impl>::regStats()
207{
208 lsqForwLoads
209 .name(name() + ".forwLoads")
210 .desc("Number of loads that had data forwarded from stores");
211
212 invAddrLoads
213 .name(name() + ".invAddrLoads")
214 .desc("Number of loads ignored due to an invalid address");
215
216 lsqSquashedLoads
217 .name(name() + ".squashedLoads")
218 .desc("Number of loads squashed");
219
220 lsqIgnoredResponses
221 .name(name() + ".ignoredResponses")
222 .desc("Number of memory responses ignored because the instruction is squashed");
223
224 lsqMemOrderViolation
225 .name(name() + ".memOrderViolation")
226 .desc("Number of memory ordering violations");
227
228 lsqSquashedStores
229 .name(name() + ".squashedStores")
230 .desc("Number of stores squashed");
231
232 invAddrSwpfs
233 .name(name() + ".invAddrSwpfs")
234 .desc("Number of software prefetches ignored due to an invalid address");
235
236 lsqBlockedLoads
237 .name(name() + ".blockedLoads")
238 .desc("Number of blocked loads due to partial load-store forwarding");
239
240 lsqRescheduledLoads
241 .name(name() + ".rescheduledLoads")
242 .desc("Number of loads that were rescheduled");
243
244 lsqCacheBlocked
245 .name(name() + ".cacheBlocked")
246 .desc("Number of times an access to memory failed due to the cache being blocked");
247}
248
249template<class Impl>
250void
251LSQUnit<Impl>::setDcachePort(MasterPort *dcache_port)
252{
253 dcachePort = dcache_port;
254}
255
256template<class Impl>
257void
258LSQUnit<Impl>::drainSanityCheck() const
259{
260 for (int i = 0; i < loadQueue.capacity(); ++i)
261 assert(!loadQueue[i].valid());
262
263 assert(storesToWB == 0);
264 assert(!retryPkt);
265}
266
267template<class Impl>
268void
269LSQUnit<Impl>::takeOverFrom()
270{
271 resetState();
272}
273
274template <class Impl>
275void
276LSQUnit<Impl>::insert(const DynInstPtr &inst)
277{
278 assert(inst->isMemRef());
279
280 assert(inst->isLoad() || inst->isStore() || inst->isAtomic());
281
282 if (inst->isLoad()) {
283 insertLoad(inst);
284 } else {
285 insertStore(inst);
286 }
287
288 inst->setInLSQ();
289}
290
291template <class Impl>
292void
293LSQUnit<Impl>::insertLoad(const DynInstPtr &load_inst)
294{
295 assert(!loadQueue.full());
296 assert(loads < loadQueue.capacity());
297
298 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
299 load_inst->pcState(), loadQueue.tail(), load_inst->seqNum);
300
301 /* Grow the queue. */
302 loadQueue.advance_tail();
303
304 load_inst->sqIt = storeQueue.end();
305
306 assert(!loadQueue.back().valid());
307 loadQueue.back().set(load_inst);
308 load_inst->lqIdx = loadQueue.tail();
309 load_inst->lqIt = loadQueue.getIterator(load_inst->lqIdx);
310
311 ++loads;
312}
313
314template <class Impl>
315void
316LSQUnit<Impl>::insertStore(const DynInstPtr& store_inst)
317{
318 // Make sure it is not full before inserting an instruction.
319 assert(!storeQueue.full());
320 assert(stores < storeQueue.capacity());
321
322 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
323 store_inst->pcState(), storeQueue.tail(), store_inst->seqNum);
324 storeQueue.advance_tail();
325
326 store_inst->sqIdx = storeQueue.tail();
327 store_inst->lqIdx = loadQueue.moduloAdd(loadQueue.tail(), 1);
328 store_inst->lqIt = loadQueue.end();
329
330 storeQueue.back().set(store_inst);
331
332 ++stores;
333}
334
335template <class Impl>
336typename Impl::DynInstPtr
337LSQUnit<Impl>::getMemDepViolator()
338{
339 DynInstPtr temp = memDepViolator;
340
341 memDepViolator = NULL;
342
343 return temp;
344}
345
346template <class Impl>
347unsigned
348LSQUnit<Impl>::numFreeLoadEntries()
349{
350 //LQ has an extra dummy entry to differentiate
351 //empty/full conditions. Subtract 1 from the free entries.
352 DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n",
353 1 + loadQueue.capacity(), loads);
354 return loadQueue.capacity() - loads;
355}
356
357template <class Impl>
358unsigned
359LSQUnit<Impl>::numFreeStoreEntries()
360{
361 //SQ has an extra dummy entry to differentiate
362 //empty/full conditions. Subtract 1 from the free entries.
363 DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n",
364 1 + storeQueue.capacity(), stores);
365 return storeQueue.capacity() - stores;
366
367 }
368
369template <class Impl>
370void
371LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
372{
373 // Should only ever get invalidations in here
374 assert(pkt->isInvalidate());
375
376 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
377
378 for (int x = 0; x < cpu->numContexts(); x++) {
379 ThreadContext *tc = cpu->getContext(x);
380 bool no_squash = cpu->thread[x]->noSquashFromTC;
381 cpu->thread[x]->noSquashFromTC = true;
382 TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask);
383 cpu->thread[x]->noSquashFromTC = no_squash;
384 }
385
386 if (loadQueue.empty())
387 return;
388
389 auto iter = loadQueue.begin();
390
391 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
392
393 DynInstPtr ld_inst = iter->instruction();
394 assert(ld_inst);
395 LSQRequest *req = iter->request();
396
397 // Check that this snoop didn't just invalidate our lock flag
398 if (ld_inst->effAddrValid() &&
399 req->isCacheBlockHit(invalidate_addr, cacheBlockMask)
400 && ld_inst->memReqFlags & Request::LLSC)
401 TheISA::handleLockedSnoopHit(ld_inst.get());
402
403 bool force_squash = false;
404
405 while (++iter != loadQueue.end()) {
406 ld_inst = iter->instruction();
407 assert(ld_inst);
408 req = iter->request();
409 if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered())
410 continue;
411
412 DPRINTF(LSQUnit, "-- inst [sn:%lli] to pktAddr:%#x\n",
413 ld_inst->seqNum, invalidate_addr);
414
415 if (force_squash ||
416 req->isCacheBlockHit(invalidate_addr, cacheBlockMask)) {
417 if (needsTSO) {
418 // If we have a TSO system, as all loads must be ordered with
419 // all other loads, this load as well as *all* subsequent loads
420 // need to be squashed to prevent possible load reordering.
421 force_squash = true;
422 }
423 if (ld_inst->possibleLoadViolation() || force_squash) {
424 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
425 pkt->getAddr(), ld_inst->seqNum);
426
427 // Mark the load for re-execution
428 ld_inst->fault = std::make_shared<ReExec>();
429 } else {
430 DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n",
431 pkt->getAddr(), ld_inst->seqNum);
432
433 // Make sure that we don't lose a snoop hitting a LOCKED
434 // address since the LOCK* flags don't get updated until
435 // commit.
436 if (ld_inst->memReqFlags & Request::LLSC)
437 TheISA::handleLockedSnoopHit(ld_inst.get());
438
439 // If a older load checks this and it's true
440 // then we might have missed the snoop
441 // in which case we need to invalidate to be sure
442 ld_inst->hitExternalSnoop(true);
443 }
444 }
445 }
446 return;
447}
448
449template <class Impl>
450Fault
451LSQUnit<Impl>::checkViolations(typename LoadQueue::iterator& loadIt,
452 const DynInstPtr& inst)
453{
454 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
455 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
456
457 /** @todo in theory you only need to check an instruction that has executed
458 * however, there isn't a good way in the pipeline at the moment to check
459 * all instructions that will execute before the store writes back. Thus,
460 * like the implementation that came before it, we're overly conservative.
461 */
462 while (loadIt != loadQueue.end()) {
463 DynInstPtr ld_inst = loadIt->instruction();
464 if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
465 ++loadIt;
466 continue;
467 }
468
469 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
470 Addr ld_eff_addr2 =
471 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
472
473 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
474 if (inst->isLoad()) {
475 // If this load is to the same block as an external snoop
476 // invalidate that we've observed then the load needs to be
477 // squashed as it could have newer data
478 if (ld_inst->hitExternalSnoop()) {
479 if (!memDepViolator ||
480 ld_inst->seqNum < memDepViolator->seqNum) {
481 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
482 "and [sn:%lli] at address %#x\n",
483 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
484 memDepViolator = ld_inst;
485
486 ++lsqMemOrderViolation;
487
488 return std::make_shared<GenericISA::M5PanicFault>(
489 "Detected fault with inst [sn:%lli] and "
490 "[sn:%lli] at address %#x\n",
491 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
492 }
493 }
494
495 // Otherwise, mark the load has a possible load violation
496 // and if we see a snoop before it's commited, we need to squash
497 ld_inst->possibleLoadViolation(true);
498 DPRINTF(LSQUnit, "Found possible load violation at addr: %#x"
499 " between instructions [sn:%lli] and [sn:%lli]\n",
500 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
501 } else {
502 // A load/store incorrectly passed this store.
503 // Check if we already have a violator, or if it's newer
504 // squash and refetch.
505 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
506 break;
507
508 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
509 "[sn:%lli] at address %#x\n",
510 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
511 memDepViolator = ld_inst;
512
513 ++lsqMemOrderViolation;
514
515 return std::make_shared<GenericISA::M5PanicFault>(
516 "Detected fault with "
517 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
518 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
519 }
520 }
521
522 ++loadIt;
523 }
524 return NoFault;
525}
526
527
528
529
530template <class Impl>
531Fault
532LSQUnit<Impl>::executeLoad(const DynInstPtr &inst)
533{
534 using namespace TheISA;
535 // Execute a specific load.
536 Fault load_fault = NoFault;
537
538 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
539 inst->pcState(), inst->seqNum);
540
541 assert(!inst->isSquashed());
542
543 load_fault = inst->initiateAcc();
544
545 if (load_fault == NoFault && !inst->readMemAccPredicate()) {
546 assert(inst->readPredicate());
547 inst->setExecuted();
548 inst->completeAcc(nullptr);
549 iewStage->instToCommit(inst);
550 iewStage->activityThisCycle();
551 return NoFault;
552 }
553
554 if (inst->isTranslationDelayed() && load_fault == NoFault)
555 return load_fault;
556
557 if (load_fault != NoFault && inst->translationCompleted() &&
558 inst->savedReq->isPartialFault() && !inst->savedReq->isComplete()) {
559 assert(inst->savedReq->isSplit());
560 // If we have a partial fault where the mem access is not complete yet
561 // then the cache must have been blocked. This load will be re-executed
562 // when the cache gets unblocked. We will handle the fault when the
563 // mem access is complete.
564 return NoFault;
565 }
566
567 // If the instruction faulted or predicated false, then we need to send it
568 // along to commit without the instruction completing.
569 if (load_fault != NoFault || !inst->readPredicate()) {
570 // Send this instruction to commit, also make sure iew stage
571 // realizes there is activity. Mark it as executed unless it
572 // is a strictly ordered load that needs to hit the head of
573 // commit.
574 if (!inst->readPredicate())
575 inst->forwardOldRegs();
576 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
577 inst->seqNum,
578 (load_fault != NoFault ? "fault" : "predication"));
579 if (!(inst->hasRequest() && inst->strictlyOrdered()) ||
580 inst->isAtCommit()) {
581 inst->setExecuted();
582 }
583 iewStage->instToCommit(inst);
584 iewStage->activityThisCycle();
585 } else {
586 if (inst->effAddrValid()) {
587 auto it = inst->lqIt;
588 ++it;
589
590 if (checkLoads)
591 return checkViolations(it, inst);
592 }
593 }
594
595 return load_fault;
596}
597
598template <class Impl>
599Fault
600LSQUnit<Impl>::executeStore(const DynInstPtr &store_inst)
601{
602 using namespace TheISA;
603 // Make sure that a store exists.
604 assert(stores != 0);
605
606 int store_idx = store_inst->sqIdx;
607
608 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
609 store_inst->pcState(), store_inst->seqNum);
610
611 assert(!store_inst->isSquashed());
612
613 // Check the recently completed loads to see if any match this store's
614 // address. If so, then we have a memory ordering violation.
615 typename LoadQueue::iterator loadIt = store_inst->lqIt;
616
617 Fault store_fault = store_inst->initiateAcc();
618
619 if (store_inst->isTranslationDelayed() &&
620 store_fault == NoFault)
621 return store_fault;
622
623 if (!store_inst->readPredicate()) {
624 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
625 store_inst->seqNum);
626 store_inst->forwardOldRegs();
627 return store_fault;
628 }
629
630 if (storeQueue[store_idx].size() == 0) {
631 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
632 store_inst->pcState(), store_inst->seqNum);
633
634 return store_fault;
635 }
636
637 assert(store_fault == NoFault);
638
639 if (store_inst->isStoreConditional() || store_inst->isAtomic()) {
640 // Store conditionals and Atomics need to set themselves as able to
641 // writeback if we haven't had a fault by here.
642 storeQueue[store_idx].canWB() = true;
643
644 ++storesToWB;
645 }
646
647 return checkViolations(loadIt, store_inst);
648
649}
650
651template <class Impl>
652void
653LSQUnit<Impl>::commitLoad()
654{
655 assert(loadQueue.front().valid());
656
657 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
658 loadQueue.front().instruction()->pcState());
659
660 loadQueue.front().clear();
661 loadQueue.pop_front();
662
663 --loads;
664}
665
666template <class Impl>
667void
668LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
669{
670 assert(loads == 0 || loadQueue.front().valid());
671
672 while (loads != 0 && loadQueue.front().instruction()->seqNum
673 <= youngest_inst) {
674 commitLoad();
675 }
676}
677
678template <class Impl>
679void
680LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
681{
682 assert(stores == 0 || storeQueue.front().valid());
683
684 /* Forward iterate the store queue (age order). */
685 for (auto& x : storeQueue) {
686 assert(x.valid());
687 // Mark any stores that are now committed and have not yet
688 // been marked as able to write back.
689 if (!x.canWB()) {
690 if (x.instruction()->seqNum > youngest_inst) {
691 break;
692 }
693 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
694 "%s [sn:%lli]\n",
695 x.instruction()->pcState(),
696 x.instruction()->seqNum);
697
698 x.canWB() = true;
699
700 ++storesToWB;
701 }
702 }
703}
704
705template <class Impl>
706void
707LSQUnit<Impl>::writebackBlockedStore()
708{
709 assert(isStoreBlocked);
710 storeWBIt->request()->sendPacketToCache();
711 if (storeWBIt->request()->isSent()){
712 storePostSend();
713 }
714}
715
716template <class Impl>
717void
718LSQUnit<Impl>::writebackStores()
719{
720 if (isStoreBlocked) {
721 DPRINTF(LSQUnit, "Writing back blocked store\n");
722 writebackBlockedStore();
723 }
724
725 while (storesToWB > 0 &&
726 storeWBIt.dereferenceable() &&
727 storeWBIt->valid() &&
728 storeWBIt->canWB() &&
729 ((!needsTSO) || (!storeInFlight)) &&
730 lsq->cachePortAvailable(false)) {
731
732 if (isStoreBlocked) {
733 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
734 " is blocked!\n");
735 break;
736 }
737
738 // Store didn't write any data so no need to write it back to
739 // memory.
740 if (storeWBIt->size() == 0) {
741 /* It is important that the preincrement happens at (or before)
742 * the call, as the the code of completeStore checks
743 * storeWBIt. */
744 completeStore(storeWBIt++);
745 continue;
746 }
747
748 if (storeWBIt->instruction()->isDataPrefetch()) {
749 storeWBIt++;
750 continue;
751 }
752
753 assert(storeWBIt->hasRequest());
754 assert(!storeWBIt->committed());
755
756 DynInstPtr inst = storeWBIt->instruction();
757 LSQRequest* req = storeWBIt->request();
758 storeWBIt->committed() = true;
759
760 assert(!inst->memData);
761 inst->memData = new uint8_t[req->_size];
762
763 if (storeWBIt->isAllZeros())
764 memset(inst->memData, 0, req->_size);
765 else
766 memcpy(inst->memData, storeWBIt->data(), req->_size);
767
768
769 if (req->senderState() == nullptr) {
770 SQSenderState *state = new SQSenderState(storeWBIt);
771 state->isLoad = false;
772 state->needWB = false;
773 state->inst = inst;
774
775 req->senderState(state);
776 if (inst->isStoreConditional() || inst->isAtomic()) {
777 /* Only store conditionals and atomics need a writeback. */
778 state->needWB = true;
779 }
780 }
781 req->buildPackets();
782
783 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
784 "to Addr:%#x, data:%#x [sn:%lli]\n",
785 storeWBIt.idx(), inst->pcState(),
786 req->request()->getPaddr(), (int)*(inst->memData),
787 inst->seqNum);
788
789 // @todo: Remove this SC hack once the memory system handles it.
790 if (inst->isStoreConditional()) {
791 // Disable recording the result temporarily. Writing to
792 // misc regs normally updates the result, but this is not
793 // the desired behavior when handling store conditionals.
794 inst->recordResult(false);
795 bool success = TheISA::handleLockedWrite(inst.get(),
796 req->request(), cacheBlockMask);
797 inst->recordResult(true);
798 req->packetSent();
799
800 if (!success) {
801 req->complete();
802 // Instantly complete this store.
803 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
804 "Instantly completing it.\n",
805 inst->seqNum);
806 PacketPtr new_pkt = new Packet(*req->packet());
807 WritebackEvent *wb = new WritebackEvent(inst,
808 new_pkt, this);
809 cpu->schedule(wb, curTick() + 1);
810 completeStore(storeWBIt);
811 if (!storeQueue.empty())
812 storeWBIt++;
813 else
814 storeWBIt = storeQueue.end();
815 continue;
816 }
817 }
818
819 if (req->request()->isMmappedIpr()) {
820 assert(!inst->isStoreConditional());
821 ThreadContext *thread = cpu->tcBase(lsqID);
822 PacketPtr main_pkt = new Packet(req->mainRequest(),
823 MemCmd::WriteReq);
824 main_pkt->dataStatic(inst->memData);
825 req->handleIprWrite(thread, main_pkt);
826 delete main_pkt;
827 completeStore(storeWBIt);
828 storeWBIt++;
829 continue;
830 }
831 /* Send to cache */
832 req->sendPacketToCache();
833
834 /* If successful, do the post send */
835 if (req->isSent()) {
836 storePostSend();
837 } else {
838 DPRINTF(LSQUnit, "D-Cache became blocked when writing [sn:%lli], "
839 "will retry later\n",
840 inst->seqNum);
841 }
842 }
843 assert(stores >= 0 && storesToWB >= 0);
844}
845
846template <class Impl>
847void
848LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
849{
850 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
851 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
852
853 while (loads != 0 &&
854 loadQueue.back().instruction()->seqNum > squashed_num) {
855 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
856 "[sn:%lli]\n",
857 loadQueue.back().instruction()->pcState(),
858 loadQueue.back().instruction()->seqNum);
859
860 if (isStalled() && loadQueue.tail() == stallingLoadIdx) {
861 stalled = false;
862 stallingStoreIsn = 0;
863 stallingLoadIdx = 0;
864 }
865
866 // Clear the smart pointer to make sure it is decremented.
867 loadQueue.back().instruction()->setSquashed();
868 loadQueue.back().clear();
869
870 --loads;
871
872 loadQueue.pop_back();
873 ++lsqSquashedLoads;
874 }
875
876 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
877 memDepViolator = NULL;
878 }
879
880 while (stores != 0 &&
881 storeQueue.back().instruction()->seqNum > squashed_num) {
882 // Instructions marked as can WB are already committed.
883 if (storeQueue.back().canWB()) {
884 break;
885 }
886
887 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
888 "idx:%i [sn:%lli]\n",
889 storeQueue.back().instruction()->pcState(),
890 storeQueue.tail(), storeQueue.back().instruction()->seqNum);
891
892 // I don't think this can happen. It should have been cleared
893 // by the stalling load.
894 if (isStalled() &&
895 storeQueue.back().instruction()->seqNum == stallingStoreIsn) {
896 panic("Is stalled should have been cleared by stalling load!\n");
897 stalled = false;
898 stallingStoreIsn = 0;
899 }
900
901 // Clear the smart pointer to make sure it is decremented.
902 storeQueue.back().instruction()->setSquashed();
903
904 // Must delete request now that it wasn't handed off to
905 // memory. This is quite ugly. @todo: Figure out the proper
906 // place to really handle request deletes.
907 storeQueue.back().clear();
908 --stores;
909
910 storeQueue.pop_back();
911 ++lsqSquashedStores;
912 }
913}
914
915template <class Impl>
916void
917LSQUnit<Impl>::storePostSend()
918{
919 if (isStalled() &&
920 storeWBIt->instruction()->seqNum == stallingStoreIsn) {
921 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
922 "load idx:%i\n",
923 stallingStoreIsn, stallingLoadIdx);
924 stalled = false;
925 stallingStoreIsn = 0;
926 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
927 }
928
929 if (!storeWBIt->instruction()->isStoreConditional()) {
930 // The store is basically completed at this time. This
931 // only works so long as the checker doesn't try to
932 // verify the value in memory for stores.
933 storeWBIt->instruction()->setCompleted();
934
935 if (cpu->checker) {
936 cpu->checker->verify(storeWBIt->instruction());
937 }
938 }
939
940 if (needsTSO) {
941 storeInFlight = true;
942 }
943
944 storeWBIt++;
945}
946
947template <class Impl>
948void
949LSQUnit<Impl>::writeback(const DynInstPtr &inst, PacketPtr pkt)
950{
951 iewStage->wakeCPU();
952
953 // Squashed instructions do not need to complete their access.
954 if (inst->isSquashed()) {
955 assert(!inst->isStore());
956 ++lsqIgnoredResponses;
957 return;
958 }
959
960 if (!inst->isExecuted()) {
961 inst->setExecuted();
962
963 if (inst->fault == NoFault) {
964 // Complete access to copy data to proper place.
965 inst->completeAcc(pkt);
966 } else {
967 // If the instruction has an outstanding fault, we cannot complete
968 // the access as this discards the current fault.
969
970 // If we have an outstanding fault, the fault should only be of
971 // type ReExec or - in case of a SplitRequest - a partial
972 // translation fault
973 assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr ||
974 inst->savedReq->isPartialFault());
975
976 DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access "
977 "due to pending fault.\n", inst->seqNum);
978 }
979 }
980
981 // Need to insert instruction into queue to commit
982 iewStage->instToCommit(inst);
983
984 iewStage->activityThisCycle();
985
986 // see if this load changed the PC
987 iewStage->checkMisprediction(inst);
988}
989
990template <class Impl>
991void
992LSQUnit<Impl>::completeStore(typename StoreQueue::iterator store_idx)
993{
994 assert(store_idx->valid());
995 store_idx->completed() = true;
996 --storesToWB;
997 // A bit conservative because a store completion may not free up entries,
998 // but hopefully avoids two store completions in one cycle from making
999 // the CPU tick twice.
1000 cpu->wakeCPU();
1001 cpu->activityThisCycle();
1002
1003 /* We 'need' a copy here because we may clear the entry from the
1004 * store queue. */
1005 DynInstPtr store_inst = store_idx->instruction();
1006 if (store_idx == storeQueue.begin()) {
1007 do {
1008 storeQueue.front().clear();
1009 storeQueue.pop_front();
1010 --stores;
1011 } while (storeQueue.front().completed() &&
1012 !storeQueue.empty());
1013
1014 iewStage->updateLSQNextCycle = true;
1015 }
1016
1017 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1018 "idx:%i\n",
1019 store_inst->seqNum, store_idx.idx() - 1, storeQueue.head() - 1);
1020
1021#if TRACING_ON
1022 if (DTRACE(O3PipeView)) {
1023 store_inst->storeTick =
1024 curTick() - store_inst->fetchTick;
1025 }
1026#endif
1027
1028 if (isStalled() &&
1029 store_inst->seqNum == stallingStoreIsn) {
1030 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1031 "load idx:%i\n",
1032 stallingStoreIsn, stallingLoadIdx);
1033 stalled = false;
1034 stallingStoreIsn = 0;
1035 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction());
1036 }
1037
1038 store_inst->setCompleted();
1039
1040 if (needsTSO) {
1041 storeInFlight = false;
1042 }
1043
1044 // Tell the checker we've completed this instruction. Some stores
1045 // may get reported twice to the checker, but the checker can
1046 // handle that case.
1047 // Store conditionals cannot be sent to the checker yet, they have
1048 // to update the misc registers first which should take place
1049 // when they commit
1050 if (cpu->checker && !store_inst->isStoreConditional()) {
1051 cpu->checker->verify(store_inst);
1052 }
1053}
1054
1055template <class Impl>
1056bool
1057LSQUnit<Impl>::trySendPacket(bool isLoad, PacketPtr data_pkt)
1058{
1059 bool ret = true;
1060 bool cache_got_blocked = false;
1061
1062 auto state = dynamic_cast<LSQSenderState*>(data_pkt->senderState);
1063
1064 if (!lsq->cacheBlocked() &&
1065 lsq->cachePortAvailable(isLoad)) {
1066 if (!dcachePort->sendTimingReq(data_pkt)) {
1067 ret = false;
1068 cache_got_blocked = true;
1069 }
1070 } else {
1071 ret = false;
1072 }
1073
1074 if (ret) {
1075 if (!isLoad) {
1076 isStoreBlocked = false;
1077 }
1078 lsq->cachePortBusy(isLoad);
1079 state->outstanding++;
1080 state->request()->packetSent();
1081 } else {
1082 if (cache_got_blocked) {
1083 lsq->cacheBlocked(true);
1084 ++lsqCacheBlocked;
1085 }
1086 if (!isLoad) {
1087 assert(state->request() == storeWBIt->request());
1088 isStoreBlocked = true;
1089 }
1090 state->request()->packetNotSent();
1091 }
1092 return ret;
1093}
1094
1095template <class Impl>
1096void
1097LSQUnit<Impl>::recvRetry()
1098{
1099 if (isStoreBlocked) {
1100 DPRINTF(LSQUnit, "Receiving retry: blocked store\n");
1101 writebackBlockedStore();
1102 }
1103}
1104
1105template <class Impl>
1106void
1107LSQUnit<Impl>::dumpInsts() const
1108{
1109 cprintf("Load store queue: Dumping instructions.\n");
1110 cprintf("Load queue size: %i\n", loads);
1111 cprintf("Load queue: ");
1112
1113 for (const auto& e: loadQueue) {
1114 const DynInstPtr &inst(e.instruction());
1115 cprintf("%s.[sn:%llu] ", inst->pcState(), inst->seqNum);
1116 }
1117 cprintf("\n");
1118
1119 cprintf("Store queue size: %i\n", stores);
1120 cprintf("Store queue: ");
1121
1122 for (const auto& e: storeQueue) {
1123 const DynInstPtr &inst(e.instruction());
1124 cprintf("%s.[sn:%llu] ", inst->pcState(), inst->seqNum);
1125 }
1126
1127 cprintf("\n");
1128}
1129
1130template <class Impl>
1131unsigned int
1132LSQUnit<Impl>::cacheLineSize()
1133{
1134 return cpu->cacheLineSize();
1135}
1136
1137#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__