1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Kevin Lim
29 * Korey Sewell
30 */
31
32#include "arch/locked_mem.hh"
33#include "config/use_checker.hh"
34
35#include "cpu/o3/lsq.hh"
36#include "cpu/o3/lsq_unit.hh"
37#include "base/str.hh"
38#include "mem/packet.hh"
39#include "mem/request.hh"
40
41#if USE_CHECKER
42#include "cpu/checker/cpu.hh"
43#endif
44
45template<class Impl>
46LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
47 LSQUnit *lsq_ptr)
48 : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
49{
50 this->setFlags(Event::AutoDelete);
51}
52
53template<class Impl>
54void
55LSQUnit<Impl>::WritebackEvent::process()
56{
57 if (!lsqPtr->isSwitchedOut()) {
58 lsqPtr->writeback(inst, pkt);
59 }
60
61 if (pkt->senderState)
62 delete pkt->senderState;
63
64 delete pkt->req;
65 delete pkt;
66}
67
68template<class Impl>
69const char *
| 1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Kevin Lim
29 * Korey Sewell
30 */
31
32#include "arch/locked_mem.hh"
33#include "config/use_checker.hh"
34
35#include "cpu/o3/lsq.hh"
36#include "cpu/o3/lsq_unit.hh"
37#include "base/str.hh"
38#include "mem/packet.hh"
39#include "mem/request.hh"
40
41#if USE_CHECKER
42#include "cpu/checker/cpu.hh"
43#endif
44
45template<class Impl>
46LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
47 LSQUnit *lsq_ptr)
48 : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
49{
50 this->setFlags(Event::AutoDelete);
51}
52
53template<class Impl>
54void
55LSQUnit<Impl>::WritebackEvent::process()
56{
57 if (!lsqPtr->isSwitchedOut()) {
58 lsqPtr->writeback(inst, pkt);
59 }
60
61 if (pkt->senderState)
62 delete pkt->senderState;
63
64 delete pkt->req;
65 delete pkt;
66}
67
68template<class Impl>
69const char *
|
71{
72 return "Store writeback";
73}
74
75template<class Impl>
76void
77LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
78{
79 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
80 DynInstPtr inst = state->inst;
81 DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
82 DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
83
84 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
85
86 assert(!pkt->wasNacked());
87
88 if (isSwitchedOut() || inst->isSquashed()) {
89 iewStage->decrWb(inst->seqNum);
90 } else {
91 if (!state->noWB) {
92 writeback(inst, pkt);
93 }
94
95 if (inst->isStore()) {
96 completeStore(state->idx);
97 }
98 }
99
100 delete state;
101 delete pkt->req;
102 delete pkt;
103}
104
105template <class Impl>
106LSQUnit<Impl>::LSQUnit()
107 : loads(0), stores(0), storesToWB(0), stalled(false),
108 isStoreBlocked(false), isLoadBlocked(false),
109 loadBlockedHandled(false)
110{
111}
112
113template<class Impl>
114void
115LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, Params *params, LSQ *lsq_ptr,
116 unsigned maxLQEntries, unsigned maxSQEntries, unsigned id)
117{
118 cpu = cpu_ptr;
119 iewStage = iew_ptr;
120
121 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
122
123 switchedOut = false;
124
125 lsq = lsq_ptr;
126
127 lsqID = id;
128
129 // Add 1 for the sentinel entry (they are circular queues).
130 LQEntries = maxLQEntries + 1;
131 SQEntries = maxSQEntries + 1;
132
133 loadQueue.resize(LQEntries);
134 storeQueue.resize(SQEntries);
135
136 loadHead = loadTail = 0;
137
138 storeHead = storeWBIdx = storeTail = 0;
139
140 usedPorts = 0;
141 cachePorts = params->cachePorts;
142
143 retryPkt = NULL;
144 memDepViolator = NULL;
145
146 blockedLoadSeqNum = 0;
147}
148
149template<class Impl>
150std::string
151LSQUnit<Impl>::name() const
152{
153 if (Impl::MaxThreads == 1) {
154 return iewStage->name() + ".lsq";
155 } else {
156 return iewStage->name() + ".lsq.thread." + to_string(lsqID);
157 }
158}
159
160template<class Impl>
161void
162LSQUnit<Impl>::regStats()
163{
164 lsqForwLoads
165 .name(name() + ".forwLoads")
166 .desc("Number of loads that had data forwarded from stores");
167
168 invAddrLoads
169 .name(name() + ".invAddrLoads")
170 .desc("Number of loads ignored due to an invalid address");
171
172 lsqSquashedLoads
173 .name(name() + ".squashedLoads")
174 .desc("Number of loads squashed");
175
176 lsqIgnoredResponses
177 .name(name() + ".ignoredResponses")
178 .desc("Number of memory responses ignored because the instruction is squashed");
179
180 lsqMemOrderViolation
181 .name(name() + ".memOrderViolation")
182 .desc("Number of memory ordering violations");
183
184 lsqSquashedStores
185 .name(name() + ".squashedStores")
186 .desc("Number of stores squashed");
187
188 invAddrSwpfs
189 .name(name() + ".invAddrSwpfs")
190 .desc("Number of software prefetches ignored due to an invalid address");
191
192 lsqBlockedLoads
193 .name(name() + ".blockedLoads")
194 .desc("Number of blocked loads due to partial load-store forwarding");
195
196 lsqRescheduledLoads
197 .name(name() + ".rescheduledLoads")
198 .desc("Number of loads that were rescheduled");
199
200 lsqCacheBlocked
201 .name(name() + ".cacheBlocked")
202 .desc("Number of times an access to memory failed due to the cache being blocked");
203}
204
205template<class Impl>
206void
207LSQUnit<Impl>::setDcachePort(Port *dcache_port)
208{
209 dcachePort = dcache_port;
210
211#if USE_CHECKER
212 if (cpu->checker) {
213 cpu->checker->setDcachePort(dcachePort);
214 }
215#endif
216}
217
218template<class Impl>
219void
220LSQUnit<Impl>::clearLQ()
221{
222 loadQueue.clear();
223}
224
225template<class Impl>
226void
227LSQUnit<Impl>::clearSQ()
228{
229 storeQueue.clear();
230}
231
232template<class Impl>
233void
234LSQUnit<Impl>::switchOut()
235{
236 switchedOut = true;
237 for (int i = 0; i < loadQueue.size(); ++i) {
238 assert(!loadQueue[i]);
239 loadQueue[i] = NULL;
240 }
241
242 assert(storesToWB == 0);
243}
244
245template<class Impl>
246void
247LSQUnit<Impl>::takeOverFrom()
248{
249 switchedOut = false;
250 loads = stores = storesToWB = 0;
251
252 loadHead = loadTail = 0;
253
254 storeHead = storeWBIdx = storeTail = 0;
255
256 usedPorts = 0;
257
258 memDepViolator = NULL;
259
260 blockedLoadSeqNum = 0;
261
262 stalled = false;
263 isLoadBlocked = false;
264 loadBlockedHandled = false;
265}
266
267template<class Impl>
268void
269LSQUnit<Impl>::resizeLQ(unsigned size)
270{
271 unsigned size_plus_sentinel = size + 1;
272 assert(size_plus_sentinel >= LQEntries);
273
274 if (size_plus_sentinel > LQEntries) {
275 while (size_plus_sentinel > loadQueue.size()) {
276 DynInstPtr dummy;
277 loadQueue.push_back(dummy);
278 LQEntries++;
279 }
280 } else {
281 LQEntries = size_plus_sentinel;
282 }
283
284}
285
286template<class Impl>
287void
288LSQUnit<Impl>::resizeSQ(unsigned size)
289{
290 unsigned size_plus_sentinel = size + 1;
291 if (size_plus_sentinel > SQEntries) {
292 while (size_plus_sentinel > storeQueue.size()) {
293 SQEntry dummy;
294 storeQueue.push_back(dummy);
295 SQEntries++;
296 }
297 } else {
298 SQEntries = size_plus_sentinel;
299 }
300}
301
302template <class Impl>
303void
304LSQUnit<Impl>::insert(DynInstPtr &inst)
305{
306 assert(inst->isMemRef());
307
308 assert(inst->isLoad() || inst->isStore());
309
310 if (inst->isLoad()) {
311 insertLoad(inst);
312 } else {
313 insertStore(inst);
314 }
315
316 inst->setInLSQ();
317}
318
319template <class Impl>
320void
321LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
322{
323 assert((loadTail + 1) % LQEntries != loadHead);
324 assert(loads < LQEntries);
325
326 DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
327 load_inst->readPC(), loadTail, load_inst->seqNum);
328
329 load_inst->lqIdx = loadTail;
330
331 if (stores == 0) {
332 load_inst->sqIdx = -1;
333 } else {
334 load_inst->sqIdx = storeTail;
335 }
336
337 loadQueue[loadTail] = load_inst;
338
339 incrLdIdx(loadTail);
340
341 ++loads;
342}
343
344template <class Impl>
345void
346LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
347{
348 // Make sure it is not full before inserting an instruction.
349 assert((storeTail + 1) % SQEntries != storeHead);
350 assert(stores < SQEntries);
351
352 DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
353 store_inst->readPC(), storeTail, store_inst->seqNum);
354
355 store_inst->sqIdx = storeTail;
356 store_inst->lqIdx = loadTail;
357
358 storeQueue[storeTail] = SQEntry(store_inst);
359
360 incrStIdx(storeTail);
361
362 ++stores;
363}
364
365template <class Impl>
366typename Impl::DynInstPtr
367LSQUnit<Impl>::getMemDepViolator()
368{
369 DynInstPtr temp = memDepViolator;
370
371 memDepViolator = NULL;
372
373 return temp;
374}
375
376template <class Impl>
377unsigned
378LSQUnit<Impl>::numFreeEntries()
379{
380 unsigned free_lq_entries = LQEntries - loads;
381 unsigned free_sq_entries = SQEntries - stores;
382
383 // Both the LQ and SQ entries have an extra dummy entry to differentiate
384 // empty/full conditions. Subtract 1 from the free entries.
385 if (free_lq_entries < free_sq_entries) {
386 return free_lq_entries - 1;
387 } else {
388 return free_sq_entries - 1;
389 }
390}
391
392template <class Impl>
393int
394LSQUnit<Impl>::numLoadsReady()
395{
396 int load_idx = loadHead;
397 int retval = 0;
398
399 while (load_idx != loadTail) {
400 assert(loadQueue[load_idx]);
401
402 if (loadQueue[load_idx]->readyToIssue()) {
403 ++retval;
404 }
405 }
406
407 return retval;
408}
409
410template <class Impl>
411Fault
412LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
413{
414 using namespace TheISA;
415 // Execute a specific load.
416 Fault load_fault = NoFault;
417
418 DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
419 inst->readPC(),inst->seqNum);
420
421 assert(!inst->isSquashed());
422
423 load_fault = inst->initiateAcc();
424
425 // If the instruction faulted, then we need to send it along to commit
426 // without the instruction completing.
427 if (load_fault != NoFault) {
428 // Send this instruction to commit, also make sure iew stage
429 // realizes there is activity.
430 // Mark it as executed unless it is an uncached load that
431 // needs to hit the head of commit.
432 if (!(inst->hasRequest() && inst->uncacheable()) ||
433 inst->isAtCommit()) {
434 inst->setExecuted();
435 }
436 iewStage->instToCommit(inst);
437 iewStage->activityThisCycle();
438 } else if (!loadBlocked()) {
439 assert(inst->effAddrValid);
440 int load_idx = inst->lqIdx;
441 incrLdIdx(load_idx);
442 while (load_idx != loadTail) {
443 // Really only need to check loads that have actually executed
444
445 // @todo: For now this is extra conservative, detecting a
446 // violation if the addresses match assuming all accesses
447 // are quad word accesses.
448
449 // @todo: Fix this, magic number being used here
450 if (loadQueue[load_idx]->effAddrValid &&
451 (loadQueue[load_idx]->effAddr >> 8) ==
452 (inst->effAddr >> 8)) {
453 // A load incorrectly passed this load. Squash and refetch.
454 // For now return a fault to show that it was unsuccessful.
455 DynInstPtr violator = loadQueue[load_idx];
456 if (!memDepViolator ||
457 (violator->seqNum < memDepViolator->seqNum)) {
458 memDepViolator = violator;
459 } else {
460 break;
461 }
462
463 ++lsqMemOrderViolation;
464
465 return genMachineCheckFault();
466 }
467
468 incrLdIdx(load_idx);
469 }
470 }
471
472 return load_fault;
473}
474
475template <class Impl>
476Fault
477LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
478{
479 using namespace TheISA;
480 // Make sure that a store exists.
481 assert(stores != 0);
482
483 int store_idx = store_inst->sqIdx;
484
485 DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
486 store_inst->readPC(), store_inst->seqNum);
487
488 assert(!store_inst->isSquashed());
489
490 // Check the recently completed loads to see if any match this store's
491 // address. If so, then we have a memory ordering violation.
492 int load_idx = store_inst->lqIdx;
493
494 Fault store_fault = store_inst->initiateAcc();
495
496 if (storeQueue[store_idx].size == 0) {
497 DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
498 store_inst->readPC(),store_inst->seqNum);
499
500 return store_fault;
501 }
502
503 assert(store_fault == NoFault);
504
505 if (store_inst->isStoreConditional()) {
506 // Store conditionals need to set themselves as able to
507 // writeback if we haven't had a fault by here.
508 storeQueue[store_idx].canWB = true;
509
510 ++storesToWB;
511 }
512
513 assert(store_inst->effAddrValid);
514 while (load_idx != loadTail) {
515 // Really only need to check loads that have actually executed
516 // It's safe to check all loads because effAddr is set to
517 // InvalAddr when the dyn inst is created.
518
519 // @todo: For now this is extra conservative, detecting a
520 // violation if the addresses match assuming all accesses
521 // are quad word accesses.
522
523 // @todo: Fix this, magic number being used here
524 if (loadQueue[load_idx]->effAddrValid &&
525 (loadQueue[load_idx]->effAddr >> 8) ==
526 (store_inst->effAddr >> 8)) {
527 // A load incorrectly passed this store. Squash and refetch.
528 // For now return a fault to show that it was unsuccessful.
529 DynInstPtr violator = loadQueue[load_idx];
530 if (!memDepViolator ||
531 (violator->seqNum < memDepViolator->seqNum)) {
532 memDepViolator = violator;
533 } else {
534 break;
535 }
536
537 ++lsqMemOrderViolation;
538
539 return genMachineCheckFault();
540 }
541
542 incrLdIdx(load_idx);
543 }
544
545 return store_fault;
546}
547
548template <class Impl>
549void
550LSQUnit<Impl>::commitLoad()
551{
552 assert(loadQueue[loadHead]);
553
554 DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
555 loadQueue[loadHead]->readPC());
556
557 loadQueue[loadHead] = NULL;
558
559 incrLdIdx(loadHead);
560
561 --loads;
562}
563
564template <class Impl>
565void
566LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
567{
568 assert(loads == 0 || loadQueue[loadHead]);
569
570 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
571 commitLoad();
572 }
573}
574
575template <class Impl>
576void
577LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
578{
579 assert(stores == 0 || storeQueue[storeHead].inst);
580
581 int store_idx = storeHead;
582
583 while (store_idx != storeTail) {
584 assert(storeQueue[store_idx].inst);
585 // Mark any stores that are now committed and have not yet
586 // been marked as able to write back.
587 if (!storeQueue[store_idx].canWB) {
588 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
589 break;
590 }
591 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
592 "%#x [sn:%lli]\n",
593 storeQueue[store_idx].inst->readPC(),
594 storeQueue[store_idx].inst->seqNum);
595
596 storeQueue[store_idx].canWB = true;
597
598 ++storesToWB;
599 }
600
601 incrStIdx(store_idx);
602 }
603}
604
605template <class Impl>
606void
607LSQUnit<Impl>::writebackStores()
608{
609 while (storesToWB > 0 &&
610 storeWBIdx != storeTail &&
611 storeQueue[storeWBIdx].inst &&
612 storeQueue[storeWBIdx].canWB &&
613 usedPorts < cachePorts) {
614
615 if (isStoreBlocked || lsq->cacheBlocked()) {
616 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
617 " is blocked!\n");
618 break;
619 }
620
621 // Store didn't write any data so no need to write it back to
622 // memory.
623 if (storeQueue[storeWBIdx].size == 0) {
624 completeStore(storeWBIdx);
625
626 incrStIdx(storeWBIdx);
627
628 continue;
629 }
630
631 ++usedPorts;
632
633 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
634 incrStIdx(storeWBIdx);
635
636 continue;
637 }
638
639 assert(storeQueue[storeWBIdx].req);
640 assert(!storeQueue[storeWBIdx].committed);
641
642 DynInstPtr inst = storeQueue[storeWBIdx].inst;
643
644 Request *req = storeQueue[storeWBIdx].req;
645 storeQueue[storeWBIdx].committed = true;
646
647 assert(!inst->memData);
648 inst->memData = new uint8_t[64];
649
650 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
651
652 MemCmd command =
653 req->isSwap() ? MemCmd::SwapReq :
654 (req->isLocked() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
655 PacketPtr data_pkt = new Packet(req, command,
656 Packet::Broadcast);
657 data_pkt->dataStatic(inst->memData);
658
659 LSQSenderState *state = new LSQSenderState;
660 state->isLoad = false;
661 state->idx = storeWBIdx;
662 state->inst = inst;
663 data_pkt->senderState = state;
664
665 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
666 "to Addr:%#x, data:%#x [sn:%lli]\n",
667 storeWBIdx, inst->readPC(),
668 req->getPaddr(), (int)*(inst->memData),
669 inst->seqNum);
670
671 // @todo: Remove this SC hack once the memory system handles it.
672 if (inst->isStoreConditional()) {
673 // Disable recording the result temporarily. Writing to
674 // misc regs normally updates the result, but this is not
675 // the desired behavior when handling store conditionals.
676 inst->recordResult = false;
677 bool success = TheISA::handleLockedWrite(inst.get(), req);
678 inst->recordResult = true;
679
680 if (!success) {
681 // Instantly complete this store.
682 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
683 "Instantly completing it.\n",
684 inst->seqNum);
685 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
686 wb->schedule(curTick + 1);
687 completeStore(storeWBIdx);
688 incrStIdx(storeWBIdx);
689 continue;
690 }
691 } else {
692 // Non-store conditionals do not need a writeback.
693 state->noWB = true;
694 }
695
696 if (!dcachePort->sendTiming(data_pkt)) {
697 // Need to handle becoming blocked on a store.
698 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
699 "retry later\n",
700 inst->seqNum);
701 isStoreBlocked = true;
702 ++lsqCacheBlocked;
703 assert(retryPkt == NULL);
704 retryPkt = data_pkt;
705 lsq->setRetryTid(lsqID);
706 } else {
707 storePostSend(data_pkt);
708 }
709 }
710
711 // Not sure this should set it to 0.
712 usedPorts = 0;
713
714 assert(stores >= 0 && storesToWB >= 0);
715}
716
717/*template <class Impl>
718void
719LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
720{
721 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
722 mshrSeqNums.end(),
723 seqNum);
724
725 if (mshr_it != mshrSeqNums.end()) {
726 mshrSeqNums.erase(mshr_it);
727 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
728 }
729}*/
730
731template <class Impl>
732void
733LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
734{
735 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
736 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
737
738 int load_idx = loadTail;
739 decrLdIdx(load_idx);
740
741 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
742 DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
743 "[sn:%lli]\n",
744 loadQueue[load_idx]->readPC(),
745 loadQueue[load_idx]->seqNum);
746
747 if (isStalled() && load_idx == stallingLoadIdx) {
748 stalled = false;
749 stallingStoreIsn = 0;
750 stallingLoadIdx = 0;
751 }
752
753 // Clear the smart pointer to make sure it is decremented.
754 loadQueue[load_idx]->setSquashed();
755 loadQueue[load_idx] = NULL;
756 --loads;
757
758 // Inefficient!
759 loadTail = load_idx;
760
761 decrLdIdx(load_idx);
762 ++lsqSquashedLoads;
763 }
764
765 if (isLoadBlocked) {
766 if (squashed_num < blockedLoadSeqNum) {
767 isLoadBlocked = false;
768 loadBlockedHandled = false;
769 blockedLoadSeqNum = 0;
770 }
771 }
772
773 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
774 memDepViolator = NULL;
775 }
776
777 int store_idx = storeTail;
778 decrStIdx(store_idx);
779
780 while (stores != 0 &&
781 storeQueue[store_idx].inst->seqNum > squashed_num) {
782 // Instructions marked as can WB are already committed.
783 if (storeQueue[store_idx].canWB) {
784 break;
785 }
786
787 DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
788 "idx:%i [sn:%lli]\n",
789 storeQueue[store_idx].inst->readPC(),
790 store_idx, storeQueue[store_idx].inst->seqNum);
791
792 // I don't think this can happen. It should have been cleared
793 // by the stalling load.
794 if (isStalled() &&
795 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
796 panic("Is stalled should have been cleared by stalling load!\n");
797 stalled = false;
798 stallingStoreIsn = 0;
799 }
800
801 // Clear the smart pointer to make sure it is decremented.
802 storeQueue[store_idx].inst->setSquashed();
803 storeQueue[store_idx].inst = NULL;
804 storeQueue[store_idx].canWB = 0;
805
806 // Must delete request now that it wasn't handed off to
807 // memory. This is quite ugly. @todo: Figure out the proper
808 // place to really handle request deletes.
809 delete storeQueue[store_idx].req;
810
811 storeQueue[store_idx].req = NULL;
812 --stores;
813
814 // Inefficient!
815 storeTail = store_idx;
816
817 decrStIdx(store_idx);
818 ++lsqSquashedStores;
819 }
820}
821
822template <class Impl>
823void
824LSQUnit<Impl>::storePostSend(PacketPtr pkt)
825{
826 if (isStalled() &&
827 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
828 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
829 "load idx:%i\n",
830 stallingStoreIsn, stallingLoadIdx);
831 stalled = false;
832 stallingStoreIsn = 0;
833 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
834 }
835
836 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
837 // The store is basically completed at this time. This
838 // only works so long as the checker doesn't try to
839 // verify the value in memory for stores.
840 storeQueue[storeWBIdx].inst->setCompleted();
841#if USE_CHECKER
842 if (cpu->checker) {
843 cpu->checker->verify(storeQueue[storeWBIdx].inst);
844 }
845#endif
846 }
847
848 incrStIdx(storeWBIdx);
849}
850
851template <class Impl>
852void
853LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
854{
855 iewStage->wakeCPU();
856
857 // Squashed instructions do not need to complete their access.
858 if (inst->isSquashed()) {
859 iewStage->decrWb(inst->seqNum);
860 assert(!inst->isStore());
861 ++lsqIgnoredResponses;
862 return;
863 }
864
865 if (!inst->isExecuted()) {
866 inst->setExecuted();
867
868 // Complete access to copy data to proper place.
869 inst->completeAcc(pkt);
870 }
871
872 // Need to insert instruction into queue to commit
873 iewStage->instToCommit(inst);
874
875 iewStage->activityThisCycle();
876}
877
878template <class Impl>
879void
880LSQUnit<Impl>::completeStore(int store_idx)
881{
882 assert(storeQueue[store_idx].inst);
883 storeQueue[store_idx].completed = true;
884 --storesToWB;
885 // A bit conservative because a store completion may not free up entries,
886 // but hopefully avoids two store completions in one cycle from making
887 // the CPU tick twice.
888 cpu->wakeCPU();
889 cpu->activityThisCycle();
890
891 if (store_idx == storeHead) {
892 do {
893 incrStIdx(storeHead);
894
895 --stores;
896 } while (storeQueue[storeHead].completed &&
897 storeHead != storeTail);
898
899 iewStage->updateLSQNextCycle = true;
900 }
901
902 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
903 "idx:%i\n",
904 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
905
906 if (isStalled() &&
907 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
908 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
909 "load idx:%i\n",
910 stallingStoreIsn, stallingLoadIdx);
911 stalled = false;
912 stallingStoreIsn = 0;
913 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
914 }
915
916 storeQueue[store_idx].inst->setCompleted();
917
918 // Tell the checker we've completed this instruction. Some stores
919 // may get reported twice to the checker, but the checker can
920 // handle that case.
921#if USE_CHECKER
922 if (cpu->checker) {
923 cpu->checker->verify(storeQueue[store_idx].inst);
924 }
925#endif
926}
927
928template <class Impl>
929void
930LSQUnit<Impl>::recvRetry()
931{
932 if (isStoreBlocked) {
933 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
934 assert(retryPkt != NULL);
935
936 if (dcachePort->sendTiming(retryPkt)) {
937 storePostSend(retryPkt);
938 retryPkt = NULL;
939 isStoreBlocked = false;
940 lsq->setRetryTid(-1);
941 } else {
942 // Still blocked!
943 ++lsqCacheBlocked;
944 lsq->setRetryTid(lsqID);
945 }
946 } else if (isLoadBlocked) {
947 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
948 "no need to resend packet.\n");
949 } else {
950 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
951 }
952}
953
954template <class Impl>
955inline void
956LSQUnit<Impl>::incrStIdx(int &store_idx)
957{
958 if (++store_idx >= SQEntries)
959 store_idx = 0;
960}
961
962template <class Impl>
963inline void
964LSQUnit<Impl>::decrStIdx(int &store_idx)
965{
966 if (--store_idx < 0)
967 store_idx += SQEntries;
968}
969
970template <class Impl>
971inline void
972LSQUnit<Impl>::incrLdIdx(int &load_idx)
973{
974 if (++load_idx >= LQEntries)
975 load_idx = 0;
976}
977
978template <class Impl>
979inline void
980LSQUnit<Impl>::decrLdIdx(int &load_idx)
981{
982 if (--load_idx < 0)
983 load_idx += LQEntries;
984}
985
986template <class Impl>
987void
988LSQUnit<Impl>::dumpInsts()
989{
990 cprintf("Load store queue: Dumping instructions.\n");
991 cprintf("Load queue size: %i\n", loads);
992 cprintf("Load queue: ");
993
994 int load_idx = loadHead;
995
996 while (load_idx != loadTail && loadQueue[load_idx]) {
997 cprintf("%#x ", loadQueue[load_idx]->readPC());
998
999 incrLdIdx(load_idx);
1000 }
1001
1002 cprintf("Store queue size: %i\n", stores);
1003 cprintf("Store queue: ");
1004
1005 int store_idx = storeHead;
1006
1007 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1008 cprintf("%#x ", storeQueue[store_idx].inst->readPC());
1009
1010 incrStIdx(store_idx);
1011 }
1012
1013 cprintf("\n");
1014}
| 71{
72 return "Store writeback";
73}
74
75template<class Impl>
76void
77LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
78{
79 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
80 DynInstPtr inst = state->inst;
81 DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
82 DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
83
84 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
85
86 assert(!pkt->wasNacked());
87
88 if (isSwitchedOut() || inst->isSquashed()) {
89 iewStage->decrWb(inst->seqNum);
90 } else {
91 if (!state->noWB) {
92 writeback(inst, pkt);
93 }
94
95 if (inst->isStore()) {
96 completeStore(state->idx);
97 }
98 }
99
100 delete state;
101 delete pkt->req;
102 delete pkt;
103}
104
105template <class Impl>
106LSQUnit<Impl>::LSQUnit()
107 : loads(0), stores(0), storesToWB(0), stalled(false),
108 isStoreBlocked(false), isLoadBlocked(false),
109 loadBlockedHandled(false)
110{
111}
112
113template<class Impl>
114void
115LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, Params *params, LSQ *lsq_ptr,
116 unsigned maxLQEntries, unsigned maxSQEntries, unsigned id)
117{
118 cpu = cpu_ptr;
119 iewStage = iew_ptr;
120
121 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
122
123 switchedOut = false;
124
125 lsq = lsq_ptr;
126
127 lsqID = id;
128
129 // Add 1 for the sentinel entry (they are circular queues).
130 LQEntries = maxLQEntries + 1;
131 SQEntries = maxSQEntries + 1;
132
133 loadQueue.resize(LQEntries);
134 storeQueue.resize(SQEntries);
135
136 loadHead = loadTail = 0;
137
138 storeHead = storeWBIdx = storeTail = 0;
139
140 usedPorts = 0;
141 cachePorts = params->cachePorts;
142
143 retryPkt = NULL;
144 memDepViolator = NULL;
145
146 blockedLoadSeqNum = 0;
147}
148
149template<class Impl>
150std::string
151LSQUnit<Impl>::name() const
152{
153 if (Impl::MaxThreads == 1) {
154 return iewStage->name() + ".lsq";
155 } else {
156 return iewStage->name() + ".lsq.thread." + to_string(lsqID);
157 }
158}
159
160template<class Impl>
161void
162LSQUnit<Impl>::regStats()
163{
164 lsqForwLoads
165 .name(name() + ".forwLoads")
166 .desc("Number of loads that had data forwarded from stores");
167
168 invAddrLoads
169 .name(name() + ".invAddrLoads")
170 .desc("Number of loads ignored due to an invalid address");
171
172 lsqSquashedLoads
173 .name(name() + ".squashedLoads")
174 .desc("Number of loads squashed");
175
176 lsqIgnoredResponses
177 .name(name() + ".ignoredResponses")
178 .desc("Number of memory responses ignored because the instruction is squashed");
179
180 lsqMemOrderViolation
181 .name(name() + ".memOrderViolation")
182 .desc("Number of memory ordering violations");
183
184 lsqSquashedStores
185 .name(name() + ".squashedStores")
186 .desc("Number of stores squashed");
187
188 invAddrSwpfs
189 .name(name() + ".invAddrSwpfs")
190 .desc("Number of software prefetches ignored due to an invalid address");
191
192 lsqBlockedLoads
193 .name(name() + ".blockedLoads")
194 .desc("Number of blocked loads due to partial load-store forwarding");
195
196 lsqRescheduledLoads
197 .name(name() + ".rescheduledLoads")
198 .desc("Number of loads that were rescheduled");
199
200 lsqCacheBlocked
201 .name(name() + ".cacheBlocked")
202 .desc("Number of times an access to memory failed due to the cache being blocked");
203}
204
205template<class Impl>
206void
207LSQUnit<Impl>::setDcachePort(Port *dcache_port)
208{
209 dcachePort = dcache_port;
210
211#if USE_CHECKER
212 if (cpu->checker) {
213 cpu->checker->setDcachePort(dcachePort);
214 }
215#endif
216}
217
218template<class Impl>
219void
220LSQUnit<Impl>::clearLQ()
221{
222 loadQueue.clear();
223}
224
225template<class Impl>
226void
227LSQUnit<Impl>::clearSQ()
228{
229 storeQueue.clear();
230}
231
232template<class Impl>
233void
234LSQUnit<Impl>::switchOut()
235{
236 switchedOut = true;
237 for (int i = 0; i < loadQueue.size(); ++i) {
238 assert(!loadQueue[i]);
239 loadQueue[i] = NULL;
240 }
241
242 assert(storesToWB == 0);
243}
244
245template<class Impl>
246void
247LSQUnit<Impl>::takeOverFrom()
248{
249 switchedOut = false;
250 loads = stores = storesToWB = 0;
251
252 loadHead = loadTail = 0;
253
254 storeHead = storeWBIdx = storeTail = 0;
255
256 usedPorts = 0;
257
258 memDepViolator = NULL;
259
260 blockedLoadSeqNum = 0;
261
262 stalled = false;
263 isLoadBlocked = false;
264 loadBlockedHandled = false;
265}
266
267template<class Impl>
268void
269LSQUnit<Impl>::resizeLQ(unsigned size)
270{
271 unsigned size_plus_sentinel = size + 1;
272 assert(size_plus_sentinel >= LQEntries);
273
274 if (size_plus_sentinel > LQEntries) {
275 while (size_plus_sentinel > loadQueue.size()) {
276 DynInstPtr dummy;
277 loadQueue.push_back(dummy);
278 LQEntries++;
279 }
280 } else {
281 LQEntries = size_plus_sentinel;
282 }
283
284}
285
286template<class Impl>
287void
288LSQUnit<Impl>::resizeSQ(unsigned size)
289{
290 unsigned size_plus_sentinel = size + 1;
291 if (size_plus_sentinel > SQEntries) {
292 while (size_plus_sentinel > storeQueue.size()) {
293 SQEntry dummy;
294 storeQueue.push_back(dummy);
295 SQEntries++;
296 }
297 } else {
298 SQEntries = size_plus_sentinel;
299 }
300}
301
302template <class Impl>
303void
304LSQUnit<Impl>::insert(DynInstPtr &inst)
305{
306 assert(inst->isMemRef());
307
308 assert(inst->isLoad() || inst->isStore());
309
310 if (inst->isLoad()) {
311 insertLoad(inst);
312 } else {
313 insertStore(inst);
314 }
315
316 inst->setInLSQ();
317}
318
319template <class Impl>
320void
321LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
322{
323 assert((loadTail + 1) % LQEntries != loadHead);
324 assert(loads < LQEntries);
325
326 DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
327 load_inst->readPC(), loadTail, load_inst->seqNum);
328
329 load_inst->lqIdx = loadTail;
330
331 if (stores == 0) {
332 load_inst->sqIdx = -1;
333 } else {
334 load_inst->sqIdx = storeTail;
335 }
336
337 loadQueue[loadTail] = load_inst;
338
339 incrLdIdx(loadTail);
340
341 ++loads;
342}
343
344template <class Impl>
345void
346LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
347{
348 // Make sure it is not full before inserting an instruction.
349 assert((storeTail + 1) % SQEntries != storeHead);
350 assert(stores < SQEntries);
351
352 DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
353 store_inst->readPC(), storeTail, store_inst->seqNum);
354
355 store_inst->sqIdx = storeTail;
356 store_inst->lqIdx = loadTail;
357
358 storeQueue[storeTail] = SQEntry(store_inst);
359
360 incrStIdx(storeTail);
361
362 ++stores;
363}
364
365template <class Impl>
366typename Impl::DynInstPtr
367LSQUnit<Impl>::getMemDepViolator()
368{
369 DynInstPtr temp = memDepViolator;
370
371 memDepViolator = NULL;
372
373 return temp;
374}
375
376template <class Impl>
377unsigned
378LSQUnit<Impl>::numFreeEntries()
379{
380 unsigned free_lq_entries = LQEntries - loads;
381 unsigned free_sq_entries = SQEntries - stores;
382
383 // Both the LQ and SQ entries have an extra dummy entry to differentiate
384 // empty/full conditions. Subtract 1 from the free entries.
385 if (free_lq_entries < free_sq_entries) {
386 return free_lq_entries - 1;
387 } else {
388 return free_sq_entries - 1;
389 }
390}
391
392template <class Impl>
393int
394LSQUnit<Impl>::numLoadsReady()
395{
396 int load_idx = loadHead;
397 int retval = 0;
398
399 while (load_idx != loadTail) {
400 assert(loadQueue[load_idx]);
401
402 if (loadQueue[load_idx]->readyToIssue()) {
403 ++retval;
404 }
405 }
406
407 return retval;
408}
409
410template <class Impl>
411Fault
412LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
413{
414 using namespace TheISA;
415 // Execute a specific load.
416 Fault load_fault = NoFault;
417
418 DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
419 inst->readPC(),inst->seqNum);
420
421 assert(!inst->isSquashed());
422
423 load_fault = inst->initiateAcc();
424
425 // If the instruction faulted, then we need to send it along to commit
426 // without the instruction completing.
427 if (load_fault != NoFault) {
428 // Send this instruction to commit, also make sure iew stage
429 // realizes there is activity.
430 // Mark it as executed unless it is an uncached load that
431 // needs to hit the head of commit.
432 if (!(inst->hasRequest() && inst->uncacheable()) ||
433 inst->isAtCommit()) {
434 inst->setExecuted();
435 }
436 iewStage->instToCommit(inst);
437 iewStage->activityThisCycle();
438 } else if (!loadBlocked()) {
439 assert(inst->effAddrValid);
440 int load_idx = inst->lqIdx;
441 incrLdIdx(load_idx);
442 while (load_idx != loadTail) {
443 // Really only need to check loads that have actually executed
444
445 // @todo: For now this is extra conservative, detecting a
446 // violation if the addresses match assuming all accesses
447 // are quad word accesses.
448
449 // @todo: Fix this, magic number being used here
450 if (loadQueue[load_idx]->effAddrValid &&
451 (loadQueue[load_idx]->effAddr >> 8) ==
452 (inst->effAddr >> 8)) {
453 // A load incorrectly passed this load. Squash and refetch.
454 // For now return a fault to show that it was unsuccessful.
455 DynInstPtr violator = loadQueue[load_idx];
456 if (!memDepViolator ||
457 (violator->seqNum < memDepViolator->seqNum)) {
458 memDepViolator = violator;
459 } else {
460 break;
461 }
462
463 ++lsqMemOrderViolation;
464
465 return genMachineCheckFault();
466 }
467
468 incrLdIdx(load_idx);
469 }
470 }
471
472 return load_fault;
473}
474
475template <class Impl>
476Fault
477LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
478{
479 using namespace TheISA;
480 // Make sure that a store exists.
481 assert(stores != 0);
482
483 int store_idx = store_inst->sqIdx;
484
485 DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
486 store_inst->readPC(), store_inst->seqNum);
487
488 assert(!store_inst->isSquashed());
489
490 // Check the recently completed loads to see if any match this store's
491 // address. If so, then we have a memory ordering violation.
492 int load_idx = store_inst->lqIdx;
493
494 Fault store_fault = store_inst->initiateAcc();
495
496 if (storeQueue[store_idx].size == 0) {
497 DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
498 store_inst->readPC(),store_inst->seqNum);
499
500 return store_fault;
501 }
502
503 assert(store_fault == NoFault);
504
505 if (store_inst->isStoreConditional()) {
506 // Store conditionals need to set themselves as able to
507 // writeback if we haven't had a fault by here.
508 storeQueue[store_idx].canWB = true;
509
510 ++storesToWB;
511 }
512
513 assert(store_inst->effAddrValid);
514 while (load_idx != loadTail) {
515 // Really only need to check loads that have actually executed
516 // It's safe to check all loads because effAddr is set to
517 // InvalAddr when the dyn inst is created.
518
519 // @todo: For now this is extra conservative, detecting a
520 // violation if the addresses match assuming all accesses
521 // are quad word accesses.
522
523 // @todo: Fix this, magic number being used here
524 if (loadQueue[load_idx]->effAddrValid &&
525 (loadQueue[load_idx]->effAddr >> 8) ==
526 (store_inst->effAddr >> 8)) {
527 // A load incorrectly passed this store. Squash and refetch.
528 // For now return a fault to show that it was unsuccessful.
529 DynInstPtr violator = loadQueue[load_idx];
530 if (!memDepViolator ||
531 (violator->seqNum < memDepViolator->seqNum)) {
532 memDepViolator = violator;
533 } else {
534 break;
535 }
536
537 ++lsqMemOrderViolation;
538
539 return genMachineCheckFault();
540 }
541
542 incrLdIdx(load_idx);
543 }
544
545 return store_fault;
546}
547
548template <class Impl>
549void
550LSQUnit<Impl>::commitLoad()
551{
552 assert(loadQueue[loadHead]);
553
554 DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
555 loadQueue[loadHead]->readPC());
556
557 loadQueue[loadHead] = NULL;
558
559 incrLdIdx(loadHead);
560
561 --loads;
562}
563
564template <class Impl>
565void
566LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
567{
568 assert(loads == 0 || loadQueue[loadHead]);
569
570 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
571 commitLoad();
572 }
573}
574
575template <class Impl>
576void
577LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
578{
579 assert(stores == 0 || storeQueue[storeHead].inst);
580
581 int store_idx = storeHead;
582
583 while (store_idx != storeTail) {
584 assert(storeQueue[store_idx].inst);
585 // Mark any stores that are now committed and have not yet
586 // been marked as able to write back.
587 if (!storeQueue[store_idx].canWB) {
588 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
589 break;
590 }
591 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
592 "%#x [sn:%lli]\n",
593 storeQueue[store_idx].inst->readPC(),
594 storeQueue[store_idx].inst->seqNum);
595
596 storeQueue[store_idx].canWB = true;
597
598 ++storesToWB;
599 }
600
601 incrStIdx(store_idx);
602 }
603}
604
605template <class Impl>
606void
607LSQUnit<Impl>::writebackStores()
608{
609 while (storesToWB > 0 &&
610 storeWBIdx != storeTail &&
611 storeQueue[storeWBIdx].inst &&
612 storeQueue[storeWBIdx].canWB &&
613 usedPorts < cachePorts) {
614
615 if (isStoreBlocked || lsq->cacheBlocked()) {
616 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
617 " is blocked!\n");
618 break;
619 }
620
621 // Store didn't write any data so no need to write it back to
622 // memory.
623 if (storeQueue[storeWBIdx].size == 0) {
624 completeStore(storeWBIdx);
625
626 incrStIdx(storeWBIdx);
627
628 continue;
629 }
630
631 ++usedPorts;
632
633 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
634 incrStIdx(storeWBIdx);
635
636 continue;
637 }
638
639 assert(storeQueue[storeWBIdx].req);
640 assert(!storeQueue[storeWBIdx].committed);
641
642 DynInstPtr inst = storeQueue[storeWBIdx].inst;
643
644 Request *req = storeQueue[storeWBIdx].req;
645 storeQueue[storeWBIdx].committed = true;
646
647 assert(!inst->memData);
648 inst->memData = new uint8_t[64];
649
650 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
651
652 MemCmd command =
653 req->isSwap() ? MemCmd::SwapReq :
654 (req->isLocked() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
655 PacketPtr data_pkt = new Packet(req, command,
656 Packet::Broadcast);
657 data_pkt->dataStatic(inst->memData);
658
659 LSQSenderState *state = new LSQSenderState;
660 state->isLoad = false;
661 state->idx = storeWBIdx;
662 state->inst = inst;
663 data_pkt->senderState = state;
664
665 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
666 "to Addr:%#x, data:%#x [sn:%lli]\n",
667 storeWBIdx, inst->readPC(),
668 req->getPaddr(), (int)*(inst->memData),
669 inst->seqNum);
670
671 // @todo: Remove this SC hack once the memory system handles it.
672 if (inst->isStoreConditional()) {
673 // Disable recording the result temporarily. Writing to
674 // misc regs normally updates the result, but this is not
675 // the desired behavior when handling store conditionals.
676 inst->recordResult = false;
677 bool success = TheISA::handleLockedWrite(inst.get(), req);
678 inst->recordResult = true;
679
680 if (!success) {
681 // Instantly complete this store.
682 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
683 "Instantly completing it.\n",
684 inst->seqNum);
685 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
686 wb->schedule(curTick + 1);
687 completeStore(storeWBIdx);
688 incrStIdx(storeWBIdx);
689 continue;
690 }
691 } else {
692 // Non-store conditionals do not need a writeback.
693 state->noWB = true;
694 }
695
696 if (!dcachePort->sendTiming(data_pkt)) {
697 // Need to handle becoming blocked on a store.
698 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
699 "retry later\n",
700 inst->seqNum);
701 isStoreBlocked = true;
702 ++lsqCacheBlocked;
703 assert(retryPkt == NULL);
704 retryPkt = data_pkt;
705 lsq->setRetryTid(lsqID);
706 } else {
707 storePostSend(data_pkt);
708 }
709 }
710
711 // Not sure this should set it to 0.
712 usedPorts = 0;
713
714 assert(stores >= 0 && storesToWB >= 0);
715}
716
717/*template <class Impl>
718void
719LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
720{
721 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
722 mshrSeqNums.end(),
723 seqNum);
724
725 if (mshr_it != mshrSeqNums.end()) {
726 mshrSeqNums.erase(mshr_it);
727 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
728 }
729}*/
730
731template <class Impl>
732void
733LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
734{
735 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
736 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
737
738 int load_idx = loadTail;
739 decrLdIdx(load_idx);
740
741 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
742 DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
743 "[sn:%lli]\n",
744 loadQueue[load_idx]->readPC(),
745 loadQueue[load_idx]->seqNum);
746
747 if (isStalled() && load_idx == stallingLoadIdx) {
748 stalled = false;
749 stallingStoreIsn = 0;
750 stallingLoadIdx = 0;
751 }
752
753 // Clear the smart pointer to make sure it is decremented.
754 loadQueue[load_idx]->setSquashed();
755 loadQueue[load_idx] = NULL;
756 --loads;
757
758 // Inefficient!
759 loadTail = load_idx;
760
761 decrLdIdx(load_idx);
762 ++lsqSquashedLoads;
763 }
764
765 if (isLoadBlocked) {
766 if (squashed_num < blockedLoadSeqNum) {
767 isLoadBlocked = false;
768 loadBlockedHandled = false;
769 blockedLoadSeqNum = 0;
770 }
771 }
772
773 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
774 memDepViolator = NULL;
775 }
776
777 int store_idx = storeTail;
778 decrStIdx(store_idx);
779
780 while (stores != 0 &&
781 storeQueue[store_idx].inst->seqNum > squashed_num) {
782 // Instructions marked as can WB are already committed.
783 if (storeQueue[store_idx].canWB) {
784 break;
785 }
786
787 DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
788 "idx:%i [sn:%lli]\n",
789 storeQueue[store_idx].inst->readPC(),
790 store_idx, storeQueue[store_idx].inst->seqNum);
791
792 // I don't think this can happen. It should have been cleared
793 // by the stalling load.
794 if (isStalled() &&
795 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
796 panic("Is stalled should have been cleared by stalling load!\n");
797 stalled = false;
798 stallingStoreIsn = 0;
799 }
800
801 // Clear the smart pointer to make sure it is decremented.
802 storeQueue[store_idx].inst->setSquashed();
803 storeQueue[store_idx].inst = NULL;
804 storeQueue[store_idx].canWB = 0;
805
806 // Must delete request now that it wasn't handed off to
807 // memory. This is quite ugly. @todo: Figure out the proper
808 // place to really handle request deletes.
809 delete storeQueue[store_idx].req;
810
811 storeQueue[store_idx].req = NULL;
812 --stores;
813
814 // Inefficient!
815 storeTail = store_idx;
816
817 decrStIdx(store_idx);
818 ++lsqSquashedStores;
819 }
820}
821
822template <class Impl>
823void
824LSQUnit<Impl>::storePostSend(PacketPtr pkt)
825{
826 if (isStalled() &&
827 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
828 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
829 "load idx:%i\n",
830 stallingStoreIsn, stallingLoadIdx);
831 stalled = false;
832 stallingStoreIsn = 0;
833 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
834 }
835
836 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
837 // The store is basically completed at this time. This
838 // only works so long as the checker doesn't try to
839 // verify the value in memory for stores.
840 storeQueue[storeWBIdx].inst->setCompleted();
841#if USE_CHECKER
842 if (cpu->checker) {
843 cpu->checker->verify(storeQueue[storeWBIdx].inst);
844 }
845#endif
846 }
847
848 incrStIdx(storeWBIdx);
849}
850
851template <class Impl>
852void
853LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
854{
855 iewStage->wakeCPU();
856
857 // Squashed instructions do not need to complete their access.
858 if (inst->isSquashed()) {
859 iewStage->decrWb(inst->seqNum);
860 assert(!inst->isStore());
861 ++lsqIgnoredResponses;
862 return;
863 }
864
865 if (!inst->isExecuted()) {
866 inst->setExecuted();
867
868 // Complete access to copy data to proper place.
869 inst->completeAcc(pkt);
870 }
871
872 // Need to insert instruction into queue to commit
873 iewStage->instToCommit(inst);
874
875 iewStage->activityThisCycle();
876}
877
878template <class Impl>
879void
880LSQUnit<Impl>::completeStore(int store_idx)
881{
882 assert(storeQueue[store_idx].inst);
883 storeQueue[store_idx].completed = true;
884 --storesToWB;
885 // A bit conservative because a store completion may not free up entries,
886 // but hopefully avoids two store completions in one cycle from making
887 // the CPU tick twice.
888 cpu->wakeCPU();
889 cpu->activityThisCycle();
890
891 if (store_idx == storeHead) {
892 do {
893 incrStIdx(storeHead);
894
895 --stores;
896 } while (storeQueue[storeHead].completed &&
897 storeHead != storeTail);
898
899 iewStage->updateLSQNextCycle = true;
900 }
901
902 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
903 "idx:%i\n",
904 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
905
906 if (isStalled() &&
907 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
908 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
909 "load idx:%i\n",
910 stallingStoreIsn, stallingLoadIdx);
911 stalled = false;
912 stallingStoreIsn = 0;
913 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
914 }
915
916 storeQueue[store_idx].inst->setCompleted();
917
918 // Tell the checker we've completed this instruction. Some stores
919 // may get reported twice to the checker, but the checker can
920 // handle that case.
921#if USE_CHECKER
922 if (cpu->checker) {
923 cpu->checker->verify(storeQueue[store_idx].inst);
924 }
925#endif
926}
927
928template <class Impl>
929void
930LSQUnit<Impl>::recvRetry()
931{
932 if (isStoreBlocked) {
933 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
934 assert(retryPkt != NULL);
935
936 if (dcachePort->sendTiming(retryPkt)) {
937 storePostSend(retryPkt);
938 retryPkt = NULL;
939 isStoreBlocked = false;
940 lsq->setRetryTid(-1);
941 } else {
942 // Still blocked!
943 ++lsqCacheBlocked;
944 lsq->setRetryTid(lsqID);
945 }
946 } else if (isLoadBlocked) {
947 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
948 "no need to resend packet.\n");
949 } else {
950 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
951 }
952}
953
954template <class Impl>
955inline void
956LSQUnit<Impl>::incrStIdx(int &store_idx)
957{
958 if (++store_idx >= SQEntries)
959 store_idx = 0;
960}
961
962template <class Impl>
963inline void
964LSQUnit<Impl>::decrStIdx(int &store_idx)
965{
966 if (--store_idx < 0)
967 store_idx += SQEntries;
968}
969
970template <class Impl>
971inline void
972LSQUnit<Impl>::incrLdIdx(int &load_idx)
973{
974 if (++load_idx >= LQEntries)
975 load_idx = 0;
976}
977
978template <class Impl>
979inline void
980LSQUnit<Impl>::decrLdIdx(int &load_idx)
981{
982 if (--load_idx < 0)
983 load_idx += LQEntries;
984}
985
986template <class Impl>
987void
988LSQUnit<Impl>::dumpInsts()
989{
990 cprintf("Load store queue: Dumping instructions.\n");
991 cprintf("Load queue size: %i\n", loads);
992 cprintf("Load queue: ");
993
994 int load_idx = loadHead;
995
996 while (load_idx != loadTail && loadQueue[load_idx]) {
997 cprintf("%#x ", loadQueue[load_idx]->readPC());
998
999 incrLdIdx(load_idx);
1000 }
1001
1002 cprintf("Store queue size: %i\n", stores);
1003 cprintf("Store queue: ");
1004
1005 int store_idx = storeHead;
1006
1007 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1008 cprintf("%#x ", storeQueue[store_idx].inst->readPC());
1009
1010 incrStIdx(store_idx);
1011 }
1012
1013 cprintf("\n");
1014}
|