1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Kevin Lim
29 * Korey Sewell
30 */
31
| 1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Kevin Lim
29 * Korey Sewell
30 */
31
|
| 32#include "arch/locked_mem.hh"
|
32#include "config/use_checker.hh"
33
34#include "cpu/o3/lsq.hh"
35#include "cpu/o3/lsq_unit.hh"
36#include "base/str.hh"
37#include "mem/packet.hh"
38#include "mem/request.hh"
39
40#if USE_CHECKER
41#include "cpu/checker/cpu.hh"
42#endif
43
44template<class Impl>
45LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
46 LSQUnit *lsq_ptr)
47 : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
48{
49 this->setFlags(Event::AutoDelete);
50}
51
52template<class Impl>
53void
54LSQUnit<Impl>::WritebackEvent::process()
55{
56 if (!lsqPtr->isSwitchedOut()) {
57 lsqPtr->writeback(inst, pkt);
58 }
59 delete pkt;
60}
61
62template<class Impl>
63const char *
64LSQUnit<Impl>::WritebackEvent::description()
65{
66 return "Store writeback event";
67}
68
69template<class Impl>
70void
71LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
72{
73 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
74 DynInstPtr inst = state->inst;
75 DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
76 DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
77
78 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
79
80 if (isSwitchedOut() || inst->isSquashed()) {
81 iewStage->decrWb(inst->seqNum);
82 delete state;
83 delete pkt;
84 return;
85 } else {
86 if (!state->noWB) {
87 writeback(inst, pkt);
88 }
89
90 if (inst->isStore()) {
91 completeStore(state->idx);
92 }
93 }
94
95 delete state;
96 delete pkt;
97}
98
99template <class Impl>
100LSQUnit<Impl>::LSQUnit()
101 : loads(0), stores(0), storesToWB(0), stalled(false),
102 isStoreBlocked(false), isLoadBlocked(false),
103 loadBlockedHandled(false)
104{
105}
106
107template<class Impl>
108void
109LSQUnit<Impl>::init(Params *params, LSQ *lsq_ptr, unsigned maxLQEntries,
110 unsigned maxSQEntries, unsigned id)
111{
112 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
113
114 switchedOut = false;
115
116 lsq = lsq_ptr;
117
118 lsqID = id;
119
120 // Add 1 for the sentinel entry (they are circular queues).
121 LQEntries = maxLQEntries + 1;
122 SQEntries = maxSQEntries + 1;
123
124 loadQueue.resize(LQEntries);
125 storeQueue.resize(SQEntries);
126
127 loadHead = loadTail = 0;
128
129 storeHead = storeWBIdx = storeTail = 0;
130
131 usedPorts = 0;
132 cachePorts = params->cachePorts;
133
134 retryPkt = NULL;
135 memDepViolator = NULL;
136
137 blockedLoadSeqNum = 0;
138}
139
140template<class Impl>
141void
142LSQUnit<Impl>::setCPU(O3CPU *cpu_ptr)
143{
144 cpu = cpu_ptr;
145
146#if USE_CHECKER
147 if (cpu->checker) {
148 cpu->checker->setDcachePort(dcachePort);
149 }
150#endif
151}
152
153template<class Impl>
154std::string
155LSQUnit<Impl>::name() const
156{
157 if (Impl::MaxThreads == 1) {
158 return iewStage->name() + ".lsq";
159 } else {
160 return iewStage->name() + ".lsq.thread." + to_string(lsqID);
161 }
162}
163
164template<class Impl>
165void
166LSQUnit<Impl>::regStats()
167{
168 lsqForwLoads
169 .name(name() + ".forwLoads")
170 .desc("Number of loads that had data forwarded from stores");
171
172 invAddrLoads
173 .name(name() + ".invAddrLoads")
174 .desc("Number of loads ignored due to an invalid address");
175
176 lsqSquashedLoads
177 .name(name() + ".squashedLoads")
178 .desc("Number of loads squashed");
179
180 lsqIgnoredResponses
181 .name(name() + ".ignoredResponses")
182 .desc("Number of memory responses ignored because the instruction is squashed");
183
184 lsqMemOrderViolation
185 .name(name() + ".memOrderViolation")
186 .desc("Number of memory ordering violations");
187
188 lsqSquashedStores
189 .name(name() + ".squashedStores")
190 .desc("Number of stores squashed");
191
192 invAddrSwpfs
193 .name(name() + ".invAddrSwpfs")
194 .desc("Number of software prefetches ignored due to an invalid address");
195
196 lsqBlockedLoads
197 .name(name() + ".blockedLoads")
198 .desc("Number of blocked loads due to partial load-store forwarding");
199
200 lsqRescheduledLoads
201 .name(name() + ".rescheduledLoads")
202 .desc("Number of loads that were rescheduled");
203
204 lsqCacheBlocked
205 .name(name() + ".cacheBlocked")
206 .desc("Number of times an access to memory failed due to the cache being blocked");
207}
208
209template<class Impl>
210void
211LSQUnit<Impl>::clearLQ()
212{
213 loadQueue.clear();
214}
215
216template<class Impl>
217void
218LSQUnit<Impl>::clearSQ()
219{
220 storeQueue.clear();
221}
222
223template<class Impl>
224void
225LSQUnit<Impl>::switchOut()
226{
227 switchedOut = true;
228 for (int i = 0; i < loadQueue.size(); ++i) {
229 assert(!loadQueue[i]);
230 loadQueue[i] = NULL;
231 }
232
233 assert(storesToWB == 0);
234}
235
236template<class Impl>
237void
238LSQUnit<Impl>::takeOverFrom()
239{
240 switchedOut = false;
241 loads = stores = storesToWB = 0;
242
243 loadHead = loadTail = 0;
244
245 storeHead = storeWBIdx = storeTail = 0;
246
247 usedPorts = 0;
248
249 memDepViolator = NULL;
250
251 blockedLoadSeqNum = 0;
252
253 stalled = false;
254 isLoadBlocked = false;
255 loadBlockedHandled = false;
256}
257
258template<class Impl>
259void
260LSQUnit<Impl>::resizeLQ(unsigned size)
261{
262 unsigned size_plus_sentinel = size + 1;
263 assert(size_plus_sentinel >= LQEntries);
264
265 if (size_plus_sentinel > LQEntries) {
266 while (size_plus_sentinel > loadQueue.size()) {
267 DynInstPtr dummy;
268 loadQueue.push_back(dummy);
269 LQEntries++;
270 }
271 } else {
272 LQEntries = size_plus_sentinel;
273 }
274
275}
276
277template<class Impl>
278void
279LSQUnit<Impl>::resizeSQ(unsigned size)
280{
281 unsigned size_plus_sentinel = size + 1;
282 if (size_plus_sentinel > SQEntries) {
283 while (size_plus_sentinel > storeQueue.size()) {
284 SQEntry dummy;
285 storeQueue.push_back(dummy);
286 SQEntries++;
287 }
288 } else {
289 SQEntries = size_plus_sentinel;
290 }
291}
292
293template <class Impl>
294void
295LSQUnit<Impl>::insert(DynInstPtr &inst)
296{
297 assert(inst->isMemRef());
298
299 assert(inst->isLoad() || inst->isStore());
300
301 if (inst->isLoad()) {
302 insertLoad(inst);
303 } else {
304 insertStore(inst);
305 }
306
307 inst->setInLSQ();
308}
309
310template <class Impl>
311void
312LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
313{
314 assert((loadTail + 1) % LQEntries != loadHead);
315 assert(loads < LQEntries);
316
317 DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
318 load_inst->readPC(), loadTail, load_inst->seqNum);
319
320 load_inst->lqIdx = loadTail;
321
322 if (stores == 0) {
323 load_inst->sqIdx = -1;
324 } else {
325 load_inst->sqIdx = storeTail;
326 }
327
328 loadQueue[loadTail] = load_inst;
329
330 incrLdIdx(loadTail);
331
332 ++loads;
333}
334
335template <class Impl>
336void
337LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
338{
339 // Make sure it is not full before inserting an instruction.
340 assert((storeTail + 1) % SQEntries != storeHead);
341 assert(stores < SQEntries);
342
343 DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
344 store_inst->readPC(), storeTail, store_inst->seqNum);
345
346 store_inst->sqIdx = storeTail;
347 store_inst->lqIdx = loadTail;
348
349 storeQueue[storeTail] = SQEntry(store_inst);
350
351 incrStIdx(storeTail);
352
353 ++stores;
354}
355
356template <class Impl>
357typename Impl::DynInstPtr
358LSQUnit<Impl>::getMemDepViolator()
359{
360 DynInstPtr temp = memDepViolator;
361
362 memDepViolator = NULL;
363
364 return temp;
365}
366
367template <class Impl>
368unsigned
369LSQUnit<Impl>::numFreeEntries()
370{
371 unsigned free_lq_entries = LQEntries - loads;
372 unsigned free_sq_entries = SQEntries - stores;
373
374 // Both the LQ and SQ entries have an extra dummy entry to differentiate
375 // empty/full conditions. Subtract 1 from the free entries.
376 if (free_lq_entries < free_sq_entries) {
377 return free_lq_entries - 1;
378 } else {
379 return free_sq_entries - 1;
380 }
381}
382
383template <class Impl>
384int
385LSQUnit<Impl>::numLoadsReady()
386{
387 int load_idx = loadHead;
388 int retval = 0;
389
390 while (load_idx != loadTail) {
391 assert(loadQueue[load_idx]);
392
393 if (loadQueue[load_idx]->readyToIssue()) {
394 ++retval;
395 }
396 }
397
398 return retval;
399}
400
401template <class Impl>
402Fault
403LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
404{
405 // Execute a specific load.
406 Fault load_fault = NoFault;
407
408 DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
409 inst->readPC(),inst->seqNum);
410
411 load_fault = inst->initiateAcc();
412
413 // If the instruction faulted, then we need to send it along to commit
414 // without the instruction completing.
415 if (load_fault != NoFault) {
416 // Send this instruction to commit, also make sure iew stage
417 // realizes there is activity.
418 // Mark it as executed unless it is an uncached load that
419 // needs to hit the head of commit.
420 if (!(inst->req->isUncacheable()) || inst->isAtCommit()) {
421 inst->setExecuted();
422 }
423 iewStage->instToCommit(inst);
424 iewStage->activityThisCycle();
425 }
426
427 return load_fault;
428}
429
430template <class Impl>
431Fault
432LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
433{
434 using namespace TheISA;
435 // Make sure that a store exists.
436 assert(stores != 0);
437
438 int store_idx = store_inst->sqIdx;
439
440 DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
441 store_inst->readPC(), store_inst->seqNum);
442
443 // Check the recently completed loads to see if any match this store's
444 // address. If so, then we have a memory ordering violation.
445 int load_idx = store_inst->lqIdx;
446
447 Fault store_fault = store_inst->initiateAcc();
448
449 if (storeQueue[store_idx].size == 0) {
450 DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
451 store_inst->readPC(),store_inst->seqNum);
452
453 return store_fault;
454 }
455
456 assert(store_fault == NoFault);
457
458 if (store_inst->isStoreConditional()) {
459 // Store conditionals need to set themselves as able to
460 // writeback if we haven't had a fault by here.
461 storeQueue[store_idx].canWB = true;
462
463 ++storesToWB;
464 }
465
466 if (!memDepViolator) {
467 while (load_idx != loadTail) {
468 // Really only need to check loads that have actually executed
469 // It's safe to check all loads because effAddr is set to
470 // InvalAddr when the dyn inst is created.
471
472 // @todo: For now this is extra conservative, detecting a
473 // violation if the addresses match assuming all accesses
474 // are quad word accesses.
475
476 // @todo: Fix this, magic number being used here
477 if ((loadQueue[load_idx]->effAddr >> 8) ==
478 (store_inst->effAddr >> 8)) {
479 // A load incorrectly passed this store. Squash and refetch.
480 // For now return a fault to show that it was unsuccessful.
481 memDepViolator = loadQueue[load_idx];
482 ++lsqMemOrderViolation;
483
484 return genMachineCheckFault();
485 }
486
487 incrLdIdx(load_idx);
488 }
489
490 // If we've reached this point, there was no violation.
491 memDepViolator = NULL;
492 }
493
494 return store_fault;
495}
496
497template <class Impl>
498void
499LSQUnit<Impl>::commitLoad()
500{
501 assert(loadQueue[loadHead]);
502
503 DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
504 loadQueue[loadHead]->readPC());
505
506 loadQueue[loadHead] = NULL;
507
508 incrLdIdx(loadHead);
509
510 --loads;
511}
512
513template <class Impl>
514void
515LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
516{
517 assert(loads == 0 || loadQueue[loadHead]);
518
519 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
520 commitLoad();
521 }
522}
523
524template <class Impl>
525void
526LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
527{
528 assert(stores == 0 || storeQueue[storeHead].inst);
529
530 int store_idx = storeHead;
531
532 while (store_idx != storeTail) {
533 assert(storeQueue[store_idx].inst);
534 // Mark any stores that are now committed and have not yet
535 // been marked as able to write back.
536 if (!storeQueue[store_idx].canWB) {
537 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
538 break;
539 }
540 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
541 "%#x [sn:%lli]\n",
542 storeQueue[store_idx].inst->readPC(),
543 storeQueue[store_idx].inst->seqNum);
544
545 storeQueue[store_idx].canWB = true;
546
547 ++storesToWB;
548 }
549
550 incrStIdx(store_idx);
551 }
552}
553
554template <class Impl>
555void
556LSQUnit<Impl>::writebackStores()
557{
558 while (storesToWB > 0 &&
559 storeWBIdx != storeTail &&
560 storeQueue[storeWBIdx].inst &&
561 storeQueue[storeWBIdx].canWB &&
562 usedPorts < cachePorts) {
563
564 if (isStoreBlocked || lsq->cacheBlocked()) {
565 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
566 " is blocked!\n");
567 break;
568 }
569
570 // Store didn't write any data so no need to write it back to
571 // memory.
572 if (storeQueue[storeWBIdx].size == 0) {
573 completeStore(storeWBIdx);
574
575 incrStIdx(storeWBIdx);
576
577 continue;
578 }
579
580 ++usedPorts;
581
582 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
583 incrStIdx(storeWBIdx);
584
585 continue;
586 }
587
588 assert(storeQueue[storeWBIdx].req);
589 assert(!storeQueue[storeWBIdx].committed);
590
591 DynInstPtr inst = storeQueue[storeWBIdx].inst;
592
593 Request *req = storeQueue[storeWBIdx].req;
594 storeQueue[storeWBIdx].committed = true;
595
596 assert(!inst->memData);
597 inst->memData = new uint8_t[64];
598 memcpy(inst->memData, (uint8_t *)&storeQueue[storeWBIdx].data,
599 req->getSize());
600
601 PacketPtr data_pkt = new Packet(req, Packet::WriteReq, Packet::Broadcast);
602 data_pkt->dataStatic(inst->memData);
603
604 LSQSenderState *state = new LSQSenderState;
605 state->isLoad = false;
606 state->idx = storeWBIdx;
607 state->inst = inst;
608 data_pkt->senderState = state;
609
610 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
611 "to Addr:%#x, data:%#x [sn:%lli]\n",
612 storeWBIdx, inst->readPC(),
613 req->getPaddr(), *(inst->memData),
614 inst->seqNum);
615
616 // @todo: Remove this SC hack once the memory system handles it.
617 if (req->isLocked()) {
| 33#include "config/use_checker.hh"
34
35#include "cpu/o3/lsq.hh"
36#include "cpu/o3/lsq_unit.hh"
37#include "base/str.hh"
38#include "mem/packet.hh"
39#include "mem/request.hh"
40
41#if USE_CHECKER
42#include "cpu/checker/cpu.hh"
43#endif
44
45template<class Impl>
46LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
47 LSQUnit *lsq_ptr)
48 : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
49{
50 this->setFlags(Event::AutoDelete);
51}
52
53template<class Impl>
54void
55LSQUnit<Impl>::WritebackEvent::process()
56{
57 if (!lsqPtr->isSwitchedOut()) {
58 lsqPtr->writeback(inst, pkt);
59 }
60 delete pkt;
61}
62
63template<class Impl>
64const char *
65LSQUnit<Impl>::WritebackEvent::description()
66{
67 return "Store writeback event";
68}
69
70template<class Impl>
71void
72LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
73{
74 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
75 DynInstPtr inst = state->inst;
76 DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
77 DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
78
79 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
80
81 if (isSwitchedOut() || inst->isSquashed()) {
82 iewStage->decrWb(inst->seqNum);
83 delete state;
84 delete pkt;
85 return;
86 } else {
87 if (!state->noWB) {
88 writeback(inst, pkt);
89 }
90
91 if (inst->isStore()) {
92 completeStore(state->idx);
93 }
94 }
95
96 delete state;
97 delete pkt;
98}
99
100template <class Impl>
101LSQUnit<Impl>::LSQUnit()
102 : loads(0), stores(0), storesToWB(0), stalled(false),
103 isStoreBlocked(false), isLoadBlocked(false),
104 loadBlockedHandled(false)
105{
106}
107
108template<class Impl>
109void
110LSQUnit<Impl>::init(Params *params, LSQ *lsq_ptr, unsigned maxLQEntries,
111 unsigned maxSQEntries, unsigned id)
112{
113 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
114
115 switchedOut = false;
116
117 lsq = lsq_ptr;
118
119 lsqID = id;
120
121 // Add 1 for the sentinel entry (they are circular queues).
122 LQEntries = maxLQEntries + 1;
123 SQEntries = maxSQEntries + 1;
124
125 loadQueue.resize(LQEntries);
126 storeQueue.resize(SQEntries);
127
128 loadHead = loadTail = 0;
129
130 storeHead = storeWBIdx = storeTail = 0;
131
132 usedPorts = 0;
133 cachePorts = params->cachePorts;
134
135 retryPkt = NULL;
136 memDepViolator = NULL;
137
138 blockedLoadSeqNum = 0;
139}
140
141template<class Impl>
142void
143LSQUnit<Impl>::setCPU(O3CPU *cpu_ptr)
144{
145 cpu = cpu_ptr;
146
147#if USE_CHECKER
148 if (cpu->checker) {
149 cpu->checker->setDcachePort(dcachePort);
150 }
151#endif
152}
153
154template<class Impl>
155std::string
156LSQUnit<Impl>::name() const
157{
158 if (Impl::MaxThreads == 1) {
159 return iewStage->name() + ".lsq";
160 } else {
161 return iewStage->name() + ".lsq.thread." + to_string(lsqID);
162 }
163}
164
165template<class Impl>
166void
167LSQUnit<Impl>::regStats()
168{
169 lsqForwLoads
170 .name(name() + ".forwLoads")
171 .desc("Number of loads that had data forwarded from stores");
172
173 invAddrLoads
174 .name(name() + ".invAddrLoads")
175 .desc("Number of loads ignored due to an invalid address");
176
177 lsqSquashedLoads
178 .name(name() + ".squashedLoads")
179 .desc("Number of loads squashed");
180
181 lsqIgnoredResponses
182 .name(name() + ".ignoredResponses")
183 .desc("Number of memory responses ignored because the instruction is squashed");
184
185 lsqMemOrderViolation
186 .name(name() + ".memOrderViolation")
187 .desc("Number of memory ordering violations");
188
189 lsqSquashedStores
190 .name(name() + ".squashedStores")
191 .desc("Number of stores squashed");
192
193 invAddrSwpfs
194 .name(name() + ".invAddrSwpfs")
195 .desc("Number of software prefetches ignored due to an invalid address");
196
197 lsqBlockedLoads
198 .name(name() + ".blockedLoads")
199 .desc("Number of blocked loads due to partial load-store forwarding");
200
201 lsqRescheduledLoads
202 .name(name() + ".rescheduledLoads")
203 .desc("Number of loads that were rescheduled");
204
205 lsqCacheBlocked
206 .name(name() + ".cacheBlocked")
207 .desc("Number of times an access to memory failed due to the cache being blocked");
208}
209
210template<class Impl>
211void
212LSQUnit<Impl>::clearLQ()
213{
214 loadQueue.clear();
215}
216
217template<class Impl>
218void
219LSQUnit<Impl>::clearSQ()
220{
221 storeQueue.clear();
222}
223
224template<class Impl>
225void
226LSQUnit<Impl>::switchOut()
227{
228 switchedOut = true;
229 for (int i = 0; i < loadQueue.size(); ++i) {
230 assert(!loadQueue[i]);
231 loadQueue[i] = NULL;
232 }
233
234 assert(storesToWB == 0);
235}
236
237template<class Impl>
238void
239LSQUnit<Impl>::takeOverFrom()
240{
241 switchedOut = false;
242 loads = stores = storesToWB = 0;
243
244 loadHead = loadTail = 0;
245
246 storeHead = storeWBIdx = storeTail = 0;
247
248 usedPorts = 0;
249
250 memDepViolator = NULL;
251
252 blockedLoadSeqNum = 0;
253
254 stalled = false;
255 isLoadBlocked = false;
256 loadBlockedHandled = false;
257}
258
259template<class Impl>
260void
261LSQUnit<Impl>::resizeLQ(unsigned size)
262{
263 unsigned size_plus_sentinel = size + 1;
264 assert(size_plus_sentinel >= LQEntries);
265
266 if (size_plus_sentinel > LQEntries) {
267 while (size_plus_sentinel > loadQueue.size()) {
268 DynInstPtr dummy;
269 loadQueue.push_back(dummy);
270 LQEntries++;
271 }
272 } else {
273 LQEntries = size_plus_sentinel;
274 }
275
276}
277
278template<class Impl>
279void
280LSQUnit<Impl>::resizeSQ(unsigned size)
281{
282 unsigned size_plus_sentinel = size + 1;
283 if (size_plus_sentinel > SQEntries) {
284 while (size_plus_sentinel > storeQueue.size()) {
285 SQEntry dummy;
286 storeQueue.push_back(dummy);
287 SQEntries++;
288 }
289 } else {
290 SQEntries = size_plus_sentinel;
291 }
292}
293
294template <class Impl>
295void
296LSQUnit<Impl>::insert(DynInstPtr &inst)
297{
298 assert(inst->isMemRef());
299
300 assert(inst->isLoad() || inst->isStore());
301
302 if (inst->isLoad()) {
303 insertLoad(inst);
304 } else {
305 insertStore(inst);
306 }
307
308 inst->setInLSQ();
309}
310
311template <class Impl>
312void
313LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
314{
315 assert((loadTail + 1) % LQEntries != loadHead);
316 assert(loads < LQEntries);
317
318 DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
319 load_inst->readPC(), loadTail, load_inst->seqNum);
320
321 load_inst->lqIdx = loadTail;
322
323 if (stores == 0) {
324 load_inst->sqIdx = -1;
325 } else {
326 load_inst->sqIdx = storeTail;
327 }
328
329 loadQueue[loadTail] = load_inst;
330
331 incrLdIdx(loadTail);
332
333 ++loads;
334}
335
336template <class Impl>
337void
338LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
339{
340 // Make sure it is not full before inserting an instruction.
341 assert((storeTail + 1) % SQEntries != storeHead);
342 assert(stores < SQEntries);
343
344 DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
345 store_inst->readPC(), storeTail, store_inst->seqNum);
346
347 store_inst->sqIdx = storeTail;
348 store_inst->lqIdx = loadTail;
349
350 storeQueue[storeTail] = SQEntry(store_inst);
351
352 incrStIdx(storeTail);
353
354 ++stores;
355}
356
357template <class Impl>
358typename Impl::DynInstPtr
359LSQUnit<Impl>::getMemDepViolator()
360{
361 DynInstPtr temp = memDepViolator;
362
363 memDepViolator = NULL;
364
365 return temp;
366}
367
368template <class Impl>
369unsigned
370LSQUnit<Impl>::numFreeEntries()
371{
372 unsigned free_lq_entries = LQEntries - loads;
373 unsigned free_sq_entries = SQEntries - stores;
374
375 // Both the LQ and SQ entries have an extra dummy entry to differentiate
376 // empty/full conditions. Subtract 1 from the free entries.
377 if (free_lq_entries < free_sq_entries) {
378 return free_lq_entries - 1;
379 } else {
380 return free_sq_entries - 1;
381 }
382}
383
384template <class Impl>
385int
386LSQUnit<Impl>::numLoadsReady()
387{
388 int load_idx = loadHead;
389 int retval = 0;
390
391 while (load_idx != loadTail) {
392 assert(loadQueue[load_idx]);
393
394 if (loadQueue[load_idx]->readyToIssue()) {
395 ++retval;
396 }
397 }
398
399 return retval;
400}
401
402template <class Impl>
403Fault
404LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
405{
406 // Execute a specific load.
407 Fault load_fault = NoFault;
408
409 DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
410 inst->readPC(),inst->seqNum);
411
412 load_fault = inst->initiateAcc();
413
414 // If the instruction faulted, then we need to send it along to commit
415 // without the instruction completing.
416 if (load_fault != NoFault) {
417 // Send this instruction to commit, also make sure iew stage
418 // realizes there is activity.
419 // Mark it as executed unless it is an uncached load that
420 // needs to hit the head of commit.
421 if (!(inst->req->isUncacheable()) || inst->isAtCommit()) {
422 inst->setExecuted();
423 }
424 iewStage->instToCommit(inst);
425 iewStage->activityThisCycle();
426 }
427
428 return load_fault;
429}
430
431template <class Impl>
432Fault
433LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
434{
435 using namespace TheISA;
436 // Make sure that a store exists.
437 assert(stores != 0);
438
439 int store_idx = store_inst->sqIdx;
440
441 DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
442 store_inst->readPC(), store_inst->seqNum);
443
444 // Check the recently completed loads to see if any match this store's
445 // address. If so, then we have a memory ordering violation.
446 int load_idx = store_inst->lqIdx;
447
448 Fault store_fault = store_inst->initiateAcc();
449
450 if (storeQueue[store_idx].size == 0) {
451 DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
452 store_inst->readPC(),store_inst->seqNum);
453
454 return store_fault;
455 }
456
457 assert(store_fault == NoFault);
458
459 if (store_inst->isStoreConditional()) {
460 // Store conditionals need to set themselves as able to
461 // writeback if we haven't had a fault by here.
462 storeQueue[store_idx].canWB = true;
463
464 ++storesToWB;
465 }
466
467 if (!memDepViolator) {
468 while (load_idx != loadTail) {
469 // Really only need to check loads that have actually executed
470 // It's safe to check all loads because effAddr is set to
471 // InvalAddr when the dyn inst is created.
472
473 // @todo: For now this is extra conservative, detecting a
474 // violation if the addresses match assuming all accesses
475 // are quad word accesses.
476
477 // @todo: Fix this, magic number being used here
478 if ((loadQueue[load_idx]->effAddr >> 8) ==
479 (store_inst->effAddr >> 8)) {
480 // A load incorrectly passed this store. Squash and refetch.
481 // For now return a fault to show that it was unsuccessful.
482 memDepViolator = loadQueue[load_idx];
483 ++lsqMemOrderViolation;
484
485 return genMachineCheckFault();
486 }
487
488 incrLdIdx(load_idx);
489 }
490
491 // If we've reached this point, there was no violation.
492 memDepViolator = NULL;
493 }
494
495 return store_fault;
496}
497
498template <class Impl>
499void
500LSQUnit<Impl>::commitLoad()
501{
502 assert(loadQueue[loadHead]);
503
504 DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
505 loadQueue[loadHead]->readPC());
506
507 loadQueue[loadHead] = NULL;
508
509 incrLdIdx(loadHead);
510
511 --loads;
512}
513
514template <class Impl>
515void
516LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
517{
518 assert(loads == 0 || loadQueue[loadHead]);
519
520 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
521 commitLoad();
522 }
523}
524
525template <class Impl>
526void
527LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
528{
529 assert(stores == 0 || storeQueue[storeHead].inst);
530
531 int store_idx = storeHead;
532
533 while (store_idx != storeTail) {
534 assert(storeQueue[store_idx].inst);
535 // Mark any stores that are now committed and have not yet
536 // been marked as able to write back.
537 if (!storeQueue[store_idx].canWB) {
538 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
539 break;
540 }
541 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
542 "%#x [sn:%lli]\n",
543 storeQueue[store_idx].inst->readPC(),
544 storeQueue[store_idx].inst->seqNum);
545
546 storeQueue[store_idx].canWB = true;
547
548 ++storesToWB;
549 }
550
551 incrStIdx(store_idx);
552 }
553}
554
555template <class Impl>
556void
557LSQUnit<Impl>::writebackStores()
558{
559 while (storesToWB > 0 &&
560 storeWBIdx != storeTail &&
561 storeQueue[storeWBIdx].inst &&
562 storeQueue[storeWBIdx].canWB &&
563 usedPorts < cachePorts) {
564
565 if (isStoreBlocked || lsq->cacheBlocked()) {
566 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
567 " is blocked!\n");
568 break;
569 }
570
571 // Store didn't write any data so no need to write it back to
572 // memory.
573 if (storeQueue[storeWBIdx].size == 0) {
574 completeStore(storeWBIdx);
575
576 incrStIdx(storeWBIdx);
577
578 continue;
579 }
580
581 ++usedPorts;
582
583 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
584 incrStIdx(storeWBIdx);
585
586 continue;
587 }
588
589 assert(storeQueue[storeWBIdx].req);
590 assert(!storeQueue[storeWBIdx].committed);
591
592 DynInstPtr inst = storeQueue[storeWBIdx].inst;
593
594 Request *req = storeQueue[storeWBIdx].req;
595 storeQueue[storeWBIdx].committed = true;
596
597 assert(!inst->memData);
598 inst->memData = new uint8_t[64];
599 memcpy(inst->memData, (uint8_t *)&storeQueue[storeWBIdx].data,
600 req->getSize());
601
602 PacketPtr data_pkt = new Packet(req, Packet::WriteReq, Packet::Broadcast);
603 data_pkt->dataStatic(inst->memData);
604
605 LSQSenderState *state = new LSQSenderState;
606 state->isLoad = false;
607 state->idx = storeWBIdx;
608 state->inst = inst;
609 data_pkt->senderState = state;
610
611 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
612 "to Addr:%#x, data:%#x [sn:%lli]\n",
613 storeWBIdx, inst->readPC(),
614 req->getPaddr(), *(inst->memData),
615 inst->seqNum);
616
617 // @todo: Remove this SC hack once the memory system handles it.
618 if (req->isLocked()) {
|
618 if (req->isUncacheable()) {
619 req->setScResult(2);
620 } else {
621 if (cpu->lockFlag) {
622 req->setScResult(1);
623 DPRINTF(LSQUnit, "Store conditional [sn:%lli] succeeded.",
624 inst->seqNum);
625 } else {
626 req->setScResult(0);
627 // Hack: Instantly complete this store.
628// completeDataAccess(data_pkt);
629 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
630 "Instantly completing it.\n",
631 inst->seqNum);
632 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
633 wb->schedule(curTick + 1);
634 delete state;
635 completeStore(storeWBIdx);
636 incrStIdx(storeWBIdx);
637 continue;
638 }
| 619 // Disable recording the result temporarily. Writing to
620 // misc regs normally updates the result, but this is not
621 // the desired behavior when handling store conditionals.
622 inst->recordResult = false;
623 bool success = TheISA::handleLockedWrite(inst.get(), req);
624 inst->recordResult = true;
625
626 if (!success) {
627 // Instantly complete this store.
628 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
629 "Instantly completing it.\n",
630 inst->seqNum);
631 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
632 wb->schedule(curTick + 1);
633 delete state;
634 completeStore(storeWBIdx);
635 incrStIdx(storeWBIdx);
636 continue;
|
639 }
640 } else {
641 // Non-store conditionals do not need a writeback.
642 state->noWB = true;
643 }
644
645 if (!dcachePort->sendTiming(data_pkt)) {
646 if (data_pkt->result == Packet::BadAddress) {
647 panic("LSQ sent out a bad address for a completed store!");
648 }
649 // Need to handle becoming blocked on a store.
650 DPRINTF(IEW, "D-Cache became blcoked when writing [sn:%lli], will"
651 "retry later\n",
652 inst->seqNum);
653 isStoreBlocked = true;
654 ++lsqCacheBlocked;
655 assert(retryPkt == NULL);
656 retryPkt = data_pkt;
657 lsq->setRetryTid(lsqID);
658 } else {
659 storePostSend(data_pkt);
660 }
661 }
662
663 // Not sure this should set it to 0.
664 usedPorts = 0;
665
666 assert(stores >= 0 && storesToWB >= 0);
667}
668
669/*template <class Impl>
670void
671LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
672{
673 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
674 mshrSeqNums.end(),
675 seqNum);
676
677 if (mshr_it != mshrSeqNums.end()) {
678 mshrSeqNums.erase(mshr_it);
679 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
680 }
681}*/
682
683template <class Impl>
684void
685LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
686{
687 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
688 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
689
690 int load_idx = loadTail;
691 decrLdIdx(load_idx);
692
693 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
694 DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
695 "[sn:%lli]\n",
696 loadQueue[load_idx]->readPC(),
697 loadQueue[load_idx]->seqNum);
698
699 if (isStalled() && load_idx == stallingLoadIdx) {
700 stalled = false;
701 stallingStoreIsn = 0;
702 stallingLoadIdx = 0;
703 }
704
705 // Clear the smart pointer to make sure it is decremented.
706 loadQueue[load_idx]->setSquashed();
707 loadQueue[load_idx] = NULL;
708 --loads;
709
710 // Inefficient!
711 loadTail = load_idx;
712
713 decrLdIdx(load_idx);
714 ++lsqSquashedLoads;
715 }
716
717 if (isLoadBlocked) {
718 if (squashed_num < blockedLoadSeqNum) {
719 isLoadBlocked = false;
720 loadBlockedHandled = false;
721 blockedLoadSeqNum = 0;
722 }
723 }
724
725 int store_idx = storeTail;
726 decrStIdx(store_idx);
727
728 while (stores != 0 &&
729 storeQueue[store_idx].inst->seqNum > squashed_num) {
730 // Instructions marked as can WB are already committed.
731 if (storeQueue[store_idx].canWB) {
732 break;
733 }
734
735 DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
736 "idx:%i [sn:%lli]\n",
737 storeQueue[store_idx].inst->readPC(),
738 store_idx, storeQueue[store_idx].inst->seqNum);
739
740 // I don't think this can happen. It should have been cleared
741 // by the stalling load.
742 if (isStalled() &&
743 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
744 panic("Is stalled should have been cleared by stalling load!\n");
745 stalled = false;
746 stallingStoreIsn = 0;
747 }
748
749 // Clear the smart pointer to make sure it is decremented.
750 storeQueue[store_idx].inst->setSquashed();
751 storeQueue[store_idx].inst = NULL;
752 storeQueue[store_idx].canWB = 0;
753
754 storeQueue[store_idx].req = NULL;
755 --stores;
756
757 // Inefficient!
758 storeTail = store_idx;
759
760 decrStIdx(store_idx);
761 ++lsqSquashedStores;
762 }
763}
764
765template <class Impl>
766void
767LSQUnit<Impl>::storePostSend(PacketPtr pkt)
768{
769 if (isStalled() &&
770 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
771 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
772 "load idx:%i\n",
773 stallingStoreIsn, stallingLoadIdx);
774 stalled = false;
775 stallingStoreIsn = 0;
776 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
777 }
778
779 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
780 // The store is basically completed at this time. This
781 // only works so long as the checker doesn't try to
782 // verify the value in memory for stores.
783 storeQueue[storeWBIdx].inst->setCompleted();
784#if USE_CHECKER
785 if (cpu->checker) {
786 cpu->checker->verify(storeQueue[storeWBIdx].inst);
787 }
788#endif
789 }
790
791 if (pkt->result != Packet::Success) {
792 DPRINTF(LSQUnit,"D-Cache Write Miss on idx:%i!\n",
793 storeWBIdx);
794
795 DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
796 storeQueue[storeWBIdx].inst->seqNum);
797
798 //mshrSeqNums.push_back(storeQueue[storeWBIdx].inst->seqNum);
799
800 //DPRINTF(LSQUnit, "Added MSHR. count = %i\n",mshrSeqNums.size());
801
802 // @todo: Increment stat here.
803 } else {
804 DPRINTF(LSQUnit,"D-Cache: Write Hit on idx:%i !\n",
805 storeWBIdx);
806
807 DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
808 storeQueue[storeWBIdx].inst->seqNum);
809 }
810
811 incrStIdx(storeWBIdx);
812}
813
814template <class Impl>
815void
816LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
817{
818 iewStage->wakeCPU();
819
820 // Squashed instructions do not need to complete their access.
821 if (inst->isSquashed()) {
822 iewStage->decrWb(inst->seqNum);
823 assert(!inst->isStore());
824 ++lsqIgnoredResponses;
825 return;
826 }
827
828 if (!inst->isExecuted()) {
829 inst->setExecuted();
830
831 // Complete access to copy data to proper place.
832 inst->completeAcc(pkt);
833 }
834
835 // Need to insert instruction into queue to commit
836 iewStage->instToCommit(inst);
837
838 iewStage->activityThisCycle();
839}
840
841template <class Impl>
842void
843LSQUnit<Impl>::completeStore(int store_idx)
844{
845 assert(storeQueue[store_idx].inst);
846 storeQueue[store_idx].completed = true;
847 --storesToWB;
848 // A bit conservative because a store completion may not free up entries,
849 // but hopefully avoids two store completions in one cycle from making
850 // the CPU tick twice.
851 cpu->wakeCPU();
852 cpu->activityThisCycle();
853
854 if (store_idx == storeHead) {
855 do {
856 incrStIdx(storeHead);
857
858 --stores;
859 } while (storeQueue[storeHead].completed &&
860 storeHead != storeTail);
861
862 iewStage->updateLSQNextCycle = true;
863 }
864
865 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
866 "idx:%i\n",
867 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
868
869 if (isStalled() &&
870 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
871 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
872 "load idx:%i\n",
873 stallingStoreIsn, stallingLoadIdx);
874 stalled = false;
875 stallingStoreIsn = 0;
876 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
877 }
878
879 storeQueue[store_idx].inst->setCompleted();
880
881 // Tell the checker we've completed this instruction. Some stores
882 // may get reported twice to the checker, but the checker can
883 // handle that case.
884#if USE_CHECKER
885 if (cpu->checker) {
886 cpu->checker->verify(storeQueue[store_idx].inst);
887 }
888#endif
889}
890
891template <class Impl>
892void
893LSQUnit<Impl>::recvRetry()
894{
895 if (isStoreBlocked) {
896 assert(retryPkt != NULL);
897
898 if (dcachePort->sendTiming(retryPkt)) {
899 if (retryPkt->result == Packet::BadAddress) {
900 panic("LSQ sent out a bad address for a completed store!");
901 }
902 storePostSend(retryPkt);
903 retryPkt = NULL;
904 isStoreBlocked = false;
905 lsq->setRetryTid(-1);
906 } else {
907 // Still blocked!
908 ++lsqCacheBlocked;
909 lsq->setRetryTid(lsqID);
910 }
911 } else if (isLoadBlocked) {
912 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
913 "no need to resend packet.\n");
914 } else {
915 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
916 }
917}
918
919template <class Impl>
920inline void
921LSQUnit<Impl>::incrStIdx(int &store_idx)
922{
923 if (++store_idx >= SQEntries)
924 store_idx = 0;
925}
926
927template <class Impl>
928inline void
929LSQUnit<Impl>::decrStIdx(int &store_idx)
930{
931 if (--store_idx < 0)
932 store_idx += SQEntries;
933}
934
935template <class Impl>
936inline void
937LSQUnit<Impl>::incrLdIdx(int &load_idx)
938{
939 if (++load_idx >= LQEntries)
940 load_idx = 0;
941}
942
943template <class Impl>
944inline void
945LSQUnit<Impl>::decrLdIdx(int &load_idx)
946{
947 if (--load_idx < 0)
948 load_idx += LQEntries;
949}
950
951template <class Impl>
952void
953LSQUnit<Impl>::dumpInsts()
954{
955 cprintf("Load store queue: Dumping instructions.\n");
956 cprintf("Load queue size: %i\n", loads);
957 cprintf("Load queue: ");
958
959 int load_idx = loadHead;
960
961 while (load_idx != loadTail && loadQueue[load_idx]) {
962 cprintf("%#x ", loadQueue[load_idx]->readPC());
963
964 incrLdIdx(load_idx);
965 }
966
967 cprintf("Store queue size: %i\n", stores);
968 cprintf("Store queue: ");
969
970 int store_idx = storeHead;
971
972 while (store_idx != storeTail && storeQueue[store_idx].inst) {
973 cprintf("%#x ", storeQueue[store_idx].inst->readPC());
974
975 incrStIdx(store_idx);
976 }
977
978 cprintf("\n");
979}
| 637 }
638 } else {
639 // Non-store conditionals do not need a writeback.
640 state->noWB = true;
641 }
642
643 if (!dcachePort->sendTiming(data_pkt)) {
644 if (data_pkt->result == Packet::BadAddress) {
645 panic("LSQ sent out a bad address for a completed store!");
646 }
647 // Need to handle becoming blocked on a store.
648 DPRINTF(IEW, "D-Cache became blcoked when writing [sn:%lli], will"
649 "retry later\n",
650 inst->seqNum);
651 isStoreBlocked = true;
652 ++lsqCacheBlocked;
653 assert(retryPkt == NULL);
654 retryPkt = data_pkt;
655 lsq->setRetryTid(lsqID);
656 } else {
657 storePostSend(data_pkt);
658 }
659 }
660
661 // Not sure this should set it to 0.
662 usedPorts = 0;
663
664 assert(stores >= 0 && storesToWB >= 0);
665}
666
667/*template <class Impl>
668void
669LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
670{
671 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
672 mshrSeqNums.end(),
673 seqNum);
674
675 if (mshr_it != mshrSeqNums.end()) {
676 mshrSeqNums.erase(mshr_it);
677 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
678 }
679}*/
680
681template <class Impl>
682void
683LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
684{
685 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
686 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
687
688 int load_idx = loadTail;
689 decrLdIdx(load_idx);
690
691 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
692 DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
693 "[sn:%lli]\n",
694 loadQueue[load_idx]->readPC(),
695 loadQueue[load_idx]->seqNum);
696
697 if (isStalled() && load_idx == stallingLoadIdx) {
698 stalled = false;
699 stallingStoreIsn = 0;
700 stallingLoadIdx = 0;
701 }
702
703 // Clear the smart pointer to make sure it is decremented.
704 loadQueue[load_idx]->setSquashed();
705 loadQueue[load_idx] = NULL;
706 --loads;
707
708 // Inefficient!
709 loadTail = load_idx;
710
711 decrLdIdx(load_idx);
712 ++lsqSquashedLoads;
713 }
714
715 if (isLoadBlocked) {
716 if (squashed_num < blockedLoadSeqNum) {
717 isLoadBlocked = false;
718 loadBlockedHandled = false;
719 blockedLoadSeqNum = 0;
720 }
721 }
722
723 int store_idx = storeTail;
724 decrStIdx(store_idx);
725
726 while (stores != 0 &&
727 storeQueue[store_idx].inst->seqNum > squashed_num) {
728 // Instructions marked as can WB are already committed.
729 if (storeQueue[store_idx].canWB) {
730 break;
731 }
732
733 DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
734 "idx:%i [sn:%lli]\n",
735 storeQueue[store_idx].inst->readPC(),
736 store_idx, storeQueue[store_idx].inst->seqNum);
737
738 // I don't think this can happen. It should have been cleared
739 // by the stalling load.
740 if (isStalled() &&
741 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
742 panic("Is stalled should have been cleared by stalling load!\n");
743 stalled = false;
744 stallingStoreIsn = 0;
745 }
746
747 // Clear the smart pointer to make sure it is decremented.
748 storeQueue[store_idx].inst->setSquashed();
749 storeQueue[store_idx].inst = NULL;
750 storeQueue[store_idx].canWB = 0;
751
752 storeQueue[store_idx].req = NULL;
753 --stores;
754
755 // Inefficient!
756 storeTail = store_idx;
757
758 decrStIdx(store_idx);
759 ++lsqSquashedStores;
760 }
761}
762
763template <class Impl>
764void
765LSQUnit<Impl>::storePostSend(PacketPtr pkt)
766{
767 if (isStalled() &&
768 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
769 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
770 "load idx:%i\n",
771 stallingStoreIsn, stallingLoadIdx);
772 stalled = false;
773 stallingStoreIsn = 0;
774 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
775 }
776
777 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
778 // The store is basically completed at this time. This
779 // only works so long as the checker doesn't try to
780 // verify the value in memory for stores.
781 storeQueue[storeWBIdx].inst->setCompleted();
782#if USE_CHECKER
783 if (cpu->checker) {
784 cpu->checker->verify(storeQueue[storeWBIdx].inst);
785 }
786#endif
787 }
788
789 if (pkt->result != Packet::Success) {
790 DPRINTF(LSQUnit,"D-Cache Write Miss on idx:%i!\n",
791 storeWBIdx);
792
793 DPRINTF(Activity, "Active st accessing mem miss [sn:%lli]\n",
794 storeQueue[storeWBIdx].inst->seqNum);
795
796 //mshrSeqNums.push_back(storeQueue[storeWBIdx].inst->seqNum);
797
798 //DPRINTF(LSQUnit, "Added MSHR. count = %i\n",mshrSeqNums.size());
799
800 // @todo: Increment stat here.
801 } else {
802 DPRINTF(LSQUnit,"D-Cache: Write Hit on idx:%i !\n",
803 storeWBIdx);
804
805 DPRINTF(Activity, "Active st accessing mem hit [sn:%lli]\n",
806 storeQueue[storeWBIdx].inst->seqNum);
807 }
808
809 incrStIdx(storeWBIdx);
810}
811
812template <class Impl>
813void
814LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
815{
816 iewStage->wakeCPU();
817
818 // Squashed instructions do not need to complete their access.
819 if (inst->isSquashed()) {
820 iewStage->decrWb(inst->seqNum);
821 assert(!inst->isStore());
822 ++lsqIgnoredResponses;
823 return;
824 }
825
826 if (!inst->isExecuted()) {
827 inst->setExecuted();
828
829 // Complete access to copy data to proper place.
830 inst->completeAcc(pkt);
831 }
832
833 // Need to insert instruction into queue to commit
834 iewStage->instToCommit(inst);
835
836 iewStage->activityThisCycle();
837}
838
839template <class Impl>
840void
841LSQUnit<Impl>::completeStore(int store_idx)
842{
843 assert(storeQueue[store_idx].inst);
844 storeQueue[store_idx].completed = true;
845 --storesToWB;
846 // A bit conservative because a store completion may not free up entries,
847 // but hopefully avoids two store completions in one cycle from making
848 // the CPU tick twice.
849 cpu->wakeCPU();
850 cpu->activityThisCycle();
851
852 if (store_idx == storeHead) {
853 do {
854 incrStIdx(storeHead);
855
856 --stores;
857 } while (storeQueue[storeHead].completed &&
858 storeHead != storeTail);
859
860 iewStage->updateLSQNextCycle = true;
861 }
862
863 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
864 "idx:%i\n",
865 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
866
867 if (isStalled() &&
868 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
869 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
870 "load idx:%i\n",
871 stallingStoreIsn, stallingLoadIdx);
872 stalled = false;
873 stallingStoreIsn = 0;
874 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
875 }
876
877 storeQueue[store_idx].inst->setCompleted();
878
879 // Tell the checker we've completed this instruction. Some stores
880 // may get reported twice to the checker, but the checker can
881 // handle that case.
882#if USE_CHECKER
883 if (cpu->checker) {
884 cpu->checker->verify(storeQueue[store_idx].inst);
885 }
886#endif
887}
888
889template <class Impl>
890void
891LSQUnit<Impl>::recvRetry()
892{
893 if (isStoreBlocked) {
894 assert(retryPkt != NULL);
895
896 if (dcachePort->sendTiming(retryPkt)) {
897 if (retryPkt->result == Packet::BadAddress) {
898 panic("LSQ sent out a bad address for a completed store!");
899 }
900 storePostSend(retryPkt);
901 retryPkt = NULL;
902 isStoreBlocked = false;
903 lsq->setRetryTid(-1);
904 } else {
905 // Still blocked!
906 ++lsqCacheBlocked;
907 lsq->setRetryTid(lsqID);
908 }
909 } else if (isLoadBlocked) {
910 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
911 "no need to resend packet.\n");
912 } else {
913 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
914 }
915}
916
917template <class Impl>
918inline void
919LSQUnit<Impl>::incrStIdx(int &store_idx)
920{
921 if (++store_idx >= SQEntries)
922 store_idx = 0;
923}
924
925template <class Impl>
926inline void
927LSQUnit<Impl>::decrStIdx(int &store_idx)
928{
929 if (--store_idx < 0)
930 store_idx += SQEntries;
931}
932
933template <class Impl>
934inline void
935LSQUnit<Impl>::incrLdIdx(int &load_idx)
936{
937 if (++load_idx >= LQEntries)
938 load_idx = 0;
939}
940
941template <class Impl>
942inline void
943LSQUnit<Impl>::decrLdIdx(int &load_idx)
944{
945 if (--load_idx < 0)
946 load_idx += LQEntries;
947}
948
949template <class Impl>
950void
951LSQUnit<Impl>::dumpInsts()
952{
953 cprintf("Load store queue: Dumping instructions.\n");
954 cprintf("Load queue size: %i\n", loads);
955 cprintf("Load queue: ");
956
957 int load_idx = loadHead;
958
959 while (load_idx != loadTail && loadQueue[load_idx]) {
960 cprintf("%#x ", loadQueue[load_idx]->readPC());
961
962 incrLdIdx(load_idx);
963 }
964
965 cprintf("Store queue size: %i\n", stores);
966 cprintf("Store queue: ");
967
968 int store_idx = storeHead;
969
970 while (store_idx != storeTail && storeQueue[store_idx].inst) {
971 cprintf("%#x ", storeQueue[store_idx].inst->readPC());
972
973 incrStIdx(store_idx);
974 }
975
976 cprintf("\n");
977}
|