1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 * Korey Sewell
42 */
43
| 1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 * Korey Sewell
42 */
43
|
| 44#include "arch/generic/debugfaults.hh"
|
44#include "arch/locked_mem.hh"
45#include "base/str.hh"
46#include "config/the_isa.hh"
47#include "config/use_checker.hh"
48#include "cpu/o3/lsq.hh"
49#include "cpu/o3/lsq_unit.hh"
50#include "debug/Activity.hh"
51#include "debug/IEW.hh"
52#include "debug/LSQUnit.hh"
53#include "mem/packet.hh"
54#include "mem/request.hh"
55
56#if USE_CHECKER
57#include "cpu/checker/cpu.hh"
58#endif
59
60template<class Impl>
61LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
62 LSQUnit *lsq_ptr)
63 : Event(Default_Pri, AutoDelete),
64 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
65{
66}
67
68template<class Impl>
69void
70LSQUnit<Impl>::WritebackEvent::process()
71{
72 if (!lsqPtr->isSwitchedOut()) {
73 lsqPtr->writeback(inst, pkt);
74 }
75
76 if (pkt->senderState)
77 delete pkt->senderState;
78
79 delete pkt->req;
80 delete pkt;
81}
82
83template<class Impl>
84const char *
85LSQUnit<Impl>::WritebackEvent::description() const
86{
87 return "Store writeback";
88}
89
90template<class Impl>
91void
92LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
93{
94 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
95 DynInstPtr inst = state->inst;
96 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
97 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
98
99 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
100
101 assert(!pkt->wasNacked());
102
103 // If this is a split access, wait until all packets are received.
104 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
105 delete pkt->req;
106 delete pkt;
107 return;
108 }
109
110 if (isSwitchedOut() || inst->isSquashed()) {
111 iewStage->decrWb(inst->seqNum);
112 } else {
113 if (!state->noWB) {
114 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
115 !state->isLoad) {
116 writeback(inst, pkt);
117 } else {
118 writeback(inst, state->mainPkt);
119 }
120 }
121
122 if (inst->isStore()) {
123 completeStore(state->idx);
124 }
125 }
126
127 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
128 delete state->mainPkt->req;
129 delete state->mainPkt;
130 }
131 delete state;
132 delete pkt->req;
133 delete pkt;
134}
135
136template <class Impl>
137LSQUnit<Impl>::LSQUnit()
138 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
139 isStoreBlocked(false), isLoadBlocked(false),
140 loadBlockedHandled(false), hasPendingPkt(false)
141{
142}
143
144template<class Impl>
145void
146LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
147 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
148 unsigned id)
149{
150 cpu = cpu_ptr;
151 iewStage = iew_ptr;
152
153 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
154
155 switchedOut = false;
156
157 cacheBlockMask = 0;
158
159 lsq = lsq_ptr;
160
161 lsqID = id;
162
163 // Add 1 for the sentinel entry (they are circular queues).
164 LQEntries = maxLQEntries + 1;
165 SQEntries = maxSQEntries + 1;
166
167 loadQueue.resize(LQEntries);
168 storeQueue.resize(SQEntries);
169
170 depCheckShift = params->LSQDepCheckShift;
171 checkLoads = params->LSQCheckLoads;
172
173 loadHead = loadTail = 0;
174
175 storeHead = storeWBIdx = storeTail = 0;
176
177 usedPorts = 0;
178 cachePorts = params->cachePorts;
179
180 retryPkt = NULL;
181 memDepViolator = NULL;
182
183 blockedLoadSeqNum = 0;
184}
185
186template<class Impl>
187std::string
188LSQUnit<Impl>::name() const
189{
190 if (Impl::MaxThreads == 1) {
191 return iewStage->name() + ".lsq";
192 } else {
193 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
194 }
195}
196
197template<class Impl>
198void
199LSQUnit<Impl>::regStats()
200{
201 lsqForwLoads
202 .name(name() + ".forwLoads")
203 .desc("Number of loads that had data forwarded from stores");
204
205 invAddrLoads
206 .name(name() + ".invAddrLoads")
207 .desc("Number of loads ignored due to an invalid address");
208
209 lsqSquashedLoads
210 .name(name() + ".squashedLoads")
211 .desc("Number of loads squashed");
212
213 lsqIgnoredResponses
214 .name(name() + ".ignoredResponses")
215 .desc("Number of memory responses ignored because the instruction is squashed");
216
217 lsqMemOrderViolation
218 .name(name() + ".memOrderViolation")
219 .desc("Number of memory ordering violations");
220
221 lsqSquashedStores
222 .name(name() + ".squashedStores")
223 .desc("Number of stores squashed");
224
225 invAddrSwpfs
226 .name(name() + ".invAddrSwpfs")
227 .desc("Number of software prefetches ignored due to an invalid address");
228
229 lsqBlockedLoads
230 .name(name() + ".blockedLoads")
231 .desc("Number of blocked loads due to partial load-store forwarding");
232
233 lsqRescheduledLoads
234 .name(name() + ".rescheduledLoads")
235 .desc("Number of loads that were rescheduled");
236
237 lsqCacheBlocked
238 .name(name() + ".cacheBlocked")
239 .desc("Number of times an access to memory failed due to the cache being blocked");
240}
241
242template<class Impl>
243void
244LSQUnit<Impl>::setDcachePort(Port *dcache_port)
245{
246 dcachePort = dcache_port;
247
248#if USE_CHECKER
249 if (cpu->checker) {
250 cpu->checker->setDcachePort(dcachePort);
251 }
252#endif
253}
254
255template<class Impl>
256void
257LSQUnit<Impl>::clearLQ()
258{
259 loadQueue.clear();
260}
261
262template<class Impl>
263void
264LSQUnit<Impl>::clearSQ()
265{
266 storeQueue.clear();
267}
268
269template<class Impl>
270void
271LSQUnit<Impl>::switchOut()
272{
273 switchedOut = true;
274 for (int i = 0; i < loadQueue.size(); ++i) {
275 assert(!loadQueue[i]);
276 loadQueue[i] = NULL;
277 }
278
279 assert(storesToWB == 0);
280}
281
282template<class Impl>
283void
284LSQUnit<Impl>::takeOverFrom()
285{
286 switchedOut = false;
287 loads = stores = storesToWB = 0;
288
289 loadHead = loadTail = 0;
290
291 storeHead = storeWBIdx = storeTail = 0;
292
293 usedPorts = 0;
294
295 memDepViolator = NULL;
296
297 blockedLoadSeqNum = 0;
298
299 stalled = false;
300 isLoadBlocked = false;
301 loadBlockedHandled = false;
302
303 // Just incase the memory system changed out from under us
304 cacheBlockMask = 0;
305}
306
307template<class Impl>
308void
309LSQUnit<Impl>::resizeLQ(unsigned size)
310{
311 unsigned size_plus_sentinel = size + 1;
312 assert(size_plus_sentinel >= LQEntries);
313
314 if (size_plus_sentinel > LQEntries) {
315 while (size_plus_sentinel > loadQueue.size()) {
316 DynInstPtr dummy;
317 loadQueue.push_back(dummy);
318 LQEntries++;
319 }
320 } else {
321 LQEntries = size_plus_sentinel;
322 }
323
324}
325
326template<class Impl>
327void
328LSQUnit<Impl>::resizeSQ(unsigned size)
329{
330 unsigned size_plus_sentinel = size + 1;
331 if (size_plus_sentinel > SQEntries) {
332 while (size_plus_sentinel > storeQueue.size()) {
333 SQEntry dummy;
334 storeQueue.push_back(dummy);
335 SQEntries++;
336 }
337 } else {
338 SQEntries = size_plus_sentinel;
339 }
340}
341
342template <class Impl>
343void
344LSQUnit<Impl>::insert(DynInstPtr &inst)
345{
346 assert(inst->isMemRef());
347
348 assert(inst->isLoad() || inst->isStore());
349
350 if (inst->isLoad()) {
351 insertLoad(inst);
352 } else {
353 insertStore(inst);
354 }
355
356 inst->setInLSQ();
357}
358
359template <class Impl>
360void
361LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
362{
363 assert((loadTail + 1) % LQEntries != loadHead);
364 assert(loads < LQEntries);
365
366 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
367 load_inst->pcState(), loadTail, load_inst->seqNum);
368
369 load_inst->lqIdx = loadTail;
370
371 if (stores == 0) {
372 load_inst->sqIdx = -1;
373 } else {
374 load_inst->sqIdx = storeTail;
375 }
376
377 loadQueue[loadTail] = load_inst;
378
379 incrLdIdx(loadTail);
380
381 ++loads;
382}
383
384template <class Impl>
385void
386LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
387{
388 // Make sure it is not full before inserting an instruction.
389 assert((storeTail + 1) % SQEntries != storeHead);
390 assert(stores < SQEntries);
391
392 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
393 store_inst->pcState(), storeTail, store_inst->seqNum);
394
395 store_inst->sqIdx = storeTail;
396 store_inst->lqIdx = loadTail;
397
398 storeQueue[storeTail] = SQEntry(store_inst);
399
400 incrStIdx(storeTail);
401
402 ++stores;
403}
404
405template <class Impl>
406typename Impl::DynInstPtr
407LSQUnit<Impl>::getMemDepViolator()
408{
409 DynInstPtr temp = memDepViolator;
410
411 memDepViolator = NULL;
412
413 return temp;
414}
415
416template <class Impl>
417unsigned
418LSQUnit<Impl>::numFreeEntries()
419{
420 unsigned free_lq_entries = LQEntries - loads;
421 unsigned free_sq_entries = SQEntries - stores;
422
423 // Both the LQ and SQ entries have an extra dummy entry to differentiate
424 // empty/full conditions. Subtract 1 from the free entries.
425 if (free_lq_entries < free_sq_entries) {
426 return free_lq_entries - 1;
427 } else {
428 return free_sq_entries - 1;
429 }
430}
431
432template <class Impl>
433int
434LSQUnit<Impl>::numLoadsReady()
435{
436 int load_idx = loadHead;
437 int retval = 0;
438
439 while (load_idx != loadTail) {
440 assert(loadQueue[load_idx]);
441
442 if (loadQueue[load_idx]->readyToIssue()) {
443 ++retval;
444 }
445 }
446
447 return retval;
448}
449
450template <class Impl>
451void
452LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
453{
454 int load_idx = loadHead;
455
456 if (!cacheBlockMask) {
457 assert(dcachePort);
458 Addr bs = dcachePort->peerBlockSize();
459
460 // Make sure we actually got a size
461 assert(bs != 0);
462
463 cacheBlockMask = ~(bs - 1);
464 }
465
466 // If this is the only load in the LSQ we don't care
467 if (load_idx == loadTail)
468 return;
469 incrLdIdx(load_idx);
470
471 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
472 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
473 while (load_idx != loadTail) {
474 DynInstPtr ld_inst = loadQueue[load_idx];
475
476 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
477 incrLdIdx(load_idx);
478 continue;
479 }
480
481 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
482 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
483 ld_inst->seqNum, load_addr, invalidate_addr);
484
485 if (load_addr == invalidate_addr) {
486 if (ld_inst->possibleLoadViolation) {
487 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
488 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
489
490 // Mark the load for re-execution
491 ld_inst->fault = new ReExec;
492 } else {
493 // If a older load checks this and it's true
494 // then we might have missed the snoop
495 // in which case we need to invalidate to be sure
496 ld_inst->hitExternalSnoop = true;
497 }
498 }
499 incrLdIdx(load_idx);
500 }
501 return;
502}
503
504template <class Impl>
505Fault
506LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
507{
508 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
509 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
510
511 /** @todo in theory you only need to check an instruction that has executed
512 * however, there isn't a good way in the pipeline at the moment to check
513 * all instructions that will execute before the store writes back. Thus,
514 * like the implementation that came before it, we're overly conservative.
515 */
516 while (load_idx != loadTail) {
517 DynInstPtr ld_inst = loadQueue[load_idx];
518 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
519 incrLdIdx(load_idx);
520 continue;
521 }
522
523 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
524 Addr ld_eff_addr2 =
525 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
526
527 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
528 if (inst->isLoad()) {
529 // If this load is to the same block as an external snoop
530 // invalidate that we've observed then the load needs to be
531 // squashed as it could have newer data
532 if (ld_inst->hitExternalSnoop) {
533 if (!memDepViolator ||
534 ld_inst->seqNum < memDepViolator->seqNum) {
535 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
536 " and [sn:%lli] at address %#x\n", inst->seqNum,
537 ld_inst->seqNum, ld_eff_addr1);
538 memDepViolator = ld_inst;
539
540 ++lsqMemOrderViolation;
541
| 45#include "arch/locked_mem.hh"
46#include "base/str.hh"
47#include "config/the_isa.hh"
48#include "config/use_checker.hh"
49#include "cpu/o3/lsq.hh"
50#include "cpu/o3/lsq_unit.hh"
51#include "debug/Activity.hh"
52#include "debug/IEW.hh"
53#include "debug/LSQUnit.hh"
54#include "mem/packet.hh"
55#include "mem/request.hh"
56
57#if USE_CHECKER
58#include "cpu/checker/cpu.hh"
59#endif
60
61template<class Impl>
62LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
63 LSQUnit *lsq_ptr)
64 : Event(Default_Pri, AutoDelete),
65 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
66{
67}
68
69template<class Impl>
70void
71LSQUnit<Impl>::WritebackEvent::process()
72{
73 if (!lsqPtr->isSwitchedOut()) {
74 lsqPtr->writeback(inst, pkt);
75 }
76
77 if (pkt->senderState)
78 delete pkt->senderState;
79
80 delete pkt->req;
81 delete pkt;
82}
83
84template<class Impl>
85const char *
86LSQUnit<Impl>::WritebackEvent::description() const
87{
88 return "Store writeback";
89}
90
91template<class Impl>
92void
93LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
94{
95 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
96 DynInstPtr inst = state->inst;
97 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
98 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
99
100 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
101
102 assert(!pkt->wasNacked());
103
104 // If this is a split access, wait until all packets are received.
105 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
106 delete pkt->req;
107 delete pkt;
108 return;
109 }
110
111 if (isSwitchedOut() || inst->isSquashed()) {
112 iewStage->decrWb(inst->seqNum);
113 } else {
114 if (!state->noWB) {
115 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
116 !state->isLoad) {
117 writeback(inst, pkt);
118 } else {
119 writeback(inst, state->mainPkt);
120 }
121 }
122
123 if (inst->isStore()) {
124 completeStore(state->idx);
125 }
126 }
127
128 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
129 delete state->mainPkt->req;
130 delete state->mainPkt;
131 }
132 delete state;
133 delete pkt->req;
134 delete pkt;
135}
136
137template <class Impl>
138LSQUnit<Impl>::LSQUnit()
139 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
140 isStoreBlocked(false), isLoadBlocked(false),
141 loadBlockedHandled(false), hasPendingPkt(false)
142{
143}
144
145template<class Impl>
146void
147LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
148 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
149 unsigned id)
150{
151 cpu = cpu_ptr;
152 iewStage = iew_ptr;
153
154 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
155
156 switchedOut = false;
157
158 cacheBlockMask = 0;
159
160 lsq = lsq_ptr;
161
162 lsqID = id;
163
164 // Add 1 for the sentinel entry (they are circular queues).
165 LQEntries = maxLQEntries + 1;
166 SQEntries = maxSQEntries + 1;
167
168 loadQueue.resize(LQEntries);
169 storeQueue.resize(SQEntries);
170
171 depCheckShift = params->LSQDepCheckShift;
172 checkLoads = params->LSQCheckLoads;
173
174 loadHead = loadTail = 0;
175
176 storeHead = storeWBIdx = storeTail = 0;
177
178 usedPorts = 0;
179 cachePorts = params->cachePorts;
180
181 retryPkt = NULL;
182 memDepViolator = NULL;
183
184 blockedLoadSeqNum = 0;
185}
186
187template<class Impl>
188std::string
189LSQUnit<Impl>::name() const
190{
191 if (Impl::MaxThreads == 1) {
192 return iewStage->name() + ".lsq";
193 } else {
194 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
195 }
196}
197
198template<class Impl>
199void
200LSQUnit<Impl>::regStats()
201{
202 lsqForwLoads
203 .name(name() + ".forwLoads")
204 .desc("Number of loads that had data forwarded from stores");
205
206 invAddrLoads
207 .name(name() + ".invAddrLoads")
208 .desc("Number of loads ignored due to an invalid address");
209
210 lsqSquashedLoads
211 .name(name() + ".squashedLoads")
212 .desc("Number of loads squashed");
213
214 lsqIgnoredResponses
215 .name(name() + ".ignoredResponses")
216 .desc("Number of memory responses ignored because the instruction is squashed");
217
218 lsqMemOrderViolation
219 .name(name() + ".memOrderViolation")
220 .desc("Number of memory ordering violations");
221
222 lsqSquashedStores
223 .name(name() + ".squashedStores")
224 .desc("Number of stores squashed");
225
226 invAddrSwpfs
227 .name(name() + ".invAddrSwpfs")
228 .desc("Number of software prefetches ignored due to an invalid address");
229
230 lsqBlockedLoads
231 .name(name() + ".blockedLoads")
232 .desc("Number of blocked loads due to partial load-store forwarding");
233
234 lsqRescheduledLoads
235 .name(name() + ".rescheduledLoads")
236 .desc("Number of loads that were rescheduled");
237
238 lsqCacheBlocked
239 .name(name() + ".cacheBlocked")
240 .desc("Number of times an access to memory failed due to the cache being blocked");
241}
242
243template<class Impl>
244void
245LSQUnit<Impl>::setDcachePort(Port *dcache_port)
246{
247 dcachePort = dcache_port;
248
249#if USE_CHECKER
250 if (cpu->checker) {
251 cpu->checker->setDcachePort(dcachePort);
252 }
253#endif
254}
255
256template<class Impl>
257void
258LSQUnit<Impl>::clearLQ()
259{
260 loadQueue.clear();
261}
262
263template<class Impl>
264void
265LSQUnit<Impl>::clearSQ()
266{
267 storeQueue.clear();
268}
269
270template<class Impl>
271void
272LSQUnit<Impl>::switchOut()
273{
274 switchedOut = true;
275 for (int i = 0; i < loadQueue.size(); ++i) {
276 assert(!loadQueue[i]);
277 loadQueue[i] = NULL;
278 }
279
280 assert(storesToWB == 0);
281}
282
283template<class Impl>
284void
285LSQUnit<Impl>::takeOverFrom()
286{
287 switchedOut = false;
288 loads = stores = storesToWB = 0;
289
290 loadHead = loadTail = 0;
291
292 storeHead = storeWBIdx = storeTail = 0;
293
294 usedPorts = 0;
295
296 memDepViolator = NULL;
297
298 blockedLoadSeqNum = 0;
299
300 stalled = false;
301 isLoadBlocked = false;
302 loadBlockedHandled = false;
303
304 // Just incase the memory system changed out from under us
305 cacheBlockMask = 0;
306}
307
308template<class Impl>
309void
310LSQUnit<Impl>::resizeLQ(unsigned size)
311{
312 unsigned size_plus_sentinel = size + 1;
313 assert(size_plus_sentinel >= LQEntries);
314
315 if (size_plus_sentinel > LQEntries) {
316 while (size_plus_sentinel > loadQueue.size()) {
317 DynInstPtr dummy;
318 loadQueue.push_back(dummy);
319 LQEntries++;
320 }
321 } else {
322 LQEntries = size_plus_sentinel;
323 }
324
325}
326
327template<class Impl>
328void
329LSQUnit<Impl>::resizeSQ(unsigned size)
330{
331 unsigned size_plus_sentinel = size + 1;
332 if (size_plus_sentinel > SQEntries) {
333 while (size_plus_sentinel > storeQueue.size()) {
334 SQEntry dummy;
335 storeQueue.push_back(dummy);
336 SQEntries++;
337 }
338 } else {
339 SQEntries = size_plus_sentinel;
340 }
341}
342
343template <class Impl>
344void
345LSQUnit<Impl>::insert(DynInstPtr &inst)
346{
347 assert(inst->isMemRef());
348
349 assert(inst->isLoad() || inst->isStore());
350
351 if (inst->isLoad()) {
352 insertLoad(inst);
353 } else {
354 insertStore(inst);
355 }
356
357 inst->setInLSQ();
358}
359
360template <class Impl>
361void
362LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
363{
364 assert((loadTail + 1) % LQEntries != loadHead);
365 assert(loads < LQEntries);
366
367 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
368 load_inst->pcState(), loadTail, load_inst->seqNum);
369
370 load_inst->lqIdx = loadTail;
371
372 if (stores == 0) {
373 load_inst->sqIdx = -1;
374 } else {
375 load_inst->sqIdx = storeTail;
376 }
377
378 loadQueue[loadTail] = load_inst;
379
380 incrLdIdx(loadTail);
381
382 ++loads;
383}
384
385template <class Impl>
386void
387LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
388{
389 // Make sure it is not full before inserting an instruction.
390 assert((storeTail + 1) % SQEntries != storeHead);
391 assert(stores < SQEntries);
392
393 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
394 store_inst->pcState(), storeTail, store_inst->seqNum);
395
396 store_inst->sqIdx = storeTail;
397 store_inst->lqIdx = loadTail;
398
399 storeQueue[storeTail] = SQEntry(store_inst);
400
401 incrStIdx(storeTail);
402
403 ++stores;
404}
405
406template <class Impl>
407typename Impl::DynInstPtr
408LSQUnit<Impl>::getMemDepViolator()
409{
410 DynInstPtr temp = memDepViolator;
411
412 memDepViolator = NULL;
413
414 return temp;
415}
416
417template <class Impl>
418unsigned
419LSQUnit<Impl>::numFreeEntries()
420{
421 unsigned free_lq_entries = LQEntries - loads;
422 unsigned free_sq_entries = SQEntries - stores;
423
424 // Both the LQ and SQ entries have an extra dummy entry to differentiate
425 // empty/full conditions. Subtract 1 from the free entries.
426 if (free_lq_entries < free_sq_entries) {
427 return free_lq_entries - 1;
428 } else {
429 return free_sq_entries - 1;
430 }
431}
432
433template <class Impl>
434int
435LSQUnit<Impl>::numLoadsReady()
436{
437 int load_idx = loadHead;
438 int retval = 0;
439
440 while (load_idx != loadTail) {
441 assert(loadQueue[load_idx]);
442
443 if (loadQueue[load_idx]->readyToIssue()) {
444 ++retval;
445 }
446 }
447
448 return retval;
449}
450
451template <class Impl>
452void
453LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
454{
455 int load_idx = loadHead;
456
457 if (!cacheBlockMask) {
458 assert(dcachePort);
459 Addr bs = dcachePort->peerBlockSize();
460
461 // Make sure we actually got a size
462 assert(bs != 0);
463
464 cacheBlockMask = ~(bs - 1);
465 }
466
467 // If this is the only load in the LSQ we don't care
468 if (load_idx == loadTail)
469 return;
470 incrLdIdx(load_idx);
471
472 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
473 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
474 while (load_idx != loadTail) {
475 DynInstPtr ld_inst = loadQueue[load_idx];
476
477 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
478 incrLdIdx(load_idx);
479 continue;
480 }
481
482 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
483 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
484 ld_inst->seqNum, load_addr, invalidate_addr);
485
486 if (load_addr == invalidate_addr) {
487 if (ld_inst->possibleLoadViolation) {
488 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
489 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
490
491 // Mark the load for re-execution
492 ld_inst->fault = new ReExec;
493 } else {
494 // If a older load checks this and it's true
495 // then we might have missed the snoop
496 // in which case we need to invalidate to be sure
497 ld_inst->hitExternalSnoop = true;
498 }
499 }
500 incrLdIdx(load_idx);
501 }
502 return;
503}
504
505template <class Impl>
506Fault
507LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
508{
509 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
510 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
511
512 /** @todo in theory you only need to check an instruction that has executed
513 * however, there isn't a good way in the pipeline at the moment to check
514 * all instructions that will execute before the store writes back. Thus,
515 * like the implementation that came before it, we're overly conservative.
516 */
517 while (load_idx != loadTail) {
518 DynInstPtr ld_inst = loadQueue[load_idx];
519 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
520 incrLdIdx(load_idx);
521 continue;
522 }
523
524 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
525 Addr ld_eff_addr2 =
526 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
527
528 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
529 if (inst->isLoad()) {
530 // If this load is to the same block as an external snoop
531 // invalidate that we've observed then the load needs to be
532 // squashed as it could have newer data
533 if (ld_inst->hitExternalSnoop) {
534 if (!memDepViolator ||
535 ld_inst->seqNum < memDepViolator->seqNum) {
536 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
537 " and [sn:%lli] at address %#x\n", inst->seqNum,
538 ld_inst->seqNum, ld_eff_addr1);
539 memDepViolator = ld_inst;
540
541 ++lsqMemOrderViolation;
542
|
542 return TheISA::genMachineCheckFault();
| 543 return new GenericISA::M5PanicFault(
544 "Detected fault with inst [sn:%lli] and "
545 "[sn:%lli] at address %#x\n",
546 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
|
543 }
544 }
545
546 // Otherwise, mark the load has a possible load violation
547 // and if we see a snoop before it's commited, we need to squash
548 ld_inst->possibleLoadViolation = true;
549 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
550 " between instructions [sn:%lli] and [sn:%lli]\n",
551 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
552 } else {
553 // A load/store incorrectly passed this store.
554 // Check if we already have a violator, or if it's newer
555 // squash and refetch.
556 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
557 break;
558
559 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and [sn:%lli]"
560 " at address %#x\n", inst->seqNum, ld_inst->seqNum,
561 ld_eff_addr1);
562 memDepViolator = ld_inst;
563
564 ++lsqMemOrderViolation;
565
| 547 }
548 }
549
550 // Otherwise, mark the load has a possible load violation
551 // and if we see a snoop before it's commited, we need to squash
552 ld_inst->possibleLoadViolation = true;
553 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
554 " between instructions [sn:%lli] and [sn:%lli]\n",
555 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
556 } else {
557 // A load/store incorrectly passed this store.
558 // Check if we already have a violator, or if it's newer
559 // squash and refetch.
560 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
561 break;
562
563 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and [sn:%lli]"
564 " at address %#x\n", inst->seqNum, ld_inst->seqNum,
565 ld_eff_addr1);
566 memDepViolator = ld_inst;
567
568 ++lsqMemOrderViolation;
569
|
566 return TheISA::genMachineCheckFault();
| 570 return new GenericISA::M5PanicFault("Detected fault with "
571 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
572 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
|
567 }
568 }
569
570 incrLdIdx(load_idx);
571 }
572 return NoFault;
573}
574
575
576
577
578template <class Impl>
579Fault
580LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
581{
582 using namespace TheISA;
583 // Execute a specific load.
584 Fault load_fault = NoFault;
585
586 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
587 inst->pcState(), inst->seqNum);
588
589 assert(!inst->isSquashed());
590
591 load_fault = inst->initiateAcc();
592
593 if (inst->isTranslationDelayed() &&
594 load_fault == NoFault)
595 return load_fault;
596
597 // If the instruction faulted or predicated false, then we need to send it
598 // along to commit without the instruction completing.
599 if (load_fault != NoFault || inst->readPredicate() == false) {
600 // Send this instruction to commit, also make sure iew stage
601 // realizes there is activity.
602 // Mark it as executed unless it is an uncached load that
603 // needs to hit the head of commit.
604 if (inst->readPredicate() == false)
605 inst->forwardOldRegs();
606 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
607 inst->seqNum,
608 (load_fault != NoFault ? "fault" : "predication"));
609 if (!(inst->hasRequest() && inst->uncacheable()) ||
610 inst->isAtCommit()) {
611 inst->setExecuted();
612 }
613 iewStage->instToCommit(inst);
614 iewStage->activityThisCycle();
615 } else if (!loadBlocked()) {
616 assert(inst->effAddrValid);
617 int load_idx = inst->lqIdx;
618 incrLdIdx(load_idx);
619
620 if (checkLoads)
621 return checkViolations(load_idx, inst);
622 }
623
624 return load_fault;
625}
626
627template <class Impl>
628Fault
629LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
630{
631 using namespace TheISA;
632 // Make sure that a store exists.
633 assert(stores != 0);
634
635 int store_idx = store_inst->sqIdx;
636
637 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
638 store_inst->pcState(), store_inst->seqNum);
639
640 assert(!store_inst->isSquashed());
641
642 // Check the recently completed loads to see if any match this store's
643 // address. If so, then we have a memory ordering violation.
644 int load_idx = store_inst->lqIdx;
645
646 Fault store_fault = store_inst->initiateAcc();
647
648 if (store_inst->isTranslationDelayed() &&
649 store_fault == NoFault)
650 return store_fault;
651
652 if (store_inst->readPredicate() == false)
653 store_inst->forwardOldRegs();
654
655 if (storeQueue[store_idx].size == 0) {
656 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
657 store_inst->pcState(), store_inst->seqNum);
658
659 return store_fault;
660 } else if (store_inst->readPredicate() == false) {
661 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
662 store_inst->seqNum);
663 return store_fault;
664 }
665
666 assert(store_fault == NoFault);
667
668 if (store_inst->isStoreConditional()) {
669 // Store conditionals need to set themselves as able to
670 // writeback if we haven't had a fault by here.
671 storeQueue[store_idx].canWB = true;
672
673 ++storesToWB;
674 }
675
676 return checkViolations(load_idx, store_inst);
677
678}
679
680template <class Impl>
681void
682LSQUnit<Impl>::commitLoad()
683{
684 assert(loadQueue[loadHead]);
685
686 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
687 loadQueue[loadHead]->pcState());
688
689 loadQueue[loadHead] = NULL;
690
691 incrLdIdx(loadHead);
692
693 --loads;
694}
695
696template <class Impl>
697void
698LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
699{
700 assert(loads == 0 || loadQueue[loadHead]);
701
702 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
703 commitLoad();
704 }
705}
706
707template <class Impl>
708void
709LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
710{
711 assert(stores == 0 || storeQueue[storeHead].inst);
712
713 int store_idx = storeHead;
714
715 while (store_idx != storeTail) {
716 assert(storeQueue[store_idx].inst);
717 // Mark any stores that are now committed and have not yet
718 // been marked as able to write back.
719 if (!storeQueue[store_idx].canWB) {
720 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
721 break;
722 }
723 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
724 "%s [sn:%lli]\n",
725 storeQueue[store_idx].inst->pcState(),
726 storeQueue[store_idx].inst->seqNum);
727
728 storeQueue[store_idx].canWB = true;
729
730 ++storesToWB;
731 }
732
733 incrStIdx(store_idx);
734 }
735}
736
737template <class Impl>
738void
739LSQUnit<Impl>::writebackPendingStore()
740{
741 if (hasPendingPkt) {
742 assert(pendingPkt != NULL);
743
744 // If the cache is blocked, this will store the packet for retry.
745 if (sendStore(pendingPkt)) {
746 storePostSend(pendingPkt);
747 }
748 pendingPkt = NULL;
749 hasPendingPkt = false;
750 }
751}
752
753template <class Impl>
754void
755LSQUnit<Impl>::writebackStores()
756{
757 // First writeback the second packet from any split store that didn't
758 // complete last cycle because there weren't enough cache ports available.
759 if (TheISA::HasUnalignedMemAcc) {
760 writebackPendingStore();
761 }
762
763 while (storesToWB > 0 &&
764 storeWBIdx != storeTail &&
765 storeQueue[storeWBIdx].inst &&
766 storeQueue[storeWBIdx].canWB &&
767 usedPorts < cachePorts) {
768
769 if (isStoreBlocked || lsq->cacheBlocked()) {
770 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
771 " is blocked!\n");
772 break;
773 }
774
775 // Store didn't write any data so no need to write it back to
776 // memory.
777 if (storeQueue[storeWBIdx].size == 0) {
778 completeStore(storeWBIdx);
779
780 incrStIdx(storeWBIdx);
781
782 continue;
783 }
784
785 ++usedPorts;
786
787 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
788 incrStIdx(storeWBIdx);
789
790 continue;
791 }
792
793 assert(storeQueue[storeWBIdx].req);
794 assert(!storeQueue[storeWBIdx].committed);
795
796 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
797 assert(storeQueue[storeWBIdx].sreqLow);
798 assert(storeQueue[storeWBIdx].sreqHigh);
799 }
800
801 DynInstPtr inst = storeQueue[storeWBIdx].inst;
802
803 Request *req = storeQueue[storeWBIdx].req;
804 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
805 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
806
807 storeQueue[storeWBIdx].committed = true;
808
809 assert(!inst->memData);
810 inst->memData = new uint8_t[64];
811
812 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
813
814 MemCmd command =
815 req->isSwap() ? MemCmd::SwapReq :
816 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
817 PacketPtr data_pkt;
818 PacketPtr snd_data_pkt = NULL;
819
820 LSQSenderState *state = new LSQSenderState;
821 state->isLoad = false;
822 state->idx = storeWBIdx;
823 state->inst = inst;
824
825 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
826
827 // Build a single data packet if the store isn't split.
828 data_pkt = new Packet(req, command, Packet::Broadcast);
829 data_pkt->dataStatic(inst->memData);
830 data_pkt->senderState = state;
831 } else {
832 // Create two packets if the store is split in two.
833 data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
834 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
835
836 data_pkt->dataStatic(inst->memData);
837 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
838
839 data_pkt->senderState = state;
840 snd_data_pkt->senderState = state;
841
842 state->isSplit = true;
843 state->outstanding = 2;
844
845 // Can delete the main request now.
846 delete req;
847 req = sreqLow;
848 }
849
850 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
851 "to Addr:%#x, data:%#x [sn:%lli]\n",
852 storeWBIdx, inst->pcState(),
853 req->getPaddr(), (int)*(inst->memData),
854 inst->seqNum);
855
856 // @todo: Remove this SC hack once the memory system handles it.
857 if (inst->isStoreConditional()) {
858 assert(!storeQueue[storeWBIdx].isSplit);
859 // Disable recording the result temporarily. Writing to
860 // misc regs normally updates the result, but this is not
861 // the desired behavior when handling store conditionals.
862 inst->recordResult = false;
863 bool success = TheISA::handleLockedWrite(inst.get(), req);
864 inst->recordResult = true;
865
866 if (!success) {
867 // Instantly complete this store.
868 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
869 "Instantly completing it.\n",
870 inst->seqNum);
871 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
872 cpu->schedule(wb, curTick() + 1);
873 completeStore(storeWBIdx);
874 incrStIdx(storeWBIdx);
875 continue;
876 }
877 } else {
878 // Non-store conditionals do not need a writeback.
879 state->noWB = true;
880 }
881
882 bool split =
883 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
884
885 ThreadContext *thread = cpu->tcBase(lsqID);
886
887 if (req->isMmappedIpr()) {
888 assert(!inst->isStoreConditional());
889 TheISA::handleIprWrite(thread, data_pkt);
890 delete data_pkt;
891 if (split) {
892 assert(snd_data_pkt->req->isMmappedIpr());
893 TheISA::handleIprWrite(thread, snd_data_pkt);
894 delete snd_data_pkt;
895 delete sreqLow;
896 delete sreqHigh;
897 }
898 delete state;
899 delete req;
900 completeStore(storeWBIdx);
901 incrStIdx(storeWBIdx);
902 } else if (!sendStore(data_pkt)) {
903 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
904 "retry later\n",
905 inst->seqNum);
906
907 // Need to store the second packet, if split.
908 if (split) {
909 state->pktToSend = true;
910 state->pendingPacket = snd_data_pkt;
911 }
912 } else {
913
914 // If split, try to send the second packet too
915 if (split) {
916 assert(snd_data_pkt);
917
918 // Ensure there are enough ports to use.
919 if (usedPorts < cachePorts) {
920 ++usedPorts;
921 if (sendStore(snd_data_pkt)) {
922 storePostSend(snd_data_pkt);
923 } else {
924 DPRINTF(IEW, "D-Cache became blocked when writing"
925 " [sn:%lli] second packet, will retry later\n",
926 inst->seqNum);
927 }
928 } else {
929
930 // Store the packet for when there's free ports.
931 assert(pendingPkt == NULL);
932 pendingPkt = snd_data_pkt;
933 hasPendingPkt = true;
934 }
935 } else {
936
937 // Not a split store.
938 storePostSend(data_pkt);
939 }
940 }
941 }
942
943 // Not sure this should set it to 0.
944 usedPorts = 0;
945
946 assert(stores >= 0 && storesToWB >= 0);
947}
948
949/*template <class Impl>
950void
951LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
952{
953 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
954 mshrSeqNums.end(),
955 seqNum);
956
957 if (mshr_it != mshrSeqNums.end()) {
958 mshrSeqNums.erase(mshr_it);
959 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
960 }
961}*/
962
963template <class Impl>
964void
965LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
966{
967 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
968 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
969
970 int load_idx = loadTail;
971 decrLdIdx(load_idx);
972
973 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
974 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
975 "[sn:%lli]\n",
976 loadQueue[load_idx]->pcState(),
977 loadQueue[load_idx]->seqNum);
978
979 if (isStalled() && load_idx == stallingLoadIdx) {
980 stalled = false;
981 stallingStoreIsn = 0;
982 stallingLoadIdx = 0;
983 }
984
985 // Clear the smart pointer to make sure it is decremented.
986 loadQueue[load_idx]->setSquashed();
987 loadQueue[load_idx] = NULL;
988 --loads;
989
990 // Inefficient!
991 loadTail = load_idx;
992
993 decrLdIdx(load_idx);
994 ++lsqSquashedLoads;
995 }
996
997 if (isLoadBlocked) {
998 if (squashed_num < blockedLoadSeqNum) {
999 isLoadBlocked = false;
1000 loadBlockedHandled = false;
1001 blockedLoadSeqNum = 0;
1002 }
1003 }
1004
1005 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1006 memDepViolator = NULL;
1007 }
1008
1009 int store_idx = storeTail;
1010 decrStIdx(store_idx);
1011
1012 while (stores != 0 &&
1013 storeQueue[store_idx].inst->seqNum > squashed_num) {
1014 // Instructions marked as can WB are already committed.
1015 if (storeQueue[store_idx].canWB) {
1016 break;
1017 }
1018
1019 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1020 "idx:%i [sn:%lli]\n",
1021 storeQueue[store_idx].inst->pcState(),
1022 store_idx, storeQueue[store_idx].inst->seqNum);
1023
1024 // I don't think this can happen. It should have been cleared
1025 // by the stalling load.
1026 if (isStalled() &&
1027 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1028 panic("Is stalled should have been cleared by stalling load!\n");
1029 stalled = false;
1030 stallingStoreIsn = 0;
1031 }
1032
1033 // Clear the smart pointer to make sure it is decremented.
1034 storeQueue[store_idx].inst->setSquashed();
1035 storeQueue[store_idx].inst = NULL;
1036 storeQueue[store_idx].canWB = 0;
1037
1038 // Must delete request now that it wasn't handed off to
1039 // memory. This is quite ugly. @todo: Figure out the proper
1040 // place to really handle request deletes.
1041 delete storeQueue[store_idx].req;
1042 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1043 delete storeQueue[store_idx].sreqLow;
1044 delete storeQueue[store_idx].sreqHigh;
1045
1046 storeQueue[store_idx].sreqLow = NULL;
1047 storeQueue[store_idx].sreqHigh = NULL;
1048 }
1049
1050 storeQueue[store_idx].req = NULL;
1051 --stores;
1052
1053 // Inefficient!
1054 storeTail = store_idx;
1055
1056 decrStIdx(store_idx);
1057 ++lsqSquashedStores;
1058 }
1059}
1060
1061template <class Impl>
1062void
1063LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1064{
1065 if (isStalled() &&
1066 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1067 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1068 "load idx:%i\n",
1069 stallingStoreIsn, stallingLoadIdx);
1070 stalled = false;
1071 stallingStoreIsn = 0;
1072 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1073 }
1074
1075 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1076 // The store is basically completed at this time. This
1077 // only works so long as the checker doesn't try to
1078 // verify the value in memory for stores.
1079 storeQueue[storeWBIdx].inst->setCompleted();
1080#if USE_CHECKER
1081 if (cpu->checker) {
1082 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1083 }
1084#endif
1085 }
1086
1087 incrStIdx(storeWBIdx);
1088}
1089
1090template <class Impl>
1091void
1092LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1093{
1094 iewStage->wakeCPU();
1095
1096 // Squashed instructions do not need to complete their access.
1097 if (inst->isSquashed()) {
1098 iewStage->decrWb(inst->seqNum);
1099 assert(!inst->isStore());
1100 ++lsqIgnoredResponses;
1101 return;
1102 }
1103
1104 if (!inst->isExecuted()) {
1105 inst->setExecuted();
1106
1107 // Complete access to copy data to proper place.
1108 inst->completeAcc(pkt);
1109 }
1110
1111 // Need to insert instruction into queue to commit
1112 iewStage->instToCommit(inst);
1113
1114 iewStage->activityThisCycle();
1115
1116 // see if this load changed the PC
1117 iewStage->checkMisprediction(inst);
1118}
1119
1120template <class Impl>
1121void
1122LSQUnit<Impl>::completeStore(int store_idx)
1123{
1124 assert(storeQueue[store_idx].inst);
1125 storeQueue[store_idx].completed = true;
1126 --storesToWB;
1127 // A bit conservative because a store completion may not free up entries,
1128 // but hopefully avoids two store completions in one cycle from making
1129 // the CPU tick twice.
1130 cpu->wakeCPU();
1131 cpu->activityThisCycle();
1132
1133 if (store_idx == storeHead) {
1134 do {
1135 incrStIdx(storeHead);
1136
1137 --stores;
1138 } while (storeQueue[storeHead].completed &&
1139 storeHead != storeTail);
1140
1141 iewStage->updateLSQNextCycle = true;
1142 }
1143
1144 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1145 "idx:%i\n",
1146 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1147
1148 if (isStalled() &&
1149 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1150 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1151 "load idx:%i\n",
1152 stallingStoreIsn, stallingLoadIdx);
1153 stalled = false;
1154 stallingStoreIsn = 0;
1155 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1156 }
1157
1158 storeQueue[store_idx].inst->setCompleted();
1159
1160 // Tell the checker we've completed this instruction. Some stores
1161 // may get reported twice to the checker, but the checker can
1162 // handle that case.
1163#if USE_CHECKER
1164 if (cpu->checker) {
1165 cpu->checker->verify(storeQueue[store_idx].inst);
1166 }
1167#endif
1168}
1169
1170template <class Impl>
1171bool
1172LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1173{
1174 if (!dcachePort->sendTiming(data_pkt)) {
1175 // Need to handle becoming blocked on a store.
1176 isStoreBlocked = true;
1177 ++lsqCacheBlocked;
1178 assert(retryPkt == NULL);
1179 retryPkt = data_pkt;
1180 lsq->setRetryTid(lsqID);
1181 return false;
1182 }
1183 return true;
1184}
1185
1186template <class Impl>
1187void
1188LSQUnit<Impl>::recvRetry()
1189{
1190 if (isStoreBlocked) {
1191 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1192 assert(retryPkt != NULL);
1193
1194 LSQSenderState *state =
1195 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1196
1197 if (dcachePort->sendTiming(retryPkt)) {
1198 // Don't finish the store unless this is the last packet.
1199 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1200 state->pendingPacket == retryPkt) {
1201 state->pktToSend = false;
1202 storePostSend(retryPkt);
1203 }
1204 retryPkt = NULL;
1205 isStoreBlocked = false;
1206 lsq->setRetryTid(InvalidThreadID);
1207
1208 // Send any outstanding packet.
1209 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1210 assert(state->pendingPacket);
1211 if (sendStore(state->pendingPacket)) {
1212 storePostSend(state->pendingPacket);
1213 }
1214 }
1215 } else {
1216 // Still blocked!
1217 ++lsqCacheBlocked;
1218 lsq->setRetryTid(lsqID);
1219 }
1220 } else if (isLoadBlocked) {
1221 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1222 "no need to resend packet.\n");
1223 } else {
1224 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1225 }
1226}
1227
1228template <class Impl>
1229inline void
1230LSQUnit<Impl>::incrStIdx(int &store_idx)
1231{
1232 if (++store_idx >= SQEntries)
1233 store_idx = 0;
1234}
1235
1236template <class Impl>
1237inline void
1238LSQUnit<Impl>::decrStIdx(int &store_idx)
1239{
1240 if (--store_idx < 0)
1241 store_idx += SQEntries;
1242}
1243
1244template <class Impl>
1245inline void
1246LSQUnit<Impl>::incrLdIdx(int &load_idx)
1247{
1248 if (++load_idx >= LQEntries)
1249 load_idx = 0;
1250}
1251
1252template <class Impl>
1253inline void
1254LSQUnit<Impl>::decrLdIdx(int &load_idx)
1255{
1256 if (--load_idx < 0)
1257 load_idx += LQEntries;
1258}
1259
1260template <class Impl>
1261void
1262LSQUnit<Impl>::dumpInsts()
1263{
1264 cprintf("Load store queue: Dumping instructions.\n");
1265 cprintf("Load queue size: %i\n", loads);
1266 cprintf("Load queue: ");
1267
1268 int load_idx = loadHead;
1269
1270 while (load_idx != loadTail && loadQueue[load_idx]) {
1271 cprintf("%s ", loadQueue[load_idx]->pcState());
1272
1273 incrLdIdx(load_idx);
1274 }
1275
1276 cprintf("Store queue size: %i\n", stores);
1277 cprintf("Store queue: ");
1278
1279 int store_idx = storeHead;
1280
1281 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1282 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1283
1284 incrStIdx(store_idx);
1285 }
1286
1287 cprintf("\n");
1288}
| 573 }
574 }
575
576 incrLdIdx(load_idx);
577 }
578 return NoFault;
579}
580
581
582
583
584template <class Impl>
585Fault
586LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
587{
588 using namespace TheISA;
589 // Execute a specific load.
590 Fault load_fault = NoFault;
591
592 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
593 inst->pcState(), inst->seqNum);
594
595 assert(!inst->isSquashed());
596
597 load_fault = inst->initiateAcc();
598
599 if (inst->isTranslationDelayed() &&
600 load_fault == NoFault)
601 return load_fault;
602
603 // If the instruction faulted or predicated false, then we need to send it
604 // along to commit without the instruction completing.
605 if (load_fault != NoFault || inst->readPredicate() == false) {
606 // Send this instruction to commit, also make sure iew stage
607 // realizes there is activity.
608 // Mark it as executed unless it is an uncached load that
609 // needs to hit the head of commit.
610 if (inst->readPredicate() == false)
611 inst->forwardOldRegs();
612 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
613 inst->seqNum,
614 (load_fault != NoFault ? "fault" : "predication"));
615 if (!(inst->hasRequest() && inst->uncacheable()) ||
616 inst->isAtCommit()) {
617 inst->setExecuted();
618 }
619 iewStage->instToCommit(inst);
620 iewStage->activityThisCycle();
621 } else if (!loadBlocked()) {
622 assert(inst->effAddrValid);
623 int load_idx = inst->lqIdx;
624 incrLdIdx(load_idx);
625
626 if (checkLoads)
627 return checkViolations(load_idx, inst);
628 }
629
630 return load_fault;
631}
632
633template <class Impl>
634Fault
635LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
636{
637 using namespace TheISA;
638 // Make sure that a store exists.
639 assert(stores != 0);
640
641 int store_idx = store_inst->sqIdx;
642
643 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
644 store_inst->pcState(), store_inst->seqNum);
645
646 assert(!store_inst->isSquashed());
647
648 // Check the recently completed loads to see if any match this store's
649 // address. If so, then we have a memory ordering violation.
650 int load_idx = store_inst->lqIdx;
651
652 Fault store_fault = store_inst->initiateAcc();
653
654 if (store_inst->isTranslationDelayed() &&
655 store_fault == NoFault)
656 return store_fault;
657
658 if (store_inst->readPredicate() == false)
659 store_inst->forwardOldRegs();
660
661 if (storeQueue[store_idx].size == 0) {
662 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
663 store_inst->pcState(), store_inst->seqNum);
664
665 return store_fault;
666 } else if (store_inst->readPredicate() == false) {
667 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
668 store_inst->seqNum);
669 return store_fault;
670 }
671
672 assert(store_fault == NoFault);
673
674 if (store_inst->isStoreConditional()) {
675 // Store conditionals need to set themselves as able to
676 // writeback if we haven't had a fault by here.
677 storeQueue[store_idx].canWB = true;
678
679 ++storesToWB;
680 }
681
682 return checkViolations(load_idx, store_inst);
683
684}
685
686template <class Impl>
687void
688LSQUnit<Impl>::commitLoad()
689{
690 assert(loadQueue[loadHead]);
691
692 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
693 loadQueue[loadHead]->pcState());
694
695 loadQueue[loadHead] = NULL;
696
697 incrLdIdx(loadHead);
698
699 --loads;
700}
701
702template <class Impl>
703void
704LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
705{
706 assert(loads == 0 || loadQueue[loadHead]);
707
708 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
709 commitLoad();
710 }
711}
712
713template <class Impl>
714void
715LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
716{
717 assert(stores == 0 || storeQueue[storeHead].inst);
718
719 int store_idx = storeHead;
720
721 while (store_idx != storeTail) {
722 assert(storeQueue[store_idx].inst);
723 // Mark any stores that are now committed and have not yet
724 // been marked as able to write back.
725 if (!storeQueue[store_idx].canWB) {
726 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
727 break;
728 }
729 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
730 "%s [sn:%lli]\n",
731 storeQueue[store_idx].inst->pcState(),
732 storeQueue[store_idx].inst->seqNum);
733
734 storeQueue[store_idx].canWB = true;
735
736 ++storesToWB;
737 }
738
739 incrStIdx(store_idx);
740 }
741}
742
743template <class Impl>
744void
745LSQUnit<Impl>::writebackPendingStore()
746{
747 if (hasPendingPkt) {
748 assert(pendingPkt != NULL);
749
750 // If the cache is blocked, this will store the packet for retry.
751 if (sendStore(pendingPkt)) {
752 storePostSend(pendingPkt);
753 }
754 pendingPkt = NULL;
755 hasPendingPkt = false;
756 }
757}
758
759template <class Impl>
760void
761LSQUnit<Impl>::writebackStores()
762{
763 // First writeback the second packet from any split store that didn't
764 // complete last cycle because there weren't enough cache ports available.
765 if (TheISA::HasUnalignedMemAcc) {
766 writebackPendingStore();
767 }
768
769 while (storesToWB > 0 &&
770 storeWBIdx != storeTail &&
771 storeQueue[storeWBIdx].inst &&
772 storeQueue[storeWBIdx].canWB &&
773 usedPorts < cachePorts) {
774
775 if (isStoreBlocked || lsq->cacheBlocked()) {
776 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
777 " is blocked!\n");
778 break;
779 }
780
781 // Store didn't write any data so no need to write it back to
782 // memory.
783 if (storeQueue[storeWBIdx].size == 0) {
784 completeStore(storeWBIdx);
785
786 incrStIdx(storeWBIdx);
787
788 continue;
789 }
790
791 ++usedPorts;
792
793 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
794 incrStIdx(storeWBIdx);
795
796 continue;
797 }
798
799 assert(storeQueue[storeWBIdx].req);
800 assert(!storeQueue[storeWBIdx].committed);
801
802 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
803 assert(storeQueue[storeWBIdx].sreqLow);
804 assert(storeQueue[storeWBIdx].sreqHigh);
805 }
806
807 DynInstPtr inst = storeQueue[storeWBIdx].inst;
808
809 Request *req = storeQueue[storeWBIdx].req;
810 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
811 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
812
813 storeQueue[storeWBIdx].committed = true;
814
815 assert(!inst->memData);
816 inst->memData = new uint8_t[64];
817
818 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
819
820 MemCmd command =
821 req->isSwap() ? MemCmd::SwapReq :
822 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
823 PacketPtr data_pkt;
824 PacketPtr snd_data_pkt = NULL;
825
826 LSQSenderState *state = new LSQSenderState;
827 state->isLoad = false;
828 state->idx = storeWBIdx;
829 state->inst = inst;
830
831 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
832
833 // Build a single data packet if the store isn't split.
834 data_pkt = new Packet(req, command, Packet::Broadcast);
835 data_pkt->dataStatic(inst->memData);
836 data_pkt->senderState = state;
837 } else {
838 // Create two packets if the store is split in two.
839 data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
840 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
841
842 data_pkt->dataStatic(inst->memData);
843 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
844
845 data_pkt->senderState = state;
846 snd_data_pkt->senderState = state;
847
848 state->isSplit = true;
849 state->outstanding = 2;
850
851 // Can delete the main request now.
852 delete req;
853 req = sreqLow;
854 }
855
856 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
857 "to Addr:%#x, data:%#x [sn:%lli]\n",
858 storeWBIdx, inst->pcState(),
859 req->getPaddr(), (int)*(inst->memData),
860 inst->seqNum);
861
862 // @todo: Remove this SC hack once the memory system handles it.
863 if (inst->isStoreConditional()) {
864 assert(!storeQueue[storeWBIdx].isSplit);
865 // Disable recording the result temporarily. Writing to
866 // misc regs normally updates the result, but this is not
867 // the desired behavior when handling store conditionals.
868 inst->recordResult = false;
869 bool success = TheISA::handleLockedWrite(inst.get(), req);
870 inst->recordResult = true;
871
872 if (!success) {
873 // Instantly complete this store.
874 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
875 "Instantly completing it.\n",
876 inst->seqNum);
877 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
878 cpu->schedule(wb, curTick() + 1);
879 completeStore(storeWBIdx);
880 incrStIdx(storeWBIdx);
881 continue;
882 }
883 } else {
884 // Non-store conditionals do not need a writeback.
885 state->noWB = true;
886 }
887
888 bool split =
889 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
890
891 ThreadContext *thread = cpu->tcBase(lsqID);
892
893 if (req->isMmappedIpr()) {
894 assert(!inst->isStoreConditional());
895 TheISA::handleIprWrite(thread, data_pkt);
896 delete data_pkt;
897 if (split) {
898 assert(snd_data_pkt->req->isMmappedIpr());
899 TheISA::handleIprWrite(thread, snd_data_pkt);
900 delete snd_data_pkt;
901 delete sreqLow;
902 delete sreqHigh;
903 }
904 delete state;
905 delete req;
906 completeStore(storeWBIdx);
907 incrStIdx(storeWBIdx);
908 } else if (!sendStore(data_pkt)) {
909 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
910 "retry later\n",
911 inst->seqNum);
912
913 // Need to store the second packet, if split.
914 if (split) {
915 state->pktToSend = true;
916 state->pendingPacket = snd_data_pkt;
917 }
918 } else {
919
920 // If split, try to send the second packet too
921 if (split) {
922 assert(snd_data_pkt);
923
924 // Ensure there are enough ports to use.
925 if (usedPorts < cachePorts) {
926 ++usedPorts;
927 if (sendStore(snd_data_pkt)) {
928 storePostSend(snd_data_pkt);
929 } else {
930 DPRINTF(IEW, "D-Cache became blocked when writing"
931 " [sn:%lli] second packet, will retry later\n",
932 inst->seqNum);
933 }
934 } else {
935
936 // Store the packet for when there's free ports.
937 assert(pendingPkt == NULL);
938 pendingPkt = snd_data_pkt;
939 hasPendingPkt = true;
940 }
941 } else {
942
943 // Not a split store.
944 storePostSend(data_pkt);
945 }
946 }
947 }
948
949 // Not sure this should set it to 0.
950 usedPorts = 0;
951
952 assert(stores >= 0 && storesToWB >= 0);
953}
954
955/*template <class Impl>
956void
957LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
958{
959 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
960 mshrSeqNums.end(),
961 seqNum);
962
963 if (mshr_it != mshrSeqNums.end()) {
964 mshrSeqNums.erase(mshr_it);
965 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
966 }
967}*/
968
969template <class Impl>
970void
971LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
972{
973 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
974 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
975
976 int load_idx = loadTail;
977 decrLdIdx(load_idx);
978
979 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
980 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
981 "[sn:%lli]\n",
982 loadQueue[load_idx]->pcState(),
983 loadQueue[load_idx]->seqNum);
984
985 if (isStalled() && load_idx == stallingLoadIdx) {
986 stalled = false;
987 stallingStoreIsn = 0;
988 stallingLoadIdx = 0;
989 }
990
991 // Clear the smart pointer to make sure it is decremented.
992 loadQueue[load_idx]->setSquashed();
993 loadQueue[load_idx] = NULL;
994 --loads;
995
996 // Inefficient!
997 loadTail = load_idx;
998
999 decrLdIdx(load_idx);
1000 ++lsqSquashedLoads;
1001 }
1002
1003 if (isLoadBlocked) {
1004 if (squashed_num < blockedLoadSeqNum) {
1005 isLoadBlocked = false;
1006 loadBlockedHandled = false;
1007 blockedLoadSeqNum = 0;
1008 }
1009 }
1010
1011 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1012 memDepViolator = NULL;
1013 }
1014
1015 int store_idx = storeTail;
1016 decrStIdx(store_idx);
1017
1018 while (stores != 0 &&
1019 storeQueue[store_idx].inst->seqNum > squashed_num) {
1020 // Instructions marked as can WB are already committed.
1021 if (storeQueue[store_idx].canWB) {
1022 break;
1023 }
1024
1025 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1026 "idx:%i [sn:%lli]\n",
1027 storeQueue[store_idx].inst->pcState(),
1028 store_idx, storeQueue[store_idx].inst->seqNum);
1029
1030 // I don't think this can happen. It should have been cleared
1031 // by the stalling load.
1032 if (isStalled() &&
1033 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1034 panic("Is stalled should have been cleared by stalling load!\n");
1035 stalled = false;
1036 stallingStoreIsn = 0;
1037 }
1038
1039 // Clear the smart pointer to make sure it is decremented.
1040 storeQueue[store_idx].inst->setSquashed();
1041 storeQueue[store_idx].inst = NULL;
1042 storeQueue[store_idx].canWB = 0;
1043
1044 // Must delete request now that it wasn't handed off to
1045 // memory. This is quite ugly. @todo: Figure out the proper
1046 // place to really handle request deletes.
1047 delete storeQueue[store_idx].req;
1048 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1049 delete storeQueue[store_idx].sreqLow;
1050 delete storeQueue[store_idx].sreqHigh;
1051
1052 storeQueue[store_idx].sreqLow = NULL;
1053 storeQueue[store_idx].sreqHigh = NULL;
1054 }
1055
1056 storeQueue[store_idx].req = NULL;
1057 --stores;
1058
1059 // Inefficient!
1060 storeTail = store_idx;
1061
1062 decrStIdx(store_idx);
1063 ++lsqSquashedStores;
1064 }
1065}
1066
1067template <class Impl>
1068void
1069LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1070{
1071 if (isStalled() &&
1072 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1073 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1074 "load idx:%i\n",
1075 stallingStoreIsn, stallingLoadIdx);
1076 stalled = false;
1077 stallingStoreIsn = 0;
1078 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1079 }
1080
1081 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1082 // The store is basically completed at this time. This
1083 // only works so long as the checker doesn't try to
1084 // verify the value in memory for stores.
1085 storeQueue[storeWBIdx].inst->setCompleted();
1086#if USE_CHECKER
1087 if (cpu->checker) {
1088 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1089 }
1090#endif
1091 }
1092
1093 incrStIdx(storeWBIdx);
1094}
1095
1096template <class Impl>
1097void
1098LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1099{
1100 iewStage->wakeCPU();
1101
1102 // Squashed instructions do not need to complete their access.
1103 if (inst->isSquashed()) {
1104 iewStage->decrWb(inst->seqNum);
1105 assert(!inst->isStore());
1106 ++lsqIgnoredResponses;
1107 return;
1108 }
1109
1110 if (!inst->isExecuted()) {
1111 inst->setExecuted();
1112
1113 // Complete access to copy data to proper place.
1114 inst->completeAcc(pkt);
1115 }
1116
1117 // Need to insert instruction into queue to commit
1118 iewStage->instToCommit(inst);
1119
1120 iewStage->activityThisCycle();
1121
1122 // see if this load changed the PC
1123 iewStage->checkMisprediction(inst);
1124}
1125
1126template <class Impl>
1127void
1128LSQUnit<Impl>::completeStore(int store_idx)
1129{
1130 assert(storeQueue[store_idx].inst);
1131 storeQueue[store_idx].completed = true;
1132 --storesToWB;
1133 // A bit conservative because a store completion may not free up entries,
1134 // but hopefully avoids two store completions in one cycle from making
1135 // the CPU tick twice.
1136 cpu->wakeCPU();
1137 cpu->activityThisCycle();
1138
1139 if (store_idx == storeHead) {
1140 do {
1141 incrStIdx(storeHead);
1142
1143 --stores;
1144 } while (storeQueue[storeHead].completed &&
1145 storeHead != storeTail);
1146
1147 iewStage->updateLSQNextCycle = true;
1148 }
1149
1150 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1151 "idx:%i\n",
1152 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1153
1154 if (isStalled() &&
1155 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1156 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1157 "load idx:%i\n",
1158 stallingStoreIsn, stallingLoadIdx);
1159 stalled = false;
1160 stallingStoreIsn = 0;
1161 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1162 }
1163
1164 storeQueue[store_idx].inst->setCompleted();
1165
1166 // Tell the checker we've completed this instruction. Some stores
1167 // may get reported twice to the checker, but the checker can
1168 // handle that case.
1169#if USE_CHECKER
1170 if (cpu->checker) {
1171 cpu->checker->verify(storeQueue[store_idx].inst);
1172 }
1173#endif
1174}
1175
1176template <class Impl>
1177bool
1178LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1179{
1180 if (!dcachePort->sendTiming(data_pkt)) {
1181 // Need to handle becoming blocked on a store.
1182 isStoreBlocked = true;
1183 ++lsqCacheBlocked;
1184 assert(retryPkt == NULL);
1185 retryPkt = data_pkt;
1186 lsq->setRetryTid(lsqID);
1187 return false;
1188 }
1189 return true;
1190}
1191
1192template <class Impl>
1193void
1194LSQUnit<Impl>::recvRetry()
1195{
1196 if (isStoreBlocked) {
1197 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1198 assert(retryPkt != NULL);
1199
1200 LSQSenderState *state =
1201 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1202
1203 if (dcachePort->sendTiming(retryPkt)) {
1204 // Don't finish the store unless this is the last packet.
1205 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1206 state->pendingPacket == retryPkt) {
1207 state->pktToSend = false;
1208 storePostSend(retryPkt);
1209 }
1210 retryPkt = NULL;
1211 isStoreBlocked = false;
1212 lsq->setRetryTid(InvalidThreadID);
1213
1214 // Send any outstanding packet.
1215 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1216 assert(state->pendingPacket);
1217 if (sendStore(state->pendingPacket)) {
1218 storePostSend(state->pendingPacket);
1219 }
1220 }
1221 } else {
1222 // Still blocked!
1223 ++lsqCacheBlocked;
1224 lsq->setRetryTid(lsqID);
1225 }
1226 } else if (isLoadBlocked) {
1227 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1228 "no need to resend packet.\n");
1229 } else {
1230 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1231 }
1232}
1233
1234template <class Impl>
1235inline void
1236LSQUnit<Impl>::incrStIdx(int &store_idx)
1237{
1238 if (++store_idx >= SQEntries)
1239 store_idx = 0;
1240}
1241
1242template <class Impl>
1243inline void
1244LSQUnit<Impl>::decrStIdx(int &store_idx)
1245{
1246 if (--store_idx < 0)
1247 store_idx += SQEntries;
1248}
1249
1250template <class Impl>
1251inline void
1252LSQUnit<Impl>::incrLdIdx(int &load_idx)
1253{
1254 if (++load_idx >= LQEntries)
1255 load_idx = 0;
1256}
1257
1258template <class Impl>
1259inline void
1260LSQUnit<Impl>::decrLdIdx(int &load_idx)
1261{
1262 if (--load_idx < 0)
1263 load_idx += LQEntries;
1264}
1265
1266template <class Impl>
1267void
1268LSQUnit<Impl>::dumpInsts()
1269{
1270 cprintf("Load store queue: Dumping instructions.\n");
1271 cprintf("Load queue size: %i\n", loads);
1272 cprintf("Load queue: ");
1273
1274 int load_idx = loadHead;
1275
1276 while (load_idx != loadTail && loadQueue[load_idx]) {
1277 cprintf("%s ", loadQueue[load_idx]->pcState());
1278
1279 incrLdIdx(load_idx);
1280 }
1281
1282 cprintf("Store queue size: %i\n", stores);
1283 cprintf("Store queue: ");
1284
1285 int store_idx = storeHead;
1286
1287 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1288 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1289
1290 incrStIdx(store_idx);
1291 }
1292
1293 cprintf("\n");
1294}
|