lsq_unit_impl.hh (8592:30a97c4198df) lsq_unit_impl.hh (8727:b3995530319f)
1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 * Korey Sewell
42 */
43
44#include "arch/generic/debugfaults.hh"
45#include "arch/locked_mem.hh"
46#include "base/str.hh"
47#include "config/the_isa.hh"
48#include "config/use_checker.hh"
49#include "cpu/o3/lsq.hh"
50#include "cpu/o3/lsq_unit.hh"
51#include "debug/Activity.hh"
52#include "debug/IEW.hh"
53#include "debug/LSQUnit.hh"
54#include "mem/packet.hh"
55#include "mem/request.hh"
56
57#if USE_CHECKER
58#include "cpu/checker/cpu.hh"
59#endif
60
61template<class Impl>
62LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
63 LSQUnit *lsq_ptr)
64 : Event(Default_Pri, AutoDelete),
65 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
66{
67}
68
69template<class Impl>
70void
71LSQUnit<Impl>::WritebackEvent::process()
72{
73 if (!lsqPtr->isSwitchedOut()) {
74 lsqPtr->writeback(inst, pkt);
75 }
76
77 if (pkt->senderState)
78 delete pkt->senderState;
79
80 delete pkt->req;
81 delete pkt;
82}
83
84template<class Impl>
85const char *
86LSQUnit<Impl>::WritebackEvent::description() const
87{
88 return "Store writeback";
89}
90
91template<class Impl>
92void
93LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
94{
95 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
96 DynInstPtr inst = state->inst;
97 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
98 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
99
100 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
101
102 assert(!pkt->wasNacked());
103
104 // If this is a split access, wait until all packets are received.
105 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
106 delete pkt->req;
107 delete pkt;
108 return;
109 }
110
111 if (isSwitchedOut() || inst->isSquashed()) {
112 iewStage->decrWb(inst->seqNum);
113 } else {
114 if (!state->noWB) {
115 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
116 !state->isLoad) {
117 writeback(inst, pkt);
118 } else {
119 writeback(inst, state->mainPkt);
120 }
121 }
122
123 if (inst->isStore()) {
124 completeStore(state->idx);
125 }
126 }
127
128 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
129 delete state->mainPkt->req;
130 delete state->mainPkt;
131 }
132 delete state;
133 delete pkt->req;
134 delete pkt;
135}
136
137template <class Impl>
138LSQUnit<Impl>::LSQUnit()
139 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
140 isStoreBlocked(false), isLoadBlocked(false),
1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 * Korey Sewell
42 */
43
44#include "arch/generic/debugfaults.hh"
45#include "arch/locked_mem.hh"
46#include "base/str.hh"
47#include "config/the_isa.hh"
48#include "config/use_checker.hh"
49#include "cpu/o3/lsq.hh"
50#include "cpu/o3/lsq_unit.hh"
51#include "debug/Activity.hh"
52#include "debug/IEW.hh"
53#include "debug/LSQUnit.hh"
54#include "mem/packet.hh"
55#include "mem/request.hh"
56
57#if USE_CHECKER
58#include "cpu/checker/cpu.hh"
59#endif
60
61template<class Impl>
62LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
63 LSQUnit *lsq_ptr)
64 : Event(Default_Pri, AutoDelete),
65 inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
66{
67}
68
69template<class Impl>
70void
71LSQUnit<Impl>::WritebackEvent::process()
72{
73 if (!lsqPtr->isSwitchedOut()) {
74 lsqPtr->writeback(inst, pkt);
75 }
76
77 if (pkt->senderState)
78 delete pkt->senderState;
79
80 delete pkt->req;
81 delete pkt;
82}
83
84template<class Impl>
85const char *
86LSQUnit<Impl>::WritebackEvent::description() const
87{
88 return "Store writeback";
89}
90
91template<class Impl>
92void
93LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
94{
95 LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
96 DynInstPtr inst = state->inst;
97 DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
98 DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
99
100 //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
101
102 assert(!pkt->wasNacked());
103
104 // If this is a split access, wait until all packets are received.
105 if (TheISA::HasUnalignedMemAcc && !state->complete()) {
106 delete pkt->req;
107 delete pkt;
108 return;
109 }
110
111 if (isSwitchedOut() || inst->isSquashed()) {
112 iewStage->decrWb(inst->seqNum);
113 } else {
114 if (!state->noWB) {
115 if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
116 !state->isLoad) {
117 writeback(inst, pkt);
118 } else {
119 writeback(inst, state->mainPkt);
120 }
121 }
122
123 if (inst->isStore()) {
124 completeStore(state->idx);
125 }
126 }
127
128 if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
129 delete state->mainPkt->req;
130 delete state->mainPkt;
131 }
132 delete state;
133 delete pkt->req;
134 delete pkt;
135}
136
137template <class Impl>
138LSQUnit<Impl>::LSQUnit()
139 : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
140 isStoreBlocked(false), isLoadBlocked(false),
141 loadBlockedHandled(false), hasPendingPkt(false)
141 loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false)
142{
143}
144
145template<class Impl>
146void
147LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
148 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
149 unsigned id)
150{
151 cpu = cpu_ptr;
152 iewStage = iew_ptr;
153
154 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
155
156 switchedOut = false;
157
158 cacheBlockMask = 0;
159
160 lsq = lsq_ptr;
161
162 lsqID = id;
163
164 // Add 1 for the sentinel entry (they are circular queues).
165 LQEntries = maxLQEntries + 1;
166 SQEntries = maxSQEntries + 1;
167
168 loadQueue.resize(LQEntries);
169 storeQueue.resize(SQEntries);
170
171 depCheckShift = params->LSQDepCheckShift;
172 checkLoads = params->LSQCheckLoads;
173
174 loadHead = loadTail = 0;
175
176 storeHead = storeWBIdx = storeTail = 0;
177
178 usedPorts = 0;
179 cachePorts = params->cachePorts;
180
181 retryPkt = NULL;
182 memDepViolator = NULL;
183
184 blockedLoadSeqNum = 0;
142{
143}
144
145template<class Impl>
146void
147LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
148 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
149 unsigned id)
150{
151 cpu = cpu_ptr;
152 iewStage = iew_ptr;
153
154 DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
155
156 switchedOut = false;
157
158 cacheBlockMask = 0;
159
160 lsq = lsq_ptr;
161
162 lsqID = id;
163
164 // Add 1 for the sentinel entry (they are circular queues).
165 LQEntries = maxLQEntries + 1;
166 SQEntries = maxSQEntries + 1;
167
168 loadQueue.resize(LQEntries);
169 storeQueue.resize(SQEntries);
170
171 depCheckShift = params->LSQDepCheckShift;
172 checkLoads = params->LSQCheckLoads;
173
174 loadHead = loadTail = 0;
175
176 storeHead = storeWBIdx = storeTail = 0;
177
178 usedPorts = 0;
179 cachePorts = params->cachePorts;
180
181 retryPkt = NULL;
182 memDepViolator = NULL;
183
184 blockedLoadSeqNum = 0;
185 needsTSO = params->needsTSO;
185}
186
187template<class Impl>
188std::string
189LSQUnit<Impl>::name() const
190{
191 if (Impl::MaxThreads == 1) {
192 return iewStage->name() + ".lsq";
193 } else {
194 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
195 }
196}
197
198template<class Impl>
199void
200LSQUnit<Impl>::regStats()
201{
202 lsqForwLoads
203 .name(name() + ".forwLoads")
204 .desc("Number of loads that had data forwarded from stores");
205
206 invAddrLoads
207 .name(name() + ".invAddrLoads")
208 .desc("Number of loads ignored due to an invalid address");
209
210 lsqSquashedLoads
211 .name(name() + ".squashedLoads")
212 .desc("Number of loads squashed");
213
214 lsqIgnoredResponses
215 .name(name() + ".ignoredResponses")
216 .desc("Number of memory responses ignored because the instruction is squashed");
217
218 lsqMemOrderViolation
219 .name(name() + ".memOrderViolation")
220 .desc("Number of memory ordering violations");
221
222 lsqSquashedStores
223 .name(name() + ".squashedStores")
224 .desc("Number of stores squashed");
225
226 invAddrSwpfs
227 .name(name() + ".invAddrSwpfs")
228 .desc("Number of software prefetches ignored due to an invalid address");
229
230 lsqBlockedLoads
231 .name(name() + ".blockedLoads")
232 .desc("Number of blocked loads due to partial load-store forwarding");
233
234 lsqRescheduledLoads
235 .name(name() + ".rescheduledLoads")
236 .desc("Number of loads that were rescheduled");
237
238 lsqCacheBlocked
239 .name(name() + ".cacheBlocked")
240 .desc("Number of times an access to memory failed due to the cache being blocked");
241}
242
243template<class Impl>
244void
245LSQUnit<Impl>::setDcachePort(Port *dcache_port)
246{
247 dcachePort = dcache_port;
248
249#if USE_CHECKER
250 if (cpu->checker) {
251 cpu->checker->setDcachePort(dcachePort);
252 }
253#endif
254}
255
256template<class Impl>
257void
258LSQUnit<Impl>::clearLQ()
259{
260 loadQueue.clear();
261}
262
263template<class Impl>
264void
265LSQUnit<Impl>::clearSQ()
266{
267 storeQueue.clear();
268}
269
270template<class Impl>
271void
272LSQUnit<Impl>::switchOut()
273{
274 switchedOut = true;
275 for (int i = 0; i < loadQueue.size(); ++i) {
276 assert(!loadQueue[i]);
277 loadQueue[i] = NULL;
278 }
279
280 assert(storesToWB == 0);
281}
282
283template<class Impl>
284void
285LSQUnit<Impl>::takeOverFrom()
286{
287 switchedOut = false;
288 loads = stores = storesToWB = 0;
289
290 loadHead = loadTail = 0;
291
292 storeHead = storeWBIdx = storeTail = 0;
293
294 usedPorts = 0;
295
296 memDepViolator = NULL;
297
298 blockedLoadSeqNum = 0;
299
300 stalled = false;
301 isLoadBlocked = false;
302 loadBlockedHandled = false;
303
304 // Just incase the memory system changed out from under us
305 cacheBlockMask = 0;
306}
307
308template<class Impl>
309void
310LSQUnit<Impl>::resizeLQ(unsigned size)
311{
312 unsigned size_plus_sentinel = size + 1;
313 assert(size_plus_sentinel >= LQEntries);
314
315 if (size_plus_sentinel > LQEntries) {
316 while (size_plus_sentinel > loadQueue.size()) {
317 DynInstPtr dummy;
318 loadQueue.push_back(dummy);
319 LQEntries++;
320 }
321 } else {
322 LQEntries = size_plus_sentinel;
323 }
324
325}
326
327template<class Impl>
328void
329LSQUnit<Impl>::resizeSQ(unsigned size)
330{
331 unsigned size_plus_sentinel = size + 1;
332 if (size_plus_sentinel > SQEntries) {
333 while (size_plus_sentinel > storeQueue.size()) {
334 SQEntry dummy;
335 storeQueue.push_back(dummy);
336 SQEntries++;
337 }
338 } else {
339 SQEntries = size_plus_sentinel;
340 }
341}
342
343template <class Impl>
344void
345LSQUnit<Impl>::insert(DynInstPtr &inst)
346{
347 assert(inst->isMemRef());
348
349 assert(inst->isLoad() || inst->isStore());
350
351 if (inst->isLoad()) {
352 insertLoad(inst);
353 } else {
354 insertStore(inst);
355 }
356
357 inst->setInLSQ();
358}
359
360template <class Impl>
361void
362LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
363{
364 assert((loadTail + 1) % LQEntries != loadHead);
365 assert(loads < LQEntries);
366
367 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
368 load_inst->pcState(), loadTail, load_inst->seqNum);
369
370 load_inst->lqIdx = loadTail;
371
372 if (stores == 0) {
373 load_inst->sqIdx = -1;
374 } else {
375 load_inst->sqIdx = storeTail;
376 }
377
378 loadQueue[loadTail] = load_inst;
379
380 incrLdIdx(loadTail);
381
382 ++loads;
383}
384
385template <class Impl>
386void
387LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
388{
389 // Make sure it is not full before inserting an instruction.
390 assert((storeTail + 1) % SQEntries != storeHead);
391 assert(stores < SQEntries);
392
393 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
394 store_inst->pcState(), storeTail, store_inst->seqNum);
395
396 store_inst->sqIdx = storeTail;
397 store_inst->lqIdx = loadTail;
398
399 storeQueue[storeTail] = SQEntry(store_inst);
400
401 incrStIdx(storeTail);
402
403 ++stores;
404}
405
406template <class Impl>
407typename Impl::DynInstPtr
408LSQUnit<Impl>::getMemDepViolator()
409{
410 DynInstPtr temp = memDepViolator;
411
412 memDepViolator = NULL;
413
414 return temp;
415}
416
417template <class Impl>
418unsigned
419LSQUnit<Impl>::numFreeEntries()
420{
421 unsigned free_lq_entries = LQEntries - loads;
422 unsigned free_sq_entries = SQEntries - stores;
423
424 // Both the LQ and SQ entries have an extra dummy entry to differentiate
425 // empty/full conditions. Subtract 1 from the free entries.
426 if (free_lq_entries < free_sq_entries) {
427 return free_lq_entries - 1;
428 } else {
429 return free_sq_entries - 1;
430 }
431}
432
433template <class Impl>
434int
435LSQUnit<Impl>::numLoadsReady()
436{
437 int load_idx = loadHead;
438 int retval = 0;
439
440 while (load_idx != loadTail) {
441 assert(loadQueue[load_idx]);
442
443 if (loadQueue[load_idx]->readyToIssue()) {
444 ++retval;
445 }
446 }
447
448 return retval;
449}
450
451template <class Impl>
452void
453LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
454{
455 int load_idx = loadHead;
456
457 if (!cacheBlockMask) {
458 assert(dcachePort);
459 Addr bs = dcachePort->peerBlockSize();
460
461 // Make sure we actually got a size
462 assert(bs != 0);
463
464 cacheBlockMask = ~(bs - 1);
465 }
466
467 // If this is the only load in the LSQ we don't care
468 if (load_idx == loadTail)
469 return;
470 incrLdIdx(load_idx);
471
472 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
473 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
474 while (load_idx != loadTail) {
475 DynInstPtr ld_inst = loadQueue[load_idx];
476
477 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
478 incrLdIdx(load_idx);
479 continue;
480 }
481
482 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
483 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
484 ld_inst->seqNum, load_addr, invalidate_addr);
485
486 if (load_addr == invalidate_addr) {
487 if (ld_inst->possibleLoadViolation) {
488 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
489 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
490
491 // Mark the load for re-execution
492 ld_inst->fault = new ReExec;
493 } else {
494 // If a older load checks this and it's true
495 // then we might have missed the snoop
496 // in which case we need to invalidate to be sure
497 ld_inst->hitExternalSnoop = true;
498 }
499 }
500 incrLdIdx(load_idx);
501 }
502 return;
503}
504
505template <class Impl>
506Fault
507LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
508{
509 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
510 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
511
512 /** @todo in theory you only need to check an instruction that has executed
513 * however, there isn't a good way in the pipeline at the moment to check
514 * all instructions that will execute before the store writes back. Thus,
515 * like the implementation that came before it, we're overly conservative.
516 */
517 while (load_idx != loadTail) {
518 DynInstPtr ld_inst = loadQueue[load_idx];
519 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
520 incrLdIdx(load_idx);
521 continue;
522 }
523
524 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
525 Addr ld_eff_addr2 =
526 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
527
528 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
529 if (inst->isLoad()) {
530 // If this load is to the same block as an external snoop
531 // invalidate that we've observed then the load needs to be
532 // squashed as it could have newer data
533 if (ld_inst->hitExternalSnoop) {
534 if (!memDepViolator ||
535 ld_inst->seqNum < memDepViolator->seqNum) {
536 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
537 "and [sn:%lli] at address %#x\n",
538 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
539 memDepViolator = ld_inst;
540
541 ++lsqMemOrderViolation;
542
543 return new GenericISA::M5PanicFault(
544 "Detected fault with inst [sn:%lli] and "
545 "[sn:%lli] at address %#x\n",
546 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
547 }
548 }
549
550 // Otherwise, mark the load has a possible load violation
551 // and if we see a snoop before it's commited, we need to squash
552 ld_inst->possibleLoadViolation = true;
553 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
554 " between instructions [sn:%lli] and [sn:%lli]\n",
555 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
556 } else {
557 // A load/store incorrectly passed this store.
558 // Check if we already have a violator, or if it's newer
559 // squash and refetch.
560 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
561 break;
562
563 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
564 "[sn:%lli] at address %#x\n",
565 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
566 memDepViolator = ld_inst;
567
568 ++lsqMemOrderViolation;
569
570 return new GenericISA::M5PanicFault("Detected fault with "
571 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
572 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
573 }
574 }
575
576 incrLdIdx(load_idx);
577 }
578 return NoFault;
579}
580
581
582
583
584template <class Impl>
585Fault
586LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
587{
588 using namespace TheISA;
589 // Execute a specific load.
590 Fault load_fault = NoFault;
591
592 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
593 inst->pcState(), inst->seqNum);
594
595 assert(!inst->isSquashed());
596
597 load_fault = inst->initiateAcc();
598
599 if (inst->isTranslationDelayed() &&
600 load_fault == NoFault)
601 return load_fault;
602
603 // If the instruction faulted or predicated false, then we need to send it
604 // along to commit without the instruction completing.
605 if (load_fault != NoFault || inst->readPredicate() == false) {
606 // Send this instruction to commit, also make sure iew stage
607 // realizes there is activity.
608 // Mark it as executed unless it is an uncached load that
609 // needs to hit the head of commit.
610 if (inst->readPredicate() == false)
611 inst->forwardOldRegs();
612 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
613 inst->seqNum,
614 (load_fault != NoFault ? "fault" : "predication"));
615 if (!(inst->hasRequest() && inst->uncacheable()) ||
616 inst->isAtCommit()) {
617 inst->setExecuted();
618 }
619 iewStage->instToCommit(inst);
620 iewStage->activityThisCycle();
621 } else if (!loadBlocked()) {
622 assert(inst->effAddrValid);
623 int load_idx = inst->lqIdx;
624 incrLdIdx(load_idx);
625
626 if (checkLoads)
627 return checkViolations(load_idx, inst);
628 }
629
630 return load_fault;
631}
632
633template <class Impl>
634Fault
635LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
636{
637 using namespace TheISA;
638 // Make sure that a store exists.
639 assert(stores != 0);
640
641 int store_idx = store_inst->sqIdx;
642
643 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
644 store_inst->pcState(), store_inst->seqNum);
645
646 assert(!store_inst->isSquashed());
647
648 // Check the recently completed loads to see if any match this store's
649 // address. If so, then we have a memory ordering violation.
650 int load_idx = store_inst->lqIdx;
651
652 Fault store_fault = store_inst->initiateAcc();
653
654 if (store_inst->isTranslationDelayed() &&
655 store_fault == NoFault)
656 return store_fault;
657
658 if (store_inst->readPredicate() == false)
659 store_inst->forwardOldRegs();
660
661 if (storeQueue[store_idx].size == 0) {
662 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
663 store_inst->pcState(), store_inst->seqNum);
664
665 return store_fault;
666 } else if (store_inst->readPredicate() == false) {
667 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
668 store_inst->seqNum);
669 return store_fault;
670 }
671
672 assert(store_fault == NoFault);
673
674 if (store_inst->isStoreConditional()) {
675 // Store conditionals need to set themselves as able to
676 // writeback if we haven't had a fault by here.
677 storeQueue[store_idx].canWB = true;
678
679 ++storesToWB;
680 }
681
682 return checkViolations(load_idx, store_inst);
683
684}
685
686template <class Impl>
687void
688LSQUnit<Impl>::commitLoad()
689{
690 assert(loadQueue[loadHead]);
691
692 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
693 loadQueue[loadHead]->pcState());
694
695 loadQueue[loadHead] = NULL;
696
697 incrLdIdx(loadHead);
698
699 --loads;
700}
701
702template <class Impl>
703void
704LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
705{
706 assert(loads == 0 || loadQueue[loadHead]);
707
708 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
709 commitLoad();
710 }
711}
712
713template <class Impl>
714void
715LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
716{
717 assert(stores == 0 || storeQueue[storeHead].inst);
718
719 int store_idx = storeHead;
720
721 while (store_idx != storeTail) {
722 assert(storeQueue[store_idx].inst);
723 // Mark any stores that are now committed and have not yet
724 // been marked as able to write back.
725 if (!storeQueue[store_idx].canWB) {
726 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
727 break;
728 }
729 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
730 "%s [sn:%lli]\n",
731 storeQueue[store_idx].inst->pcState(),
732 storeQueue[store_idx].inst->seqNum);
733
734 storeQueue[store_idx].canWB = true;
735
736 ++storesToWB;
737 }
738
739 incrStIdx(store_idx);
740 }
741}
742
743template <class Impl>
744void
745LSQUnit<Impl>::writebackPendingStore()
746{
747 if (hasPendingPkt) {
748 assert(pendingPkt != NULL);
749
750 // If the cache is blocked, this will store the packet for retry.
751 if (sendStore(pendingPkt)) {
752 storePostSend(pendingPkt);
753 }
754 pendingPkt = NULL;
755 hasPendingPkt = false;
756 }
757}
758
759template <class Impl>
760void
761LSQUnit<Impl>::writebackStores()
762{
763 // First writeback the second packet from any split store that didn't
764 // complete last cycle because there weren't enough cache ports available.
765 if (TheISA::HasUnalignedMemAcc) {
766 writebackPendingStore();
767 }
768
769 while (storesToWB > 0 &&
770 storeWBIdx != storeTail &&
771 storeQueue[storeWBIdx].inst &&
772 storeQueue[storeWBIdx].canWB &&
186}
187
188template<class Impl>
189std::string
190LSQUnit<Impl>::name() const
191{
192 if (Impl::MaxThreads == 1) {
193 return iewStage->name() + ".lsq";
194 } else {
195 return iewStage->name() + ".lsq.thread" + to_string(lsqID);
196 }
197}
198
199template<class Impl>
200void
201LSQUnit<Impl>::regStats()
202{
203 lsqForwLoads
204 .name(name() + ".forwLoads")
205 .desc("Number of loads that had data forwarded from stores");
206
207 invAddrLoads
208 .name(name() + ".invAddrLoads")
209 .desc("Number of loads ignored due to an invalid address");
210
211 lsqSquashedLoads
212 .name(name() + ".squashedLoads")
213 .desc("Number of loads squashed");
214
215 lsqIgnoredResponses
216 .name(name() + ".ignoredResponses")
217 .desc("Number of memory responses ignored because the instruction is squashed");
218
219 lsqMemOrderViolation
220 .name(name() + ".memOrderViolation")
221 .desc("Number of memory ordering violations");
222
223 lsqSquashedStores
224 .name(name() + ".squashedStores")
225 .desc("Number of stores squashed");
226
227 invAddrSwpfs
228 .name(name() + ".invAddrSwpfs")
229 .desc("Number of software prefetches ignored due to an invalid address");
230
231 lsqBlockedLoads
232 .name(name() + ".blockedLoads")
233 .desc("Number of blocked loads due to partial load-store forwarding");
234
235 lsqRescheduledLoads
236 .name(name() + ".rescheduledLoads")
237 .desc("Number of loads that were rescheduled");
238
239 lsqCacheBlocked
240 .name(name() + ".cacheBlocked")
241 .desc("Number of times an access to memory failed due to the cache being blocked");
242}
243
244template<class Impl>
245void
246LSQUnit<Impl>::setDcachePort(Port *dcache_port)
247{
248 dcachePort = dcache_port;
249
250#if USE_CHECKER
251 if (cpu->checker) {
252 cpu->checker->setDcachePort(dcachePort);
253 }
254#endif
255}
256
257template<class Impl>
258void
259LSQUnit<Impl>::clearLQ()
260{
261 loadQueue.clear();
262}
263
264template<class Impl>
265void
266LSQUnit<Impl>::clearSQ()
267{
268 storeQueue.clear();
269}
270
271template<class Impl>
272void
273LSQUnit<Impl>::switchOut()
274{
275 switchedOut = true;
276 for (int i = 0; i < loadQueue.size(); ++i) {
277 assert(!loadQueue[i]);
278 loadQueue[i] = NULL;
279 }
280
281 assert(storesToWB == 0);
282}
283
284template<class Impl>
285void
286LSQUnit<Impl>::takeOverFrom()
287{
288 switchedOut = false;
289 loads = stores = storesToWB = 0;
290
291 loadHead = loadTail = 0;
292
293 storeHead = storeWBIdx = storeTail = 0;
294
295 usedPorts = 0;
296
297 memDepViolator = NULL;
298
299 blockedLoadSeqNum = 0;
300
301 stalled = false;
302 isLoadBlocked = false;
303 loadBlockedHandled = false;
304
305 // Just incase the memory system changed out from under us
306 cacheBlockMask = 0;
307}
308
309template<class Impl>
310void
311LSQUnit<Impl>::resizeLQ(unsigned size)
312{
313 unsigned size_plus_sentinel = size + 1;
314 assert(size_plus_sentinel >= LQEntries);
315
316 if (size_plus_sentinel > LQEntries) {
317 while (size_plus_sentinel > loadQueue.size()) {
318 DynInstPtr dummy;
319 loadQueue.push_back(dummy);
320 LQEntries++;
321 }
322 } else {
323 LQEntries = size_plus_sentinel;
324 }
325
326}
327
328template<class Impl>
329void
330LSQUnit<Impl>::resizeSQ(unsigned size)
331{
332 unsigned size_plus_sentinel = size + 1;
333 if (size_plus_sentinel > SQEntries) {
334 while (size_plus_sentinel > storeQueue.size()) {
335 SQEntry dummy;
336 storeQueue.push_back(dummy);
337 SQEntries++;
338 }
339 } else {
340 SQEntries = size_plus_sentinel;
341 }
342}
343
344template <class Impl>
345void
346LSQUnit<Impl>::insert(DynInstPtr &inst)
347{
348 assert(inst->isMemRef());
349
350 assert(inst->isLoad() || inst->isStore());
351
352 if (inst->isLoad()) {
353 insertLoad(inst);
354 } else {
355 insertStore(inst);
356 }
357
358 inst->setInLSQ();
359}
360
361template <class Impl>
362void
363LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
364{
365 assert((loadTail + 1) % LQEntries != loadHead);
366 assert(loads < LQEntries);
367
368 DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
369 load_inst->pcState(), loadTail, load_inst->seqNum);
370
371 load_inst->lqIdx = loadTail;
372
373 if (stores == 0) {
374 load_inst->sqIdx = -1;
375 } else {
376 load_inst->sqIdx = storeTail;
377 }
378
379 loadQueue[loadTail] = load_inst;
380
381 incrLdIdx(loadTail);
382
383 ++loads;
384}
385
386template <class Impl>
387void
388LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
389{
390 // Make sure it is not full before inserting an instruction.
391 assert((storeTail + 1) % SQEntries != storeHead);
392 assert(stores < SQEntries);
393
394 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
395 store_inst->pcState(), storeTail, store_inst->seqNum);
396
397 store_inst->sqIdx = storeTail;
398 store_inst->lqIdx = loadTail;
399
400 storeQueue[storeTail] = SQEntry(store_inst);
401
402 incrStIdx(storeTail);
403
404 ++stores;
405}
406
407template <class Impl>
408typename Impl::DynInstPtr
409LSQUnit<Impl>::getMemDepViolator()
410{
411 DynInstPtr temp = memDepViolator;
412
413 memDepViolator = NULL;
414
415 return temp;
416}
417
418template <class Impl>
419unsigned
420LSQUnit<Impl>::numFreeEntries()
421{
422 unsigned free_lq_entries = LQEntries - loads;
423 unsigned free_sq_entries = SQEntries - stores;
424
425 // Both the LQ and SQ entries have an extra dummy entry to differentiate
426 // empty/full conditions. Subtract 1 from the free entries.
427 if (free_lq_entries < free_sq_entries) {
428 return free_lq_entries - 1;
429 } else {
430 return free_sq_entries - 1;
431 }
432}
433
434template <class Impl>
435int
436LSQUnit<Impl>::numLoadsReady()
437{
438 int load_idx = loadHead;
439 int retval = 0;
440
441 while (load_idx != loadTail) {
442 assert(loadQueue[load_idx]);
443
444 if (loadQueue[load_idx]->readyToIssue()) {
445 ++retval;
446 }
447 }
448
449 return retval;
450}
451
452template <class Impl>
453void
454LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
455{
456 int load_idx = loadHead;
457
458 if (!cacheBlockMask) {
459 assert(dcachePort);
460 Addr bs = dcachePort->peerBlockSize();
461
462 // Make sure we actually got a size
463 assert(bs != 0);
464
465 cacheBlockMask = ~(bs - 1);
466 }
467
468 // If this is the only load in the LSQ we don't care
469 if (load_idx == loadTail)
470 return;
471 incrLdIdx(load_idx);
472
473 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
474 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
475 while (load_idx != loadTail) {
476 DynInstPtr ld_inst = loadQueue[load_idx];
477
478 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
479 incrLdIdx(load_idx);
480 continue;
481 }
482
483 Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
484 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
485 ld_inst->seqNum, load_addr, invalidate_addr);
486
487 if (load_addr == invalidate_addr) {
488 if (ld_inst->possibleLoadViolation) {
489 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
490 ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
491
492 // Mark the load for re-execution
493 ld_inst->fault = new ReExec;
494 } else {
495 // If a older load checks this and it's true
496 // then we might have missed the snoop
497 // in which case we need to invalidate to be sure
498 ld_inst->hitExternalSnoop = true;
499 }
500 }
501 incrLdIdx(load_idx);
502 }
503 return;
504}
505
506template <class Impl>
507Fault
508LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
509{
510 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
511 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
512
513 /** @todo in theory you only need to check an instruction that has executed
514 * however, there isn't a good way in the pipeline at the moment to check
515 * all instructions that will execute before the store writes back. Thus,
516 * like the implementation that came before it, we're overly conservative.
517 */
518 while (load_idx != loadTail) {
519 DynInstPtr ld_inst = loadQueue[load_idx];
520 if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
521 incrLdIdx(load_idx);
522 continue;
523 }
524
525 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
526 Addr ld_eff_addr2 =
527 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
528
529 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
530 if (inst->isLoad()) {
531 // If this load is to the same block as an external snoop
532 // invalidate that we've observed then the load needs to be
533 // squashed as it could have newer data
534 if (ld_inst->hitExternalSnoop) {
535 if (!memDepViolator ||
536 ld_inst->seqNum < memDepViolator->seqNum) {
537 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
538 "and [sn:%lli] at address %#x\n",
539 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
540 memDepViolator = ld_inst;
541
542 ++lsqMemOrderViolation;
543
544 return new GenericISA::M5PanicFault(
545 "Detected fault with inst [sn:%lli] and "
546 "[sn:%lli] at address %#x\n",
547 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
548 }
549 }
550
551 // Otherwise, mark the load has a possible load violation
552 // and if we see a snoop before it's commited, we need to squash
553 ld_inst->possibleLoadViolation = true;
554 DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
555 " between instructions [sn:%lli] and [sn:%lli]\n",
556 inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
557 } else {
558 // A load/store incorrectly passed this store.
559 // Check if we already have a violator, or if it's newer
560 // squash and refetch.
561 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
562 break;
563
564 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
565 "[sn:%lli] at address %#x\n",
566 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
567 memDepViolator = ld_inst;
568
569 ++lsqMemOrderViolation;
570
571 return new GenericISA::M5PanicFault("Detected fault with "
572 "inst [sn:%lli] and [sn:%lli] at address %#x\n",
573 inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
574 }
575 }
576
577 incrLdIdx(load_idx);
578 }
579 return NoFault;
580}
581
582
583
584
585template <class Impl>
586Fault
587LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
588{
589 using namespace TheISA;
590 // Execute a specific load.
591 Fault load_fault = NoFault;
592
593 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
594 inst->pcState(), inst->seqNum);
595
596 assert(!inst->isSquashed());
597
598 load_fault = inst->initiateAcc();
599
600 if (inst->isTranslationDelayed() &&
601 load_fault == NoFault)
602 return load_fault;
603
604 // If the instruction faulted or predicated false, then we need to send it
605 // along to commit without the instruction completing.
606 if (load_fault != NoFault || inst->readPredicate() == false) {
607 // Send this instruction to commit, also make sure iew stage
608 // realizes there is activity.
609 // Mark it as executed unless it is an uncached load that
610 // needs to hit the head of commit.
611 if (inst->readPredicate() == false)
612 inst->forwardOldRegs();
613 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
614 inst->seqNum,
615 (load_fault != NoFault ? "fault" : "predication"));
616 if (!(inst->hasRequest() && inst->uncacheable()) ||
617 inst->isAtCommit()) {
618 inst->setExecuted();
619 }
620 iewStage->instToCommit(inst);
621 iewStage->activityThisCycle();
622 } else if (!loadBlocked()) {
623 assert(inst->effAddrValid);
624 int load_idx = inst->lqIdx;
625 incrLdIdx(load_idx);
626
627 if (checkLoads)
628 return checkViolations(load_idx, inst);
629 }
630
631 return load_fault;
632}
633
634template <class Impl>
635Fault
636LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
637{
638 using namespace TheISA;
639 // Make sure that a store exists.
640 assert(stores != 0);
641
642 int store_idx = store_inst->sqIdx;
643
644 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
645 store_inst->pcState(), store_inst->seqNum);
646
647 assert(!store_inst->isSquashed());
648
649 // Check the recently completed loads to see if any match this store's
650 // address. If so, then we have a memory ordering violation.
651 int load_idx = store_inst->lqIdx;
652
653 Fault store_fault = store_inst->initiateAcc();
654
655 if (store_inst->isTranslationDelayed() &&
656 store_fault == NoFault)
657 return store_fault;
658
659 if (store_inst->readPredicate() == false)
660 store_inst->forwardOldRegs();
661
662 if (storeQueue[store_idx].size == 0) {
663 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
664 store_inst->pcState(), store_inst->seqNum);
665
666 return store_fault;
667 } else if (store_inst->readPredicate() == false) {
668 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
669 store_inst->seqNum);
670 return store_fault;
671 }
672
673 assert(store_fault == NoFault);
674
675 if (store_inst->isStoreConditional()) {
676 // Store conditionals need to set themselves as able to
677 // writeback if we haven't had a fault by here.
678 storeQueue[store_idx].canWB = true;
679
680 ++storesToWB;
681 }
682
683 return checkViolations(load_idx, store_inst);
684
685}
686
687template <class Impl>
688void
689LSQUnit<Impl>::commitLoad()
690{
691 assert(loadQueue[loadHead]);
692
693 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
694 loadQueue[loadHead]->pcState());
695
696 loadQueue[loadHead] = NULL;
697
698 incrLdIdx(loadHead);
699
700 --loads;
701}
702
703template <class Impl>
704void
705LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
706{
707 assert(loads == 0 || loadQueue[loadHead]);
708
709 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
710 commitLoad();
711 }
712}
713
714template <class Impl>
715void
716LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
717{
718 assert(stores == 0 || storeQueue[storeHead].inst);
719
720 int store_idx = storeHead;
721
722 while (store_idx != storeTail) {
723 assert(storeQueue[store_idx].inst);
724 // Mark any stores that are now committed and have not yet
725 // been marked as able to write back.
726 if (!storeQueue[store_idx].canWB) {
727 if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
728 break;
729 }
730 DPRINTF(LSQUnit, "Marking store as able to write back, PC "
731 "%s [sn:%lli]\n",
732 storeQueue[store_idx].inst->pcState(),
733 storeQueue[store_idx].inst->seqNum);
734
735 storeQueue[store_idx].canWB = true;
736
737 ++storesToWB;
738 }
739
740 incrStIdx(store_idx);
741 }
742}
743
744template <class Impl>
745void
746LSQUnit<Impl>::writebackPendingStore()
747{
748 if (hasPendingPkt) {
749 assert(pendingPkt != NULL);
750
751 // If the cache is blocked, this will store the packet for retry.
752 if (sendStore(pendingPkt)) {
753 storePostSend(pendingPkt);
754 }
755 pendingPkt = NULL;
756 hasPendingPkt = false;
757 }
758}
759
760template <class Impl>
761void
762LSQUnit<Impl>::writebackStores()
763{
764 // First writeback the second packet from any split store that didn't
765 // complete last cycle because there weren't enough cache ports available.
766 if (TheISA::HasUnalignedMemAcc) {
767 writebackPendingStore();
768 }
769
770 while (storesToWB > 0 &&
771 storeWBIdx != storeTail &&
772 storeQueue[storeWBIdx].inst &&
773 storeQueue[storeWBIdx].canWB &&
774 ((!needsTSO) || (!storeInFlight)) &&
773 usedPorts < cachePorts) {
774
775 if (isStoreBlocked || lsq->cacheBlocked()) {
776 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
777 " is blocked!\n");
778 break;
779 }
780
781 // Store didn't write any data so no need to write it back to
782 // memory.
783 if (storeQueue[storeWBIdx].size == 0) {
784 completeStore(storeWBIdx);
785
786 incrStIdx(storeWBIdx);
787
788 continue;
789 }
790
791 ++usedPorts;
792
793 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
794 incrStIdx(storeWBIdx);
795
796 continue;
797 }
798
799 assert(storeQueue[storeWBIdx].req);
800 assert(!storeQueue[storeWBIdx].committed);
801
802 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
803 assert(storeQueue[storeWBIdx].sreqLow);
804 assert(storeQueue[storeWBIdx].sreqHigh);
805 }
806
807 DynInstPtr inst = storeQueue[storeWBIdx].inst;
808
809 Request *req = storeQueue[storeWBIdx].req;
810 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
811 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
812
813 storeQueue[storeWBIdx].committed = true;
814
815 assert(!inst->memData);
816 inst->memData = new uint8_t[64];
817
818 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
819
820 MemCmd command =
821 req->isSwap() ? MemCmd::SwapReq :
822 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
823 PacketPtr data_pkt;
824 PacketPtr snd_data_pkt = NULL;
825
826 LSQSenderState *state = new LSQSenderState;
827 state->isLoad = false;
828 state->idx = storeWBIdx;
829 state->inst = inst;
830
831 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
832
833 // Build a single data packet if the store isn't split.
834 data_pkt = new Packet(req, command, Packet::Broadcast);
835 data_pkt->dataStatic(inst->memData);
836 data_pkt->senderState = state;
837 } else {
838 // Create two packets if the store is split in two.
839 data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
840 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
841
842 data_pkt->dataStatic(inst->memData);
843 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
844
845 data_pkt->senderState = state;
846 snd_data_pkt->senderState = state;
847
848 state->isSplit = true;
849 state->outstanding = 2;
850
851 // Can delete the main request now.
852 delete req;
853 req = sreqLow;
854 }
855
856 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
857 "to Addr:%#x, data:%#x [sn:%lli]\n",
858 storeWBIdx, inst->pcState(),
859 req->getPaddr(), (int)*(inst->memData),
860 inst->seqNum);
861
862 // @todo: Remove this SC hack once the memory system handles it.
863 if (inst->isStoreConditional()) {
864 assert(!storeQueue[storeWBIdx].isSplit);
865 // Disable recording the result temporarily. Writing to
866 // misc regs normally updates the result, but this is not
867 // the desired behavior when handling store conditionals.
868 inst->recordResult = false;
869 bool success = TheISA::handleLockedWrite(inst.get(), req);
870 inst->recordResult = true;
871
872 if (!success) {
873 // Instantly complete this store.
874 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
875 "Instantly completing it.\n",
876 inst->seqNum);
877 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
878 cpu->schedule(wb, curTick() + 1);
879 completeStore(storeWBIdx);
880 incrStIdx(storeWBIdx);
881 continue;
882 }
883 } else {
884 // Non-store conditionals do not need a writeback.
885 state->noWB = true;
886 }
887
888 bool split =
889 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
890
891 ThreadContext *thread = cpu->tcBase(lsqID);
892
893 if (req->isMmappedIpr()) {
894 assert(!inst->isStoreConditional());
895 TheISA::handleIprWrite(thread, data_pkt);
896 delete data_pkt;
897 if (split) {
898 assert(snd_data_pkt->req->isMmappedIpr());
899 TheISA::handleIprWrite(thread, snd_data_pkt);
900 delete snd_data_pkt;
901 delete sreqLow;
902 delete sreqHigh;
903 }
904 delete state;
905 delete req;
906 completeStore(storeWBIdx);
907 incrStIdx(storeWBIdx);
908 } else if (!sendStore(data_pkt)) {
909 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
910 "retry later\n",
911 inst->seqNum);
912
913 // Need to store the second packet, if split.
914 if (split) {
915 state->pktToSend = true;
916 state->pendingPacket = snd_data_pkt;
917 }
918 } else {
919
920 // If split, try to send the second packet too
921 if (split) {
922 assert(snd_data_pkt);
923
924 // Ensure there are enough ports to use.
925 if (usedPorts < cachePorts) {
926 ++usedPorts;
927 if (sendStore(snd_data_pkt)) {
928 storePostSend(snd_data_pkt);
929 } else {
930 DPRINTF(IEW, "D-Cache became blocked when writing"
931 " [sn:%lli] second packet, will retry later\n",
932 inst->seqNum);
933 }
934 } else {
935
936 // Store the packet for when there's free ports.
937 assert(pendingPkt == NULL);
938 pendingPkt = snd_data_pkt;
939 hasPendingPkt = true;
940 }
941 } else {
942
943 // Not a split store.
944 storePostSend(data_pkt);
945 }
946 }
947 }
948
949 // Not sure this should set it to 0.
950 usedPorts = 0;
951
952 assert(stores >= 0 && storesToWB >= 0);
953}
954
955/*template <class Impl>
956void
957LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
958{
959 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
960 mshrSeqNums.end(),
961 seqNum);
962
963 if (mshr_it != mshrSeqNums.end()) {
964 mshrSeqNums.erase(mshr_it);
965 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
966 }
967}*/
968
969template <class Impl>
970void
971LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
972{
973 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
974 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
975
976 int load_idx = loadTail;
977 decrLdIdx(load_idx);
978
979 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
980 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
981 "[sn:%lli]\n",
982 loadQueue[load_idx]->pcState(),
983 loadQueue[load_idx]->seqNum);
984
985 if (isStalled() && load_idx == stallingLoadIdx) {
986 stalled = false;
987 stallingStoreIsn = 0;
988 stallingLoadIdx = 0;
989 }
990
991 // Clear the smart pointer to make sure it is decremented.
992 loadQueue[load_idx]->setSquashed();
993 loadQueue[load_idx] = NULL;
994 --loads;
995
996 // Inefficient!
997 loadTail = load_idx;
998
999 decrLdIdx(load_idx);
1000 ++lsqSquashedLoads;
1001 }
1002
1003 if (isLoadBlocked) {
1004 if (squashed_num < blockedLoadSeqNum) {
1005 isLoadBlocked = false;
1006 loadBlockedHandled = false;
1007 blockedLoadSeqNum = 0;
1008 }
1009 }
1010
1011 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1012 memDepViolator = NULL;
1013 }
1014
1015 int store_idx = storeTail;
1016 decrStIdx(store_idx);
1017
1018 while (stores != 0 &&
1019 storeQueue[store_idx].inst->seqNum > squashed_num) {
1020 // Instructions marked as can WB are already committed.
1021 if (storeQueue[store_idx].canWB) {
1022 break;
1023 }
1024
1025 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1026 "idx:%i [sn:%lli]\n",
1027 storeQueue[store_idx].inst->pcState(),
1028 store_idx, storeQueue[store_idx].inst->seqNum);
1029
1030 // I don't think this can happen. It should have been cleared
1031 // by the stalling load.
1032 if (isStalled() &&
1033 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1034 panic("Is stalled should have been cleared by stalling load!\n");
1035 stalled = false;
1036 stallingStoreIsn = 0;
1037 }
1038
1039 // Clear the smart pointer to make sure it is decremented.
1040 storeQueue[store_idx].inst->setSquashed();
1041 storeQueue[store_idx].inst = NULL;
1042 storeQueue[store_idx].canWB = 0;
1043
1044 // Must delete request now that it wasn't handed off to
1045 // memory. This is quite ugly. @todo: Figure out the proper
1046 // place to really handle request deletes.
1047 delete storeQueue[store_idx].req;
1048 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1049 delete storeQueue[store_idx].sreqLow;
1050 delete storeQueue[store_idx].sreqHigh;
1051
1052 storeQueue[store_idx].sreqLow = NULL;
1053 storeQueue[store_idx].sreqHigh = NULL;
1054 }
1055
1056 storeQueue[store_idx].req = NULL;
1057 --stores;
1058
1059 // Inefficient!
1060 storeTail = store_idx;
1061
1062 decrStIdx(store_idx);
1063 ++lsqSquashedStores;
1064 }
1065}
1066
1067template <class Impl>
1068void
1069LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1070{
1071 if (isStalled() &&
1072 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1073 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1074 "load idx:%i\n",
1075 stallingStoreIsn, stallingLoadIdx);
1076 stalled = false;
1077 stallingStoreIsn = 0;
1078 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1079 }
1080
1081 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1082 // The store is basically completed at this time. This
1083 // only works so long as the checker doesn't try to
1084 // verify the value in memory for stores.
1085 storeQueue[storeWBIdx].inst->setCompleted();
1086#if USE_CHECKER
1087 if (cpu->checker) {
1088 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1089 }
1090#endif
1091 }
1092
775 usedPorts < cachePorts) {
776
777 if (isStoreBlocked || lsq->cacheBlocked()) {
778 DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
779 " is blocked!\n");
780 break;
781 }
782
783 // Store didn't write any data so no need to write it back to
784 // memory.
785 if (storeQueue[storeWBIdx].size == 0) {
786 completeStore(storeWBIdx);
787
788 incrStIdx(storeWBIdx);
789
790 continue;
791 }
792
793 ++usedPorts;
794
795 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
796 incrStIdx(storeWBIdx);
797
798 continue;
799 }
800
801 assert(storeQueue[storeWBIdx].req);
802 assert(!storeQueue[storeWBIdx].committed);
803
804 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
805 assert(storeQueue[storeWBIdx].sreqLow);
806 assert(storeQueue[storeWBIdx].sreqHigh);
807 }
808
809 DynInstPtr inst = storeQueue[storeWBIdx].inst;
810
811 Request *req = storeQueue[storeWBIdx].req;
812 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
813 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
814
815 storeQueue[storeWBIdx].committed = true;
816
817 assert(!inst->memData);
818 inst->memData = new uint8_t[64];
819
820 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
821
822 MemCmd command =
823 req->isSwap() ? MemCmd::SwapReq :
824 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
825 PacketPtr data_pkt;
826 PacketPtr snd_data_pkt = NULL;
827
828 LSQSenderState *state = new LSQSenderState;
829 state->isLoad = false;
830 state->idx = storeWBIdx;
831 state->inst = inst;
832
833 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
834
835 // Build a single data packet if the store isn't split.
836 data_pkt = new Packet(req, command, Packet::Broadcast);
837 data_pkt->dataStatic(inst->memData);
838 data_pkt->senderState = state;
839 } else {
840 // Create two packets if the store is split in two.
841 data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
842 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
843
844 data_pkt->dataStatic(inst->memData);
845 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
846
847 data_pkt->senderState = state;
848 snd_data_pkt->senderState = state;
849
850 state->isSplit = true;
851 state->outstanding = 2;
852
853 // Can delete the main request now.
854 delete req;
855 req = sreqLow;
856 }
857
858 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
859 "to Addr:%#x, data:%#x [sn:%lli]\n",
860 storeWBIdx, inst->pcState(),
861 req->getPaddr(), (int)*(inst->memData),
862 inst->seqNum);
863
864 // @todo: Remove this SC hack once the memory system handles it.
865 if (inst->isStoreConditional()) {
866 assert(!storeQueue[storeWBIdx].isSplit);
867 // Disable recording the result temporarily. Writing to
868 // misc regs normally updates the result, but this is not
869 // the desired behavior when handling store conditionals.
870 inst->recordResult = false;
871 bool success = TheISA::handleLockedWrite(inst.get(), req);
872 inst->recordResult = true;
873
874 if (!success) {
875 // Instantly complete this store.
876 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. "
877 "Instantly completing it.\n",
878 inst->seqNum);
879 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
880 cpu->schedule(wb, curTick() + 1);
881 completeStore(storeWBIdx);
882 incrStIdx(storeWBIdx);
883 continue;
884 }
885 } else {
886 // Non-store conditionals do not need a writeback.
887 state->noWB = true;
888 }
889
890 bool split =
891 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
892
893 ThreadContext *thread = cpu->tcBase(lsqID);
894
895 if (req->isMmappedIpr()) {
896 assert(!inst->isStoreConditional());
897 TheISA::handleIprWrite(thread, data_pkt);
898 delete data_pkt;
899 if (split) {
900 assert(snd_data_pkt->req->isMmappedIpr());
901 TheISA::handleIprWrite(thread, snd_data_pkt);
902 delete snd_data_pkt;
903 delete sreqLow;
904 delete sreqHigh;
905 }
906 delete state;
907 delete req;
908 completeStore(storeWBIdx);
909 incrStIdx(storeWBIdx);
910 } else if (!sendStore(data_pkt)) {
911 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
912 "retry later\n",
913 inst->seqNum);
914
915 // Need to store the second packet, if split.
916 if (split) {
917 state->pktToSend = true;
918 state->pendingPacket = snd_data_pkt;
919 }
920 } else {
921
922 // If split, try to send the second packet too
923 if (split) {
924 assert(snd_data_pkt);
925
926 // Ensure there are enough ports to use.
927 if (usedPorts < cachePorts) {
928 ++usedPorts;
929 if (sendStore(snd_data_pkt)) {
930 storePostSend(snd_data_pkt);
931 } else {
932 DPRINTF(IEW, "D-Cache became blocked when writing"
933 " [sn:%lli] second packet, will retry later\n",
934 inst->seqNum);
935 }
936 } else {
937
938 // Store the packet for when there's free ports.
939 assert(pendingPkt == NULL);
940 pendingPkt = snd_data_pkt;
941 hasPendingPkt = true;
942 }
943 } else {
944
945 // Not a split store.
946 storePostSend(data_pkt);
947 }
948 }
949 }
950
951 // Not sure this should set it to 0.
952 usedPorts = 0;
953
954 assert(stores >= 0 && storesToWB >= 0);
955}
956
957/*template <class Impl>
958void
959LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
960{
961 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
962 mshrSeqNums.end(),
963 seqNum);
964
965 if (mshr_it != mshrSeqNums.end()) {
966 mshrSeqNums.erase(mshr_it);
967 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
968 }
969}*/
970
971template <class Impl>
972void
973LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
974{
975 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
976 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
977
978 int load_idx = loadTail;
979 decrLdIdx(load_idx);
980
981 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
982 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
983 "[sn:%lli]\n",
984 loadQueue[load_idx]->pcState(),
985 loadQueue[load_idx]->seqNum);
986
987 if (isStalled() && load_idx == stallingLoadIdx) {
988 stalled = false;
989 stallingStoreIsn = 0;
990 stallingLoadIdx = 0;
991 }
992
993 // Clear the smart pointer to make sure it is decremented.
994 loadQueue[load_idx]->setSquashed();
995 loadQueue[load_idx] = NULL;
996 --loads;
997
998 // Inefficient!
999 loadTail = load_idx;
1000
1001 decrLdIdx(load_idx);
1002 ++lsqSquashedLoads;
1003 }
1004
1005 if (isLoadBlocked) {
1006 if (squashed_num < blockedLoadSeqNum) {
1007 isLoadBlocked = false;
1008 loadBlockedHandled = false;
1009 blockedLoadSeqNum = 0;
1010 }
1011 }
1012
1013 if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1014 memDepViolator = NULL;
1015 }
1016
1017 int store_idx = storeTail;
1018 decrStIdx(store_idx);
1019
1020 while (stores != 0 &&
1021 storeQueue[store_idx].inst->seqNum > squashed_num) {
1022 // Instructions marked as can WB are already committed.
1023 if (storeQueue[store_idx].canWB) {
1024 break;
1025 }
1026
1027 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1028 "idx:%i [sn:%lli]\n",
1029 storeQueue[store_idx].inst->pcState(),
1030 store_idx, storeQueue[store_idx].inst->seqNum);
1031
1032 // I don't think this can happen. It should have been cleared
1033 // by the stalling load.
1034 if (isStalled() &&
1035 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1036 panic("Is stalled should have been cleared by stalling load!\n");
1037 stalled = false;
1038 stallingStoreIsn = 0;
1039 }
1040
1041 // Clear the smart pointer to make sure it is decremented.
1042 storeQueue[store_idx].inst->setSquashed();
1043 storeQueue[store_idx].inst = NULL;
1044 storeQueue[store_idx].canWB = 0;
1045
1046 // Must delete request now that it wasn't handed off to
1047 // memory. This is quite ugly. @todo: Figure out the proper
1048 // place to really handle request deletes.
1049 delete storeQueue[store_idx].req;
1050 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1051 delete storeQueue[store_idx].sreqLow;
1052 delete storeQueue[store_idx].sreqHigh;
1053
1054 storeQueue[store_idx].sreqLow = NULL;
1055 storeQueue[store_idx].sreqHigh = NULL;
1056 }
1057
1058 storeQueue[store_idx].req = NULL;
1059 --stores;
1060
1061 // Inefficient!
1062 storeTail = store_idx;
1063
1064 decrStIdx(store_idx);
1065 ++lsqSquashedStores;
1066 }
1067}
1068
1069template <class Impl>
1070void
1071LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1072{
1073 if (isStalled() &&
1074 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1075 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1076 "load idx:%i\n",
1077 stallingStoreIsn, stallingLoadIdx);
1078 stalled = false;
1079 stallingStoreIsn = 0;
1080 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1081 }
1082
1083 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1084 // The store is basically completed at this time. This
1085 // only works so long as the checker doesn't try to
1086 // verify the value in memory for stores.
1087 storeQueue[storeWBIdx].inst->setCompleted();
1088#if USE_CHECKER
1089 if (cpu->checker) {
1090 cpu->checker->verify(storeQueue[storeWBIdx].inst);
1091 }
1092#endif
1093 }
1094
1095 if (needsTSO) {
1096 storeInFlight = true;
1097 }
1098
1093 incrStIdx(storeWBIdx);
1094}
1095
1096template <class Impl>
1097void
1098LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1099{
1100 iewStage->wakeCPU();
1101
1102 // Squashed instructions do not need to complete their access.
1103 if (inst->isSquashed()) {
1104 iewStage->decrWb(inst->seqNum);
1105 assert(!inst->isStore());
1106 ++lsqIgnoredResponses;
1107 return;
1108 }
1109
1110 if (!inst->isExecuted()) {
1111 inst->setExecuted();
1112
1113 // Complete access to copy data to proper place.
1114 inst->completeAcc(pkt);
1115 }
1116
1117 // Need to insert instruction into queue to commit
1118 iewStage->instToCommit(inst);
1119
1120 iewStage->activityThisCycle();
1121
1122 // see if this load changed the PC
1123 iewStage->checkMisprediction(inst);
1124}
1125
1126template <class Impl>
1127void
1128LSQUnit<Impl>::completeStore(int store_idx)
1129{
1130 assert(storeQueue[store_idx].inst);
1131 storeQueue[store_idx].completed = true;
1132 --storesToWB;
1133 // A bit conservative because a store completion may not free up entries,
1134 // but hopefully avoids two store completions in one cycle from making
1135 // the CPU tick twice.
1136 cpu->wakeCPU();
1137 cpu->activityThisCycle();
1138
1139 if (store_idx == storeHead) {
1140 do {
1141 incrStIdx(storeHead);
1142
1143 --stores;
1144 } while (storeQueue[storeHead].completed &&
1145 storeHead != storeTail);
1146
1147 iewStage->updateLSQNextCycle = true;
1148 }
1149
1150 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1151 "idx:%i\n",
1152 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1153
1154 if (isStalled() &&
1155 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1156 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1157 "load idx:%i\n",
1158 stallingStoreIsn, stallingLoadIdx);
1159 stalled = false;
1160 stallingStoreIsn = 0;
1161 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1162 }
1163
1164 storeQueue[store_idx].inst->setCompleted();
1165
1099 incrStIdx(storeWBIdx);
1100}
1101
1102template <class Impl>
1103void
1104LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1105{
1106 iewStage->wakeCPU();
1107
1108 // Squashed instructions do not need to complete their access.
1109 if (inst->isSquashed()) {
1110 iewStage->decrWb(inst->seqNum);
1111 assert(!inst->isStore());
1112 ++lsqIgnoredResponses;
1113 return;
1114 }
1115
1116 if (!inst->isExecuted()) {
1117 inst->setExecuted();
1118
1119 // Complete access to copy data to proper place.
1120 inst->completeAcc(pkt);
1121 }
1122
1123 // Need to insert instruction into queue to commit
1124 iewStage->instToCommit(inst);
1125
1126 iewStage->activityThisCycle();
1127
1128 // see if this load changed the PC
1129 iewStage->checkMisprediction(inst);
1130}
1131
1132template <class Impl>
1133void
1134LSQUnit<Impl>::completeStore(int store_idx)
1135{
1136 assert(storeQueue[store_idx].inst);
1137 storeQueue[store_idx].completed = true;
1138 --storesToWB;
1139 // A bit conservative because a store completion may not free up entries,
1140 // but hopefully avoids two store completions in one cycle from making
1141 // the CPU tick twice.
1142 cpu->wakeCPU();
1143 cpu->activityThisCycle();
1144
1145 if (store_idx == storeHead) {
1146 do {
1147 incrStIdx(storeHead);
1148
1149 --stores;
1150 } while (storeQueue[storeHead].completed &&
1151 storeHead != storeTail);
1152
1153 iewStage->updateLSQNextCycle = true;
1154 }
1155
1156 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1157 "idx:%i\n",
1158 storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1159
1160 if (isStalled() &&
1161 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1162 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1163 "load idx:%i\n",
1164 stallingStoreIsn, stallingLoadIdx);
1165 stalled = false;
1166 stallingStoreIsn = 0;
1167 iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1168 }
1169
1170 storeQueue[store_idx].inst->setCompleted();
1171
1172 if (needsTSO) {
1173 storeInFlight = false;
1174 }
1175
1166 // Tell the checker we've completed this instruction. Some stores
1167 // may get reported twice to the checker, but the checker can
1168 // handle that case.
1169#if USE_CHECKER
1170 if (cpu->checker) {
1171 cpu->checker->verify(storeQueue[store_idx].inst);
1172 }
1173#endif
1174}
1175
1176template <class Impl>
1177bool
1178LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1179{
1180 if (!dcachePort->sendTiming(data_pkt)) {
1181 // Need to handle becoming blocked on a store.
1182 isStoreBlocked = true;
1183 ++lsqCacheBlocked;
1184 assert(retryPkt == NULL);
1185 retryPkt = data_pkt;
1186 lsq->setRetryTid(lsqID);
1187 return false;
1188 }
1189 return true;
1190}
1191
1192template <class Impl>
1193void
1194LSQUnit<Impl>::recvRetry()
1195{
1196 if (isStoreBlocked) {
1197 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1198 assert(retryPkt != NULL);
1199
1200 LSQSenderState *state =
1201 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1202
1203 if (dcachePort->sendTiming(retryPkt)) {
1204 // Don't finish the store unless this is the last packet.
1205 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1206 state->pendingPacket == retryPkt) {
1207 state->pktToSend = false;
1208 storePostSend(retryPkt);
1209 }
1210 retryPkt = NULL;
1211 isStoreBlocked = false;
1212 lsq->setRetryTid(InvalidThreadID);
1213
1214 // Send any outstanding packet.
1215 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1216 assert(state->pendingPacket);
1217 if (sendStore(state->pendingPacket)) {
1218 storePostSend(state->pendingPacket);
1219 }
1220 }
1221 } else {
1222 // Still blocked!
1223 ++lsqCacheBlocked;
1224 lsq->setRetryTid(lsqID);
1225 }
1226 } else if (isLoadBlocked) {
1227 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1228 "no need to resend packet.\n");
1229 } else {
1230 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1231 }
1232}
1233
1234template <class Impl>
1235inline void
1236LSQUnit<Impl>::incrStIdx(int &store_idx)
1237{
1238 if (++store_idx >= SQEntries)
1239 store_idx = 0;
1240}
1241
1242template <class Impl>
1243inline void
1244LSQUnit<Impl>::decrStIdx(int &store_idx)
1245{
1246 if (--store_idx < 0)
1247 store_idx += SQEntries;
1248}
1249
1250template <class Impl>
1251inline void
1252LSQUnit<Impl>::incrLdIdx(int &load_idx)
1253{
1254 if (++load_idx >= LQEntries)
1255 load_idx = 0;
1256}
1257
1258template <class Impl>
1259inline void
1260LSQUnit<Impl>::decrLdIdx(int &load_idx)
1261{
1262 if (--load_idx < 0)
1263 load_idx += LQEntries;
1264}
1265
1266template <class Impl>
1267void
1268LSQUnit<Impl>::dumpInsts()
1269{
1270 cprintf("Load store queue: Dumping instructions.\n");
1271 cprintf("Load queue size: %i\n", loads);
1272 cprintf("Load queue: ");
1273
1274 int load_idx = loadHead;
1275
1276 while (load_idx != loadTail && loadQueue[load_idx]) {
1277 cprintf("%s ", loadQueue[load_idx]->pcState());
1278
1279 incrLdIdx(load_idx);
1280 }
1281
1282 cprintf("Store queue size: %i\n", stores);
1283 cprintf("Store queue: ");
1284
1285 int store_idx = storeHead;
1286
1287 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1288 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1289
1290 incrStIdx(store_idx);
1291 }
1292
1293 cprintf("\n");
1294}
1176 // Tell the checker we've completed this instruction. Some stores
1177 // may get reported twice to the checker, but the checker can
1178 // handle that case.
1179#if USE_CHECKER
1180 if (cpu->checker) {
1181 cpu->checker->verify(storeQueue[store_idx].inst);
1182 }
1183#endif
1184}
1185
1186template <class Impl>
1187bool
1188LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1189{
1190 if (!dcachePort->sendTiming(data_pkt)) {
1191 // Need to handle becoming blocked on a store.
1192 isStoreBlocked = true;
1193 ++lsqCacheBlocked;
1194 assert(retryPkt == NULL);
1195 retryPkt = data_pkt;
1196 lsq->setRetryTid(lsqID);
1197 return false;
1198 }
1199 return true;
1200}
1201
1202template <class Impl>
1203void
1204LSQUnit<Impl>::recvRetry()
1205{
1206 if (isStoreBlocked) {
1207 DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1208 assert(retryPkt != NULL);
1209
1210 LSQSenderState *state =
1211 dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1212
1213 if (dcachePort->sendTiming(retryPkt)) {
1214 // Don't finish the store unless this is the last packet.
1215 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1216 state->pendingPacket == retryPkt) {
1217 state->pktToSend = false;
1218 storePostSend(retryPkt);
1219 }
1220 retryPkt = NULL;
1221 isStoreBlocked = false;
1222 lsq->setRetryTid(InvalidThreadID);
1223
1224 // Send any outstanding packet.
1225 if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1226 assert(state->pendingPacket);
1227 if (sendStore(state->pendingPacket)) {
1228 storePostSend(state->pendingPacket);
1229 }
1230 }
1231 } else {
1232 // Still blocked!
1233 ++lsqCacheBlocked;
1234 lsq->setRetryTid(lsqID);
1235 }
1236 } else if (isLoadBlocked) {
1237 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1238 "no need to resend packet.\n");
1239 } else {
1240 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1241 }
1242}
1243
1244template <class Impl>
1245inline void
1246LSQUnit<Impl>::incrStIdx(int &store_idx)
1247{
1248 if (++store_idx >= SQEntries)
1249 store_idx = 0;
1250}
1251
1252template <class Impl>
1253inline void
1254LSQUnit<Impl>::decrStIdx(int &store_idx)
1255{
1256 if (--store_idx < 0)
1257 store_idx += SQEntries;
1258}
1259
1260template <class Impl>
1261inline void
1262LSQUnit<Impl>::incrLdIdx(int &load_idx)
1263{
1264 if (++load_idx >= LQEntries)
1265 load_idx = 0;
1266}
1267
1268template <class Impl>
1269inline void
1270LSQUnit<Impl>::decrLdIdx(int &load_idx)
1271{
1272 if (--load_idx < 0)
1273 load_idx += LQEntries;
1274}
1275
1276template <class Impl>
1277void
1278LSQUnit<Impl>::dumpInsts()
1279{
1280 cprintf("Load store queue: Dumping instructions.\n");
1281 cprintf("Load queue size: %i\n", loads);
1282 cprintf("Load queue: ");
1283
1284 int load_idx = loadHead;
1285
1286 while (load_idx != loadTail && loadQueue[load_idx]) {
1287 cprintf("%s ", loadQueue[load_idx]->pcState());
1288
1289 incrLdIdx(load_idx);
1290 }
1291
1292 cprintf("Store queue size: %i\n", stores);
1293 cprintf("Store queue: ");
1294
1295 int store_idx = storeHead;
1296
1297 while (store_idx != storeTail && storeQueue[store_idx].inst) {
1298 cprintf("%s ", storeQueue[store_idx].inst->pcState());
1299
1300 incrStIdx(store_idx);
1301 }
1302
1303 cprintf("\n");
1304}