lsq_unit_impl.hh revision 5336:c7e21f4e5a2e 
1/*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Kevin Lim
29 *          Korey Sewell
30 */
31
32#include "arch/locked_mem.hh"
33#include "config/use_checker.hh"
34
35#include "cpu/o3/lsq.hh"
36#include "cpu/o3/lsq_unit.hh"
37#include "base/str.hh"
38#include "mem/packet.hh"
39#include "mem/request.hh"
40
41#if USE_CHECKER
42#include "cpu/checker/cpu.hh"
43#endif
44
45template<class Impl>
46LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
47                                              LSQUnit *lsq_ptr)
48    : Event(&mainEventQueue), inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
49{
50    this->setFlags(Event::AutoDelete);
51}
52
53template<class Impl>
54void
55LSQUnit<Impl>::WritebackEvent::process()
56{
57    if (!lsqPtr->isSwitchedOut()) {
58        lsqPtr->writeback(inst, pkt);
59    }
60
61    if (pkt->senderState)
62        delete pkt->senderState;
63
64    delete pkt->req;
65    delete pkt;
66}
67
68template<class Impl>
69const char *
70LSQUnit<Impl>::WritebackEvent::description() const
71{
72    return "Store writeback";
73}
74
75template<class Impl>
76void
77LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
78{
79    LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
80    DynInstPtr inst = state->inst;
81    DPRINTF(IEW, "Writeback event [sn:%lli]\n", inst->seqNum);
82    DPRINTF(Activity, "Activity: Writeback event [sn:%lli]\n", inst->seqNum);
83
84    //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
85
86    assert(!pkt->wasNacked());
87
88    if (isSwitchedOut() || inst->isSquashed()) {
89        iewStage->decrWb(inst->seqNum);
90    } else {
91        if (!state->noWB) {
92            writeback(inst, pkt);
93        }
94
95        if (inst->isStore()) {
96            completeStore(state->idx);
97        }
98    }
99
100    delete state;
101    delete pkt->req;
102    delete pkt;
103}
104
105template <class Impl>
106LSQUnit<Impl>::LSQUnit()
107    : loads(0), stores(0), storesToWB(0), stalled(false),
108      isStoreBlocked(false), isLoadBlocked(false),
109      loadBlockedHandled(false)
110{
111}
112
113template<class Impl>
114void
115LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, Params *params, LSQ *lsq_ptr,
116                    unsigned maxLQEntries, unsigned maxSQEntries, unsigned id)
117{
118    cpu = cpu_ptr;
119    iewStage = iew_ptr;
120
121    DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
122
123    switchedOut = false;
124
125    lsq = lsq_ptr;
126
127    lsqID = id;
128
129    // Add 1 for the sentinel entry (they are circular queues).
130    LQEntries = maxLQEntries + 1;
131    SQEntries = maxSQEntries + 1;
132
133    loadQueue.resize(LQEntries);
134    storeQueue.resize(SQEntries);
135
136    loadHead = loadTail = 0;
137
138    storeHead = storeWBIdx = storeTail = 0;
139
140    usedPorts = 0;
141    cachePorts = params->cachePorts;
142
143    retryPkt = NULL;
144    memDepViolator = NULL;
145
146    blockedLoadSeqNum = 0;
147}
148
149template<class Impl>
150std::string
151LSQUnit<Impl>::name() const
152{
153    if (Impl::MaxThreads == 1) {
154        return iewStage->name() + ".lsq";
155    } else {
156        return iewStage->name() + ".lsq.thread." + to_string(lsqID);
157    }
158}
159
160template<class Impl>
161void
162LSQUnit<Impl>::regStats()
163{
164    lsqForwLoads
165        .name(name() + ".forwLoads")
166        .desc("Number of loads that had data forwarded from stores");
167
168    invAddrLoads
169        .name(name() + ".invAddrLoads")
170        .desc("Number of loads ignored due to an invalid address");
171
172    lsqSquashedLoads
173        .name(name() + ".squashedLoads")
174        .desc("Number of loads squashed");
175
176    lsqIgnoredResponses
177        .name(name() + ".ignoredResponses")
178        .desc("Number of memory responses ignored because the instruction is squashed");
179
180    lsqMemOrderViolation
181        .name(name() + ".memOrderViolation")
182        .desc("Number of memory ordering violations");
183
184    lsqSquashedStores
185        .name(name() + ".squashedStores")
186        .desc("Number of stores squashed");
187
188    invAddrSwpfs
189        .name(name() + ".invAddrSwpfs")
190        .desc("Number of software prefetches ignored due to an invalid address");
191
192    lsqBlockedLoads
193        .name(name() + ".blockedLoads")
194        .desc("Number of blocked loads due to partial load-store forwarding");
195
196    lsqRescheduledLoads
197        .name(name() + ".rescheduledLoads")
198        .desc("Number of loads that were rescheduled");
199
200    lsqCacheBlocked
201        .name(name() + ".cacheBlocked")
202        .desc("Number of times an access to memory failed due to the cache being blocked");
203}
204
205template<class Impl>
206void
207LSQUnit<Impl>::setDcachePort(Port *dcache_port)
208{
209    dcachePort = dcache_port;
210
211#if USE_CHECKER
212    if (cpu->checker) {
213        cpu->checker->setDcachePort(dcachePort);
214    }
215#endif
216}
217
218template<class Impl>
219void
220LSQUnit<Impl>::clearLQ()
221{
222    loadQueue.clear();
223}
224
225template<class Impl>
226void
227LSQUnit<Impl>::clearSQ()
228{
229    storeQueue.clear();
230}
231
232template<class Impl>
233void
234LSQUnit<Impl>::switchOut()
235{
236    switchedOut = true;
237    for (int i = 0; i < loadQueue.size(); ++i) {
238        assert(!loadQueue[i]);
239        loadQueue[i] = NULL;
240    }
241
242    assert(storesToWB == 0);
243}
244
245template<class Impl>
246void
247LSQUnit<Impl>::takeOverFrom()
248{
249    switchedOut = false;
250    loads = stores = storesToWB = 0;
251
252    loadHead = loadTail = 0;
253
254    storeHead = storeWBIdx = storeTail = 0;
255
256    usedPorts = 0;
257
258    memDepViolator = NULL;
259
260    blockedLoadSeqNum = 0;
261
262    stalled = false;
263    isLoadBlocked = false;
264    loadBlockedHandled = false;
265}
266
267template<class Impl>
268void
269LSQUnit<Impl>::resizeLQ(unsigned size)
270{
271    unsigned size_plus_sentinel = size + 1;
272    assert(size_plus_sentinel >= LQEntries);
273
274    if (size_plus_sentinel > LQEntries) {
275        while (size_plus_sentinel > loadQueue.size()) {
276            DynInstPtr dummy;
277            loadQueue.push_back(dummy);
278            LQEntries++;
279        }
280    } else {
281        LQEntries = size_plus_sentinel;
282    }
283
284}
285
286template<class Impl>
287void
288LSQUnit<Impl>::resizeSQ(unsigned size)
289{
290    unsigned size_plus_sentinel = size + 1;
291    if (size_plus_sentinel > SQEntries) {
292        while (size_plus_sentinel > storeQueue.size()) {
293            SQEntry dummy;
294            storeQueue.push_back(dummy);
295            SQEntries++;
296        }
297    } else {
298        SQEntries = size_plus_sentinel;
299    }
300}
301
302template <class Impl>
303void
304LSQUnit<Impl>::insert(DynInstPtr &inst)
305{
306    assert(inst->isMemRef());
307
308    assert(inst->isLoad() || inst->isStore());
309
310    if (inst->isLoad()) {
311        insertLoad(inst);
312    } else {
313        insertStore(inst);
314    }
315
316    inst->setInLSQ();
317}
318
319template <class Impl>
320void
321LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
322{
323    assert((loadTail + 1) % LQEntries != loadHead);
324    assert(loads < LQEntries);
325
326    DPRINTF(LSQUnit, "Inserting load PC %#x, idx:%i [sn:%lli]\n",
327            load_inst->readPC(), loadTail, load_inst->seqNum);
328
329    load_inst->lqIdx = loadTail;
330
331    if (stores == 0) {
332        load_inst->sqIdx = -1;
333    } else {
334        load_inst->sqIdx = storeTail;
335    }
336
337    loadQueue[loadTail] = load_inst;
338
339    incrLdIdx(loadTail);
340
341    ++loads;
342}
343
344template <class Impl>
345void
346LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
347{
348    // Make sure it is not full before inserting an instruction.
349    assert((storeTail + 1) % SQEntries != storeHead);
350    assert(stores < SQEntries);
351
352    DPRINTF(LSQUnit, "Inserting store PC %#x, idx:%i [sn:%lli]\n",
353            store_inst->readPC(), storeTail, store_inst->seqNum);
354
355    store_inst->sqIdx = storeTail;
356    store_inst->lqIdx = loadTail;
357
358    storeQueue[storeTail] = SQEntry(store_inst);
359
360    incrStIdx(storeTail);
361
362    ++stores;
363}
364
365template <class Impl>
366typename Impl::DynInstPtr
367LSQUnit<Impl>::getMemDepViolator()
368{
369    DynInstPtr temp = memDepViolator;
370
371    memDepViolator = NULL;
372
373    return temp;
374}
375
376template <class Impl>
377unsigned
378LSQUnit<Impl>::numFreeEntries()
379{
380    unsigned free_lq_entries = LQEntries - loads;
381    unsigned free_sq_entries = SQEntries - stores;
382
383    // Both the LQ and SQ entries have an extra dummy entry to differentiate
384    // empty/full conditions.  Subtract 1 from the free entries.
385    if (free_lq_entries < free_sq_entries) {
386        return free_lq_entries - 1;
387    } else {
388        return free_sq_entries - 1;
389    }
390}
391
392template <class Impl>
393int
394LSQUnit<Impl>::numLoadsReady()
395{
396    int load_idx = loadHead;
397    int retval = 0;
398
399    while (load_idx != loadTail) {
400        assert(loadQueue[load_idx]);
401
402        if (loadQueue[load_idx]->readyToIssue()) {
403            ++retval;
404        }
405    }
406
407    return retval;
408}
409
410template <class Impl>
411Fault
412LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
413{
414    using namespace TheISA;
415    // Execute a specific load.
416    Fault load_fault = NoFault;
417
418    DPRINTF(LSQUnit, "Executing load PC %#x, [sn:%lli]\n",
419            inst->readPC(),inst->seqNum);
420
421    assert(!inst->isSquashed());
422
423    load_fault = inst->initiateAcc();
424
425    // If the instruction faulted, then we need to send it along to commit
426    // without the instruction completing.
427    if (load_fault != NoFault) {
428        // Send this instruction to commit, also make sure iew stage
429        // realizes there is activity.
430        // Mark it as executed unless it is an uncached load that
431        // needs to hit the head of commit.
432        if (!(inst->hasRequest() && inst->uncacheable()) ||
433            inst->isAtCommit()) {
434            inst->setExecuted();
435        }
436        iewStage->instToCommit(inst);
437        iewStage->activityThisCycle();
438    } else if (!loadBlocked()) {
439        assert(inst->effAddrValid);
440        int load_idx = inst->lqIdx;
441        incrLdIdx(load_idx);
442        while (load_idx != loadTail) {
443            // Really only need to check loads that have actually executed
444
445            // @todo: For now this is extra conservative, detecting a
446            // violation if the addresses match assuming all accesses
447            // are quad word accesses.
448
449            // @todo: Fix this, magic number being used here
450            if (loadQueue[load_idx]->effAddrValid &&
451                (loadQueue[load_idx]->effAddr >> 8) ==
452                (inst->effAddr >> 8)) {
453                // A load incorrectly passed this load.  Squash and refetch.
454                // For now return a fault to show that it was unsuccessful.
455                DynInstPtr violator = loadQueue[load_idx];
456                if (!memDepViolator ||
457                    (violator->seqNum < memDepViolator->seqNum)) {
458                    memDepViolator = violator;
459                } else {
460                    break;
461                }
462
463                ++lsqMemOrderViolation;
464
465                return genMachineCheckFault();
466            }
467
468            incrLdIdx(load_idx);
469        }
470    }
471
472    return load_fault;
473}
474
475template <class Impl>
476Fault
477LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
478{
479    using namespace TheISA;
480    // Make sure that a store exists.
481    assert(stores != 0);
482
483    int store_idx = store_inst->sqIdx;
484
485    DPRINTF(LSQUnit, "Executing store PC %#x [sn:%lli]\n",
486            store_inst->readPC(), store_inst->seqNum);
487
488    assert(!store_inst->isSquashed());
489
490    // Check the recently completed loads to see if any match this store's
491    // address.  If so, then we have a memory ordering violation.
492    int load_idx = store_inst->lqIdx;
493
494    Fault store_fault = store_inst->initiateAcc();
495
496    if (storeQueue[store_idx].size == 0) {
497        DPRINTF(LSQUnit,"Fault on Store PC %#x, [sn:%lli],Size = 0\n",
498                store_inst->readPC(),store_inst->seqNum);
499
500        return store_fault;
501    }
502
503    assert(store_fault == NoFault);
504
505    if (store_inst->isStoreConditional()) {
506        // Store conditionals need to set themselves as able to
507        // writeback if we haven't had a fault by here.
508        storeQueue[store_idx].canWB = true;
509
510        ++storesToWB;
511    }
512
513    assert(store_inst->effAddrValid);
514    while (load_idx != loadTail) {
515        // Really only need to check loads that have actually executed
516        // It's safe to check all loads because effAddr is set to
517        // InvalAddr when the dyn inst is created.
518
519        // @todo: For now this is extra conservative, detecting a
520        // violation if the addresses match assuming all accesses
521        // are quad word accesses.
522
523        // @todo: Fix this, magic number being used here
524        if (loadQueue[load_idx]->effAddrValid &&
525            (loadQueue[load_idx]->effAddr >> 8) ==
526            (store_inst->effAddr >> 8)) {
527            // A load incorrectly passed this store.  Squash and refetch.
528            // For now return a fault to show that it was unsuccessful.
529            DynInstPtr violator = loadQueue[load_idx];
530            if (!memDepViolator ||
531                (violator->seqNum < memDepViolator->seqNum)) {
532                memDepViolator = violator;
533            } else {
534                break;
535            }
536
537            ++lsqMemOrderViolation;
538
539            return genMachineCheckFault();
540        }
541
542        incrLdIdx(load_idx);
543    }
544
545    return store_fault;
546}
547
548template <class Impl>
549void
550LSQUnit<Impl>::commitLoad()
551{
552    assert(loadQueue[loadHead]);
553
554    DPRINTF(LSQUnit, "Committing head load instruction, PC %#x\n",
555            loadQueue[loadHead]->readPC());
556
557    loadQueue[loadHead] = NULL;
558
559    incrLdIdx(loadHead);
560
561    --loads;
562}
563
564template <class Impl>
565void
566LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
567{
568    assert(loads == 0 || loadQueue[loadHead]);
569
570    while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
571        commitLoad();
572    }
573}
574
575template <class Impl>
576void
577LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
578{
579    assert(stores == 0 || storeQueue[storeHead].inst);
580
581    int store_idx = storeHead;
582
583    while (store_idx != storeTail) {
584        assert(storeQueue[store_idx].inst);
585        // Mark any stores that are now committed and have not yet
586        // been marked as able to write back.
587        if (!storeQueue[store_idx].canWB) {
588            if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
589                break;
590            }
591            DPRINTF(LSQUnit, "Marking store as able to write back, PC "
592                    "%#x [sn:%lli]\n",
593                    storeQueue[store_idx].inst->readPC(),
594                    storeQueue[store_idx].inst->seqNum);
595
596            storeQueue[store_idx].canWB = true;
597
598            ++storesToWB;
599        }
600
601        incrStIdx(store_idx);
602    }
603}
604
605template <class Impl>
606void
607LSQUnit<Impl>::writebackStores()
608{
609    while (storesToWB > 0 &&
610           storeWBIdx != storeTail &&
611           storeQueue[storeWBIdx].inst &&
612           storeQueue[storeWBIdx].canWB &&
613           usedPorts < cachePorts) {
614
615        if (isStoreBlocked || lsq->cacheBlocked()) {
616            DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
617                    " is blocked!\n");
618            break;
619        }
620
621        // Store didn't write any data so no need to write it back to
622        // memory.
623        if (storeQueue[storeWBIdx].size == 0) {
624            completeStore(storeWBIdx);
625
626            incrStIdx(storeWBIdx);
627
628            continue;
629        }
630
631        ++usedPorts;
632
633        if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
634            incrStIdx(storeWBIdx);
635
636            continue;
637        }
638
639        assert(storeQueue[storeWBIdx].req);
640        assert(!storeQueue[storeWBIdx].committed);
641
642        DynInstPtr inst = storeQueue[storeWBIdx].inst;
643
644        Request *req = storeQueue[storeWBIdx].req;
645        storeQueue[storeWBIdx].committed = true;
646
647        assert(!inst->memData);
648        inst->memData = new uint8_t[64];
649
650        memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
651
652        MemCmd command =
653            req->isSwap() ? MemCmd::SwapReq :
654            (req->isLocked() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
655        PacketPtr data_pkt = new Packet(req, command,
656                                        Packet::Broadcast);
657        data_pkt->dataStatic(inst->memData);
658
659        LSQSenderState *state = new LSQSenderState;
660        state->isLoad = false;
661        state->idx = storeWBIdx;
662        state->inst = inst;
663        data_pkt->senderState = state;
664
665        DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%#x "
666                "to Addr:%#x, data:%#x [sn:%lli]\n",
667                storeWBIdx, inst->readPC(),
668                req->getPaddr(), (int)*(inst->memData),
669                inst->seqNum);
670
671        // @todo: Remove this SC hack once the memory system handles it.
672        if (inst->isStoreConditional()) {
673            // Disable recording the result temporarily.  Writing to
674            // misc regs normally updates the result, but this is not
675            // the desired behavior when handling store conditionals.
676            inst->recordResult = false;
677            bool success = TheISA::handleLockedWrite(inst.get(), req);
678            inst->recordResult = true;
679
680            if (!success) {
681                // Instantly complete this store.
682                DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
683                        "Instantly completing it.\n",
684                        inst->seqNum);
685                WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
686                wb->schedule(curTick + 1);
687                completeStore(storeWBIdx);
688                incrStIdx(storeWBIdx);
689                continue;
690            }
691        } else {
692            // Non-store conditionals do not need a writeback.
693            state->noWB = true;
694        }
695
696        if (!dcachePort->sendTiming(data_pkt)) {
697            // Need to handle becoming blocked on a store.
698            DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
699                    "retry later\n",
700                    inst->seqNum);
701            isStoreBlocked = true;
702            ++lsqCacheBlocked;
703            assert(retryPkt == NULL);
704            retryPkt = data_pkt;
705            lsq->setRetryTid(lsqID);
706        } else {
707            storePostSend(data_pkt);
708        }
709    }
710
711    // Not sure this should set it to 0.
712    usedPorts = 0;
713
714    assert(stores >= 0 && storesToWB >= 0);
715}
716
717/*template <class Impl>
718void
719LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
720{
721    list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
722                                              mshrSeqNums.end(),
723                                              seqNum);
724
725    if (mshr_it != mshrSeqNums.end()) {
726        mshrSeqNums.erase(mshr_it);
727        DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
728    }
729}*/
730
731template <class Impl>
732void
733LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
734{
735    DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
736            "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
737
738    int load_idx = loadTail;
739    decrLdIdx(load_idx);
740
741    while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
742        DPRINTF(LSQUnit,"Load Instruction PC %#x squashed, "
743                "[sn:%lli]\n",
744                loadQueue[load_idx]->readPC(),
745                loadQueue[load_idx]->seqNum);
746
747        if (isStalled() && load_idx == stallingLoadIdx) {
748            stalled = false;
749            stallingStoreIsn = 0;
750            stallingLoadIdx = 0;
751        }
752
753        // Clear the smart pointer to make sure it is decremented.
754        loadQueue[load_idx]->setSquashed();
755        loadQueue[load_idx] = NULL;
756        --loads;
757
758        // Inefficient!
759        loadTail = load_idx;
760
761        decrLdIdx(load_idx);
762        ++lsqSquashedLoads;
763    }
764
765    if (isLoadBlocked) {
766        if (squashed_num < blockedLoadSeqNum) {
767            isLoadBlocked = false;
768            loadBlockedHandled = false;
769            blockedLoadSeqNum = 0;
770        }
771    }
772
773    if (memDepViolator && squashed_num < memDepViolator->seqNum) {
774        memDepViolator = NULL;
775    }
776
777    int store_idx = storeTail;
778    decrStIdx(store_idx);
779
780    while (stores != 0 &&
781           storeQueue[store_idx].inst->seqNum > squashed_num) {
782        // Instructions marked as can WB are already committed.
783        if (storeQueue[store_idx].canWB) {
784            break;
785        }
786
787        DPRINTF(LSQUnit,"Store Instruction PC %#x squashed, "
788                "idx:%i [sn:%lli]\n",
789                storeQueue[store_idx].inst->readPC(),
790                store_idx, storeQueue[store_idx].inst->seqNum);
791
792        // I don't think this can happen.  It should have been cleared
793        // by the stalling load.
794        if (isStalled() &&
795            storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
796            panic("Is stalled should have been cleared by stalling load!\n");
797            stalled = false;
798            stallingStoreIsn = 0;
799        }
800
801        // Clear the smart pointer to make sure it is decremented.
802        storeQueue[store_idx].inst->setSquashed();
803        storeQueue[store_idx].inst = NULL;
804        storeQueue[store_idx].canWB = 0;
805
806        // Must delete request now that it wasn't handed off to
807        // memory.  This is quite ugly.  @todo: Figure out the proper
808        // place to really handle request deletes.
809        delete storeQueue[store_idx].req;
810
811        storeQueue[store_idx].req = NULL;
812        --stores;
813
814        // Inefficient!
815        storeTail = store_idx;
816
817        decrStIdx(store_idx);
818        ++lsqSquashedStores;
819    }
820}
821
822template <class Impl>
823void
824LSQUnit<Impl>::storePostSend(PacketPtr pkt)
825{
826    if (isStalled() &&
827        storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
828        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
829                "load idx:%i\n",
830                stallingStoreIsn, stallingLoadIdx);
831        stalled = false;
832        stallingStoreIsn = 0;
833        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
834    }
835
836    if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
837        // The store is basically completed at this time. This
838        // only works so long as the checker doesn't try to
839        // verify the value in memory for stores.
840        storeQueue[storeWBIdx].inst->setCompleted();
841#if USE_CHECKER
842        if (cpu->checker) {
843            cpu->checker->verify(storeQueue[storeWBIdx].inst);
844        }
845#endif
846    }
847
848    incrStIdx(storeWBIdx);
849}
850
851template <class Impl>
852void
853LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
854{
855    iewStage->wakeCPU();
856
857    // Squashed instructions do not need to complete their access.
858    if (inst->isSquashed()) {
859        iewStage->decrWb(inst->seqNum);
860        assert(!inst->isStore());
861        ++lsqIgnoredResponses;
862        return;
863    }
864
865    if (!inst->isExecuted()) {
866        inst->setExecuted();
867
868        // Complete access to copy data to proper place.
869        inst->completeAcc(pkt);
870    }
871
872    // Need to insert instruction into queue to commit
873    iewStage->instToCommit(inst);
874
875    iewStage->activityThisCycle();
876}
877
878template <class Impl>
879void
880LSQUnit<Impl>::completeStore(int store_idx)
881{
882    assert(storeQueue[store_idx].inst);
883    storeQueue[store_idx].completed = true;
884    --storesToWB;
885    // A bit conservative because a store completion may not free up entries,
886    // but hopefully avoids two store completions in one cycle from making
887    // the CPU tick twice.
888    cpu->wakeCPU();
889    cpu->activityThisCycle();
890
891    if (store_idx == storeHead) {
892        do {
893            incrStIdx(storeHead);
894
895            --stores;
896        } while (storeQueue[storeHead].completed &&
897                 storeHead != storeTail);
898
899        iewStage->updateLSQNextCycle = true;
900    }
901
902    DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
903            "idx:%i\n",
904            storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
905
906    if (isStalled() &&
907        storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
908        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
909                "load idx:%i\n",
910                stallingStoreIsn, stallingLoadIdx);
911        stalled = false;
912        stallingStoreIsn = 0;
913        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
914    }
915
916    storeQueue[store_idx].inst->setCompleted();
917
918    // Tell the checker we've completed this instruction.  Some stores
919    // may get reported twice to the checker, but the checker can
920    // handle that case.
921#if USE_CHECKER
922    if (cpu->checker) {
923        cpu->checker->verify(storeQueue[store_idx].inst);
924    }
925#endif
926}
927
928template <class Impl>
929void
930LSQUnit<Impl>::recvRetry()
931{
932    if (isStoreBlocked) {
933        DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
934        assert(retryPkt != NULL);
935
936        if (dcachePort->sendTiming(retryPkt)) {
937            storePostSend(retryPkt);
938            retryPkt = NULL;
939            isStoreBlocked = false;
940            lsq->setRetryTid(-1);
941        } else {
942            // Still blocked!
943            ++lsqCacheBlocked;
944            lsq->setRetryTid(lsqID);
945        }
946    } else if (isLoadBlocked) {
947        DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
948                "no need to resend packet.\n");
949    } else {
950        DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
951    }
952}
953
954template <class Impl>
955inline void
956LSQUnit<Impl>::incrStIdx(int &store_idx)
957{
958    if (++store_idx >= SQEntries)
959        store_idx = 0;
960}
961
962template <class Impl>
963inline void
964LSQUnit<Impl>::decrStIdx(int &store_idx)
965{
966    if (--store_idx < 0)
967        store_idx += SQEntries;
968}
969
970template <class Impl>
971inline void
972LSQUnit<Impl>::incrLdIdx(int &load_idx)
973{
974    if (++load_idx >= LQEntries)
975        load_idx = 0;
976}
977
978template <class Impl>
979inline void
980LSQUnit<Impl>::decrLdIdx(int &load_idx)
981{
982    if (--load_idx < 0)
983        load_idx += LQEntries;
984}
985
986template <class Impl>
987void
988LSQUnit<Impl>::dumpInsts()
989{
990    cprintf("Load store queue: Dumping instructions.\n");
991    cprintf("Load queue size: %i\n", loads);
992    cprintf("Load queue: ");
993
994    int load_idx = loadHead;
995
996    while (load_idx != loadTail && loadQueue[load_idx]) {
997        cprintf("%#x ", loadQueue[load_idx]->readPC());
998
999        incrLdIdx(load_idx);
1000    }
1001
1002    cprintf("Store queue size: %i\n", stores);
1003    cprintf("Store queue: ");
1004
1005    int store_idx = storeHead;
1006
1007    while (store_idx != storeTail && storeQueue[store_idx].inst) {
1008        cprintf("%#x ", storeQueue[store_idx].inst->readPC());
1009
1010        incrStIdx(store_idx);
1011    }
1012
1013    cprintf("\n");
1014}
1015