lsq_unit_impl.hh revision 8587
1/*
2 * Copyright (c) 2010 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2004-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Kevin Lim
41 *          Korey Sewell
42 */
43
44#include "arch/locked_mem.hh"
45#include "base/str.hh"
46#include "config/the_isa.hh"
47#include "config/use_checker.hh"
48#include "cpu/o3/lsq.hh"
49#include "cpu/o3/lsq_unit.hh"
50#include "debug/Activity.hh"
51#include "debug/IEW.hh"
52#include "debug/LSQUnit.hh"
53#include "mem/packet.hh"
54#include "mem/request.hh"
55
56#if USE_CHECKER
57#include "cpu/checker/cpu.hh"
58#endif
59
60template<class Impl>
61LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
62                                              LSQUnit *lsq_ptr)
63    : Event(Default_Pri, AutoDelete),
64      inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
65{
66}
67
68template<class Impl>
69void
70LSQUnit<Impl>::WritebackEvent::process()
71{
72    if (!lsqPtr->isSwitchedOut()) {
73        lsqPtr->writeback(inst, pkt);
74    }
75
76    if (pkt->senderState)
77        delete pkt->senderState;
78
79    delete pkt->req;
80    delete pkt;
81}
82
83template<class Impl>
84const char *
85LSQUnit<Impl>::WritebackEvent::description() const
86{
87    return "Store writeback";
88}
89
90template<class Impl>
91void
92LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
93{
94    LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
95    DynInstPtr inst = state->inst;
96    DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
97    DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
98
99    //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum);
100
101    assert(!pkt->wasNacked());
102
103    // If this is a split access, wait until all packets are received.
104    if (TheISA::HasUnalignedMemAcc && !state->complete()) {
105        delete pkt->req;
106        delete pkt;
107        return;
108    }
109
110    if (isSwitchedOut() || inst->isSquashed()) {
111        iewStage->decrWb(inst->seqNum);
112    } else {
113        if (!state->noWB) {
114            if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
115                !state->isLoad) {
116                writeback(inst, pkt);
117            } else {
118                writeback(inst, state->mainPkt);
119            }
120        }
121
122        if (inst->isStore()) {
123            completeStore(state->idx);
124        }
125    }
126
127    if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
128        delete state->mainPkt->req;
129        delete state->mainPkt;
130    }
131    delete state;
132    delete pkt->req;
133    delete pkt;
134}
135
136template <class Impl>
137LSQUnit<Impl>::LSQUnit()
138    : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
139      isStoreBlocked(false), isLoadBlocked(false),
140      loadBlockedHandled(false), hasPendingPkt(false)
141{
142}
143
144template<class Impl>
145void
146LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
147        LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
148        unsigned id)
149{
150    cpu = cpu_ptr;
151    iewStage = iew_ptr;
152
153    DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
154
155    switchedOut = false;
156
157    cacheBlockMask = 0;
158
159    lsq = lsq_ptr;
160
161    lsqID = id;
162
163    // Add 1 for the sentinel entry (they are circular queues).
164    LQEntries = maxLQEntries + 1;
165    SQEntries = maxSQEntries + 1;
166
167    loadQueue.resize(LQEntries);
168    storeQueue.resize(SQEntries);
169
170    depCheckShift = params->LSQDepCheckShift;
171    checkLoads = params->LSQCheckLoads;
172
173    loadHead = loadTail = 0;
174
175    storeHead = storeWBIdx = storeTail = 0;
176
177    usedPorts = 0;
178    cachePorts = params->cachePorts;
179
180    retryPkt = NULL;
181    memDepViolator = NULL;
182
183    blockedLoadSeqNum = 0;
184}
185
186template<class Impl>
187std::string
188LSQUnit<Impl>::name() const
189{
190    if (Impl::MaxThreads == 1) {
191        return iewStage->name() + ".lsq";
192    } else {
193        return iewStage->name() + ".lsq.thread" + to_string(lsqID);
194    }
195}
196
197template<class Impl>
198void
199LSQUnit<Impl>::regStats()
200{
201    lsqForwLoads
202        .name(name() + ".forwLoads")
203        .desc("Number of loads that had data forwarded from stores");
204
205    invAddrLoads
206        .name(name() + ".invAddrLoads")
207        .desc("Number of loads ignored due to an invalid address");
208
209    lsqSquashedLoads
210        .name(name() + ".squashedLoads")
211        .desc("Number of loads squashed");
212
213    lsqIgnoredResponses
214        .name(name() + ".ignoredResponses")
215        .desc("Number of memory responses ignored because the instruction is squashed");
216
217    lsqMemOrderViolation
218        .name(name() + ".memOrderViolation")
219        .desc("Number of memory ordering violations");
220
221    lsqSquashedStores
222        .name(name() + ".squashedStores")
223        .desc("Number of stores squashed");
224
225    invAddrSwpfs
226        .name(name() + ".invAddrSwpfs")
227        .desc("Number of software prefetches ignored due to an invalid address");
228
229    lsqBlockedLoads
230        .name(name() + ".blockedLoads")
231        .desc("Number of blocked loads due to partial load-store forwarding");
232
233    lsqRescheduledLoads
234        .name(name() + ".rescheduledLoads")
235        .desc("Number of loads that were rescheduled");
236
237    lsqCacheBlocked
238        .name(name() + ".cacheBlocked")
239        .desc("Number of times an access to memory failed due to the cache being blocked");
240}
241
242template<class Impl>
243void
244LSQUnit<Impl>::setDcachePort(Port *dcache_port)
245{
246    dcachePort = dcache_port;
247
248#if USE_CHECKER
249    if (cpu->checker) {
250        cpu->checker->setDcachePort(dcachePort);
251    }
252#endif
253}
254
255template<class Impl>
256void
257LSQUnit<Impl>::clearLQ()
258{
259    loadQueue.clear();
260}
261
262template<class Impl>
263void
264LSQUnit<Impl>::clearSQ()
265{
266    storeQueue.clear();
267}
268
269template<class Impl>
270void
271LSQUnit<Impl>::switchOut()
272{
273    switchedOut = true;
274    for (int i = 0; i < loadQueue.size(); ++i) {
275        assert(!loadQueue[i]);
276        loadQueue[i] = NULL;
277    }
278
279    assert(storesToWB == 0);
280}
281
282template<class Impl>
283void
284LSQUnit<Impl>::takeOverFrom()
285{
286    switchedOut = false;
287    loads = stores = storesToWB = 0;
288
289    loadHead = loadTail = 0;
290
291    storeHead = storeWBIdx = storeTail = 0;
292
293    usedPorts = 0;
294
295    memDepViolator = NULL;
296
297    blockedLoadSeqNum = 0;
298
299    stalled = false;
300    isLoadBlocked = false;
301    loadBlockedHandled = false;
302
303    // Just incase the memory system changed out from under us
304    cacheBlockMask = 0;
305}
306
307template<class Impl>
308void
309LSQUnit<Impl>::resizeLQ(unsigned size)
310{
311    unsigned size_plus_sentinel = size + 1;
312    assert(size_plus_sentinel >= LQEntries);
313
314    if (size_plus_sentinel > LQEntries) {
315        while (size_plus_sentinel > loadQueue.size()) {
316            DynInstPtr dummy;
317            loadQueue.push_back(dummy);
318            LQEntries++;
319        }
320    } else {
321        LQEntries = size_plus_sentinel;
322    }
323
324}
325
326template<class Impl>
327void
328LSQUnit<Impl>::resizeSQ(unsigned size)
329{
330    unsigned size_plus_sentinel = size + 1;
331    if (size_plus_sentinel > SQEntries) {
332        while (size_plus_sentinel > storeQueue.size()) {
333            SQEntry dummy;
334            storeQueue.push_back(dummy);
335            SQEntries++;
336        }
337    } else {
338        SQEntries = size_plus_sentinel;
339    }
340}
341
342template <class Impl>
343void
344LSQUnit<Impl>::insert(DynInstPtr &inst)
345{
346    assert(inst->isMemRef());
347
348    assert(inst->isLoad() || inst->isStore());
349
350    if (inst->isLoad()) {
351        insertLoad(inst);
352    } else {
353        insertStore(inst);
354    }
355
356    inst->setInLSQ();
357}
358
359template <class Impl>
360void
361LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
362{
363    assert((loadTail + 1) % LQEntries != loadHead);
364    assert(loads < LQEntries);
365
366    DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
367            load_inst->pcState(), loadTail, load_inst->seqNum);
368
369    load_inst->lqIdx = loadTail;
370
371    if (stores == 0) {
372        load_inst->sqIdx = -1;
373    } else {
374        load_inst->sqIdx = storeTail;
375    }
376
377    loadQueue[loadTail] = load_inst;
378
379    incrLdIdx(loadTail);
380
381    ++loads;
382}
383
384template <class Impl>
385void
386LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
387{
388    // Make sure it is not full before inserting an instruction.
389    assert((storeTail + 1) % SQEntries != storeHead);
390    assert(stores < SQEntries);
391
392    DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
393            store_inst->pcState(), storeTail, store_inst->seqNum);
394
395    store_inst->sqIdx = storeTail;
396    store_inst->lqIdx = loadTail;
397
398    storeQueue[storeTail] = SQEntry(store_inst);
399
400    incrStIdx(storeTail);
401
402    ++stores;
403}
404
405template <class Impl>
406typename Impl::DynInstPtr
407LSQUnit<Impl>::getMemDepViolator()
408{
409    DynInstPtr temp = memDepViolator;
410
411    memDepViolator = NULL;
412
413    return temp;
414}
415
416template <class Impl>
417unsigned
418LSQUnit<Impl>::numFreeEntries()
419{
420    unsigned free_lq_entries = LQEntries - loads;
421    unsigned free_sq_entries = SQEntries - stores;
422
423    // Both the LQ and SQ entries have an extra dummy entry to differentiate
424    // empty/full conditions.  Subtract 1 from the free entries.
425    if (free_lq_entries < free_sq_entries) {
426        return free_lq_entries - 1;
427    } else {
428        return free_sq_entries - 1;
429    }
430}
431
432template <class Impl>
433int
434LSQUnit<Impl>::numLoadsReady()
435{
436    int load_idx = loadHead;
437    int retval = 0;
438
439    while (load_idx != loadTail) {
440        assert(loadQueue[load_idx]);
441
442        if (loadQueue[load_idx]->readyToIssue()) {
443            ++retval;
444        }
445    }
446
447    return retval;
448}
449
450template <class Impl>
451void
452LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
453{
454    int load_idx = loadHead;
455
456    if (!cacheBlockMask) {
457        assert(dcachePort);
458        Addr bs = dcachePort->peerBlockSize();
459
460        // Make sure we actually got a size
461        assert(bs != 0);
462
463        cacheBlockMask = ~(bs - 1);
464    }
465
466    // If this is the only load in the LSQ we don't care
467    if (load_idx == loadTail)
468        return;
469    incrLdIdx(load_idx);
470
471    DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
472    Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
473    while (load_idx != loadTail) {
474        DynInstPtr ld_inst = loadQueue[load_idx];
475
476        if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
477            incrLdIdx(load_idx);
478            continue;
479        }
480
481        Addr load_addr = ld_inst->physEffAddr & cacheBlockMask;
482        DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
483                    ld_inst->seqNum, load_addr, invalidate_addr);
484
485        if (load_addr == invalidate_addr) {
486            if (ld_inst->possibleLoadViolation) {
487                DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
488                        ld_inst->physEffAddr, pkt->getAddr(), ld_inst->seqNum);
489
490                // Mark the load for re-execution
491                ld_inst->fault = new ReExec;
492            } else {
493                // If a older load checks this and it's true
494                // then we might have missed the snoop
495                // in which case we need to invalidate to be sure
496                ld_inst->hitExternalSnoop = true;
497            }
498        }
499        incrLdIdx(load_idx);
500    }
501    return;
502}
503
504template <class Impl>
505Fault
506LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
507{
508    Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
509    Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
510
511    /** @todo in theory you only need to check an instruction that has executed
512     * however, there isn't a good way in the pipeline at the moment to check
513     * all instructions that will execute before the store writes back. Thus,
514     * like the implementation that came before it, we're overly conservative.
515     */
516    while (load_idx != loadTail) {
517        DynInstPtr ld_inst = loadQueue[load_idx];
518        if (!ld_inst->effAddrValid || ld_inst->uncacheable()) {
519            incrLdIdx(load_idx);
520            continue;
521        }
522
523        Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
524        Addr ld_eff_addr2 =
525            (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
526
527        if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
528            if (inst->isLoad()) {
529                // If this load is to the same block as an external snoop
530                // invalidate that we've observed then the load needs to be
531                // squashed as it could have newer data
532                if (ld_inst->hitExternalSnoop) {
533                    if (!memDepViolator ||
534                            ld_inst->seqNum < memDepViolator->seqNum) {
535                        DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
536                                " and [sn:%lli] at address %#x\n", inst->seqNum,
537                                ld_inst->seqNum, ld_eff_addr1);
538                        memDepViolator = ld_inst;
539
540                        ++lsqMemOrderViolation;
541
542                        return TheISA::genMachineCheckFault();
543                    }
544                }
545
546                // Otherwise, mark the load has a possible load violation
547                // and if we see a snoop before it's commited, we need to squash
548                ld_inst->possibleLoadViolation = true;
549                DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x"
550                        " between instructions [sn:%lli] and [sn:%lli]\n",
551                        inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
552            } else {
553                // A load/store incorrectly passed this store.
554                // Check if we already have a violator, or if it's newer
555                // squash and refetch.
556                if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
557                    break;
558
559                DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and [sn:%lli]"
560                        " at address %#x\n", inst->seqNum, ld_inst->seqNum,
561                        ld_eff_addr1);
562                memDepViolator = ld_inst;
563
564                ++lsqMemOrderViolation;
565
566                return TheISA::genMachineCheckFault();
567            }
568        }
569
570        incrLdIdx(load_idx);
571    }
572    return NoFault;
573}
574
575
576
577
578template <class Impl>
579Fault
580LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
581{
582    using namespace TheISA;
583    // Execute a specific load.
584    Fault load_fault = NoFault;
585
586    DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
587            inst->pcState(), inst->seqNum);
588
589    assert(!inst->isSquashed());
590
591    load_fault = inst->initiateAcc();
592
593    if (inst->isTranslationDelayed() &&
594        load_fault == NoFault)
595        return load_fault;
596
597    // If the instruction faulted or predicated false, then we need to send it
598    // along to commit without the instruction completing.
599    if (load_fault != NoFault || inst->readPredicate() == false) {
600        // Send this instruction to commit, also make sure iew stage
601        // realizes there is activity.
602        // Mark it as executed unless it is an uncached load that
603        // needs to hit the head of commit.
604        if (inst->readPredicate() == false)
605            inst->forwardOldRegs();
606        DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
607                inst->seqNum,
608                (load_fault != NoFault ? "fault" : "predication"));
609        if (!(inst->hasRequest() && inst->uncacheable()) ||
610            inst->isAtCommit()) {
611            inst->setExecuted();
612        }
613        iewStage->instToCommit(inst);
614        iewStage->activityThisCycle();
615    } else if (!loadBlocked()) {
616        assert(inst->effAddrValid);
617        int load_idx = inst->lqIdx;
618        incrLdIdx(load_idx);
619
620        if (checkLoads)
621            return checkViolations(load_idx, inst);
622    }
623
624    return load_fault;
625}
626
627template <class Impl>
628Fault
629LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
630{
631    using namespace TheISA;
632    // Make sure that a store exists.
633    assert(stores != 0);
634
635    int store_idx = store_inst->sqIdx;
636
637    DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
638            store_inst->pcState(), store_inst->seqNum);
639
640    assert(!store_inst->isSquashed());
641
642    // Check the recently completed loads to see if any match this store's
643    // address.  If so, then we have a memory ordering violation.
644    int load_idx = store_inst->lqIdx;
645
646    Fault store_fault = store_inst->initiateAcc();
647
648    if (store_inst->isTranslationDelayed() &&
649        store_fault == NoFault)
650        return store_fault;
651
652    if (store_inst->readPredicate() == false)
653        store_inst->forwardOldRegs();
654
655    if (storeQueue[store_idx].size == 0) {
656        DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
657                store_inst->pcState(), store_inst->seqNum);
658
659        return store_fault;
660    } else if (store_inst->readPredicate() == false) {
661        DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
662                store_inst->seqNum);
663        return store_fault;
664    }
665
666    assert(store_fault == NoFault);
667
668    if (store_inst->isStoreConditional()) {
669        // Store conditionals need to set themselves as able to
670        // writeback if we haven't had a fault by here.
671        storeQueue[store_idx].canWB = true;
672
673        ++storesToWB;
674    }
675
676    return checkViolations(load_idx, store_inst);
677
678}
679
680template <class Impl>
681void
682LSQUnit<Impl>::commitLoad()
683{
684    assert(loadQueue[loadHead]);
685
686    DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
687            loadQueue[loadHead]->pcState());
688
689    loadQueue[loadHead] = NULL;
690
691    incrLdIdx(loadHead);
692
693    --loads;
694}
695
696template <class Impl>
697void
698LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
699{
700    assert(loads == 0 || loadQueue[loadHead]);
701
702    while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
703        commitLoad();
704    }
705}
706
707template <class Impl>
708void
709LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
710{
711    assert(stores == 0 || storeQueue[storeHead].inst);
712
713    int store_idx = storeHead;
714
715    while (store_idx != storeTail) {
716        assert(storeQueue[store_idx].inst);
717        // Mark any stores that are now committed and have not yet
718        // been marked as able to write back.
719        if (!storeQueue[store_idx].canWB) {
720            if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
721                break;
722            }
723            DPRINTF(LSQUnit, "Marking store as able to write back, PC "
724                    "%s [sn:%lli]\n",
725                    storeQueue[store_idx].inst->pcState(),
726                    storeQueue[store_idx].inst->seqNum);
727
728            storeQueue[store_idx].canWB = true;
729
730            ++storesToWB;
731        }
732
733        incrStIdx(store_idx);
734    }
735}
736
737template <class Impl>
738void
739LSQUnit<Impl>::writebackPendingStore()
740{
741    if (hasPendingPkt) {
742        assert(pendingPkt != NULL);
743
744        // If the cache is blocked, this will store the packet for retry.
745        if (sendStore(pendingPkt)) {
746            storePostSend(pendingPkt);
747        }
748        pendingPkt = NULL;
749        hasPendingPkt = false;
750    }
751}
752
753template <class Impl>
754void
755LSQUnit<Impl>::writebackStores()
756{
757    // First writeback the second packet from any split store that didn't
758    // complete last cycle because there weren't enough cache ports available.
759    if (TheISA::HasUnalignedMemAcc) {
760        writebackPendingStore();
761    }
762
763    while (storesToWB > 0 &&
764           storeWBIdx != storeTail &&
765           storeQueue[storeWBIdx].inst &&
766           storeQueue[storeWBIdx].canWB &&
767           usedPorts < cachePorts) {
768
769        if (isStoreBlocked || lsq->cacheBlocked()) {
770            DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
771                    " is blocked!\n");
772            break;
773        }
774
775        // Store didn't write any data so no need to write it back to
776        // memory.
777        if (storeQueue[storeWBIdx].size == 0) {
778            completeStore(storeWBIdx);
779
780            incrStIdx(storeWBIdx);
781
782            continue;
783        }
784
785        ++usedPorts;
786
787        if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
788            incrStIdx(storeWBIdx);
789
790            continue;
791        }
792
793        assert(storeQueue[storeWBIdx].req);
794        assert(!storeQueue[storeWBIdx].committed);
795
796        if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
797            assert(storeQueue[storeWBIdx].sreqLow);
798            assert(storeQueue[storeWBIdx].sreqHigh);
799        }
800
801        DynInstPtr inst = storeQueue[storeWBIdx].inst;
802
803        Request *req = storeQueue[storeWBIdx].req;
804        RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
805        RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
806
807        storeQueue[storeWBIdx].committed = true;
808
809        assert(!inst->memData);
810        inst->memData = new uint8_t[64];
811
812        memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
813
814        MemCmd command =
815            req->isSwap() ? MemCmd::SwapReq :
816            (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq);
817        PacketPtr data_pkt;
818        PacketPtr snd_data_pkt = NULL;
819
820        LSQSenderState *state = new LSQSenderState;
821        state->isLoad = false;
822        state->idx = storeWBIdx;
823        state->inst = inst;
824
825        if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
826
827            // Build a single data packet if the store isn't split.
828            data_pkt = new Packet(req, command, Packet::Broadcast);
829            data_pkt->dataStatic(inst->memData);
830            data_pkt->senderState = state;
831        } else {
832            // Create two packets if the store is split in two.
833            data_pkt = new Packet(sreqLow, command, Packet::Broadcast);
834            snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast);
835
836            data_pkt->dataStatic(inst->memData);
837            snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
838
839            data_pkt->senderState = state;
840            snd_data_pkt->senderState = state;
841
842            state->isSplit = true;
843            state->outstanding = 2;
844
845            // Can delete the main request now.
846            delete req;
847            req = sreqLow;
848        }
849
850        DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
851                "to Addr:%#x, data:%#x [sn:%lli]\n",
852                storeWBIdx, inst->pcState(),
853                req->getPaddr(), (int)*(inst->memData),
854                inst->seqNum);
855
856        // @todo: Remove this SC hack once the memory system handles it.
857        if (inst->isStoreConditional()) {
858            assert(!storeQueue[storeWBIdx].isSplit);
859            // Disable recording the result temporarily.  Writing to
860            // misc regs normally updates the result, but this is not
861            // the desired behavior when handling store conditionals.
862            inst->recordResult = false;
863            bool success = TheISA::handleLockedWrite(inst.get(), req);
864            inst->recordResult = true;
865
866            if (!success) {
867                // Instantly complete this store.
868                DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
869                        "Instantly completing it.\n",
870                        inst->seqNum);
871                WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
872                cpu->schedule(wb, curTick() + 1);
873                completeStore(storeWBIdx);
874                incrStIdx(storeWBIdx);
875                continue;
876            }
877        } else {
878            // Non-store conditionals do not need a writeback.
879            state->noWB = true;
880        }
881
882        bool split =
883            TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
884
885        ThreadContext *thread = cpu->tcBase(lsqID);
886
887        if (req->isMmappedIpr()) {
888            assert(!inst->isStoreConditional());
889            TheISA::handleIprWrite(thread, data_pkt);
890            delete data_pkt;
891            if (split) {
892                assert(snd_data_pkt->req->isMmappedIpr());
893                TheISA::handleIprWrite(thread, snd_data_pkt);
894                delete snd_data_pkt;
895                delete sreqLow;
896                delete sreqHigh;
897            }
898            delete state;
899            delete req;
900            completeStore(storeWBIdx);
901            incrStIdx(storeWBIdx);
902        } else if (!sendStore(data_pkt)) {
903            DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
904                    "retry later\n",
905                    inst->seqNum);
906
907            // Need to store the second packet, if split.
908            if (split) {
909                state->pktToSend = true;
910                state->pendingPacket = snd_data_pkt;
911            }
912        } else {
913
914            // If split, try to send the second packet too
915            if (split) {
916                assert(snd_data_pkt);
917
918                // Ensure there are enough ports to use.
919                if (usedPorts < cachePorts) {
920                    ++usedPorts;
921                    if (sendStore(snd_data_pkt)) {
922                        storePostSend(snd_data_pkt);
923                    } else {
924                        DPRINTF(IEW, "D-Cache became blocked when writing"
925                                " [sn:%lli] second packet, will retry later\n",
926                                inst->seqNum);
927                    }
928                } else {
929
930                    // Store the packet for when there's free ports.
931                    assert(pendingPkt == NULL);
932                    pendingPkt = snd_data_pkt;
933                    hasPendingPkt = true;
934                }
935            } else {
936
937                // Not a split store.
938                storePostSend(data_pkt);
939            }
940        }
941    }
942
943    // Not sure this should set it to 0.
944    usedPorts = 0;
945
946    assert(stores >= 0 && storesToWB >= 0);
947}
948
949/*template <class Impl>
950void
951LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
952{
953    list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
954                                              mshrSeqNums.end(),
955                                              seqNum);
956
957    if (mshr_it != mshrSeqNums.end()) {
958        mshrSeqNums.erase(mshr_it);
959        DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
960    }
961}*/
962
963template <class Impl>
964void
965LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
966{
967    DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
968            "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
969
970    int load_idx = loadTail;
971    decrLdIdx(load_idx);
972
973    while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
974        DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
975                "[sn:%lli]\n",
976                loadQueue[load_idx]->pcState(),
977                loadQueue[load_idx]->seqNum);
978
979        if (isStalled() && load_idx == stallingLoadIdx) {
980            stalled = false;
981            stallingStoreIsn = 0;
982            stallingLoadIdx = 0;
983        }
984
985        // Clear the smart pointer to make sure it is decremented.
986        loadQueue[load_idx]->setSquashed();
987        loadQueue[load_idx] = NULL;
988        --loads;
989
990        // Inefficient!
991        loadTail = load_idx;
992
993        decrLdIdx(load_idx);
994        ++lsqSquashedLoads;
995    }
996
997    if (isLoadBlocked) {
998        if (squashed_num < blockedLoadSeqNum) {
999            isLoadBlocked = false;
1000            loadBlockedHandled = false;
1001            blockedLoadSeqNum = 0;
1002        }
1003    }
1004
1005    if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1006        memDepViolator = NULL;
1007    }
1008
1009    int store_idx = storeTail;
1010    decrStIdx(store_idx);
1011
1012    while (stores != 0 &&
1013           storeQueue[store_idx].inst->seqNum > squashed_num) {
1014        // Instructions marked as can WB are already committed.
1015        if (storeQueue[store_idx].canWB) {
1016            break;
1017        }
1018
1019        DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1020                "idx:%i [sn:%lli]\n",
1021                storeQueue[store_idx].inst->pcState(),
1022                store_idx, storeQueue[store_idx].inst->seqNum);
1023
1024        // I don't think this can happen.  It should have been cleared
1025        // by the stalling load.
1026        if (isStalled() &&
1027            storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1028            panic("Is stalled should have been cleared by stalling load!\n");
1029            stalled = false;
1030            stallingStoreIsn = 0;
1031        }
1032
1033        // Clear the smart pointer to make sure it is decremented.
1034        storeQueue[store_idx].inst->setSquashed();
1035        storeQueue[store_idx].inst = NULL;
1036        storeQueue[store_idx].canWB = 0;
1037
1038        // Must delete request now that it wasn't handed off to
1039        // memory.  This is quite ugly.  @todo: Figure out the proper
1040        // place to really handle request deletes.
1041        delete storeQueue[store_idx].req;
1042        if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1043            delete storeQueue[store_idx].sreqLow;
1044            delete storeQueue[store_idx].sreqHigh;
1045
1046            storeQueue[store_idx].sreqLow = NULL;
1047            storeQueue[store_idx].sreqHigh = NULL;
1048        }
1049
1050        storeQueue[store_idx].req = NULL;
1051        --stores;
1052
1053        // Inefficient!
1054        storeTail = store_idx;
1055
1056        decrStIdx(store_idx);
1057        ++lsqSquashedStores;
1058    }
1059}
1060
1061template <class Impl>
1062void
1063LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1064{
1065    if (isStalled() &&
1066        storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1067        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1068                "load idx:%i\n",
1069                stallingStoreIsn, stallingLoadIdx);
1070        stalled = false;
1071        stallingStoreIsn = 0;
1072        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1073    }
1074
1075    if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1076        // The store is basically completed at this time. This
1077        // only works so long as the checker doesn't try to
1078        // verify the value in memory for stores.
1079        storeQueue[storeWBIdx].inst->setCompleted();
1080#if USE_CHECKER
1081        if (cpu->checker) {
1082            cpu->checker->verify(storeQueue[storeWBIdx].inst);
1083        }
1084#endif
1085    }
1086
1087    incrStIdx(storeWBIdx);
1088}
1089
1090template <class Impl>
1091void
1092LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1093{
1094    iewStage->wakeCPU();
1095
1096    // Squashed instructions do not need to complete their access.
1097    if (inst->isSquashed()) {
1098        iewStage->decrWb(inst->seqNum);
1099        assert(!inst->isStore());
1100        ++lsqIgnoredResponses;
1101        return;
1102    }
1103
1104    if (!inst->isExecuted()) {
1105        inst->setExecuted();
1106
1107        // Complete access to copy data to proper place.
1108        inst->completeAcc(pkt);
1109    }
1110
1111    // Need to insert instruction into queue to commit
1112    iewStage->instToCommit(inst);
1113
1114    iewStage->activityThisCycle();
1115
1116    // see if this load changed the PC
1117    iewStage->checkMisprediction(inst);
1118}
1119
1120template <class Impl>
1121void
1122LSQUnit<Impl>::completeStore(int store_idx)
1123{
1124    assert(storeQueue[store_idx].inst);
1125    storeQueue[store_idx].completed = true;
1126    --storesToWB;
1127    // A bit conservative because a store completion may not free up entries,
1128    // but hopefully avoids two store completions in one cycle from making
1129    // the CPU tick twice.
1130    cpu->wakeCPU();
1131    cpu->activityThisCycle();
1132
1133    if (store_idx == storeHead) {
1134        do {
1135            incrStIdx(storeHead);
1136
1137            --stores;
1138        } while (storeQueue[storeHead].completed &&
1139                 storeHead != storeTail);
1140
1141        iewStage->updateLSQNextCycle = true;
1142    }
1143
1144    DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1145            "idx:%i\n",
1146            storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1147
1148    if (isStalled() &&
1149        storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1150        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1151                "load idx:%i\n",
1152                stallingStoreIsn, stallingLoadIdx);
1153        stalled = false;
1154        stallingStoreIsn = 0;
1155        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1156    }
1157
1158    storeQueue[store_idx].inst->setCompleted();
1159
1160    // Tell the checker we've completed this instruction.  Some stores
1161    // may get reported twice to the checker, but the checker can
1162    // handle that case.
1163#if USE_CHECKER
1164    if (cpu->checker) {
1165        cpu->checker->verify(storeQueue[store_idx].inst);
1166    }
1167#endif
1168}
1169
1170template <class Impl>
1171bool
1172LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1173{
1174    if (!dcachePort->sendTiming(data_pkt)) {
1175        // Need to handle becoming blocked on a store.
1176        isStoreBlocked = true;
1177        ++lsqCacheBlocked;
1178        assert(retryPkt == NULL);
1179        retryPkt = data_pkt;
1180        lsq->setRetryTid(lsqID);
1181        return false;
1182    }
1183    return true;
1184}
1185
1186template <class Impl>
1187void
1188LSQUnit<Impl>::recvRetry()
1189{
1190    if (isStoreBlocked) {
1191        DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1192        assert(retryPkt != NULL);
1193
1194        LSQSenderState *state =
1195            dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1196
1197        if (dcachePort->sendTiming(retryPkt)) {
1198            // Don't finish the store unless this is the last packet.
1199            if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1200                    state->pendingPacket == retryPkt) {
1201                state->pktToSend = false;
1202                storePostSend(retryPkt);
1203            }
1204            retryPkt = NULL;
1205            isStoreBlocked = false;
1206            lsq->setRetryTid(InvalidThreadID);
1207
1208            // Send any outstanding packet.
1209            if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1210                assert(state->pendingPacket);
1211                if (sendStore(state->pendingPacket)) {
1212                    storePostSend(state->pendingPacket);
1213                }
1214            }
1215        } else {
1216            // Still blocked!
1217            ++lsqCacheBlocked;
1218            lsq->setRetryTid(lsqID);
1219        }
1220    } else if (isLoadBlocked) {
1221        DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, "
1222                "no need to resend packet.\n");
1223    } else {
1224        DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n");
1225    }
1226}
1227
1228template <class Impl>
1229inline void
1230LSQUnit<Impl>::incrStIdx(int &store_idx)
1231{
1232    if (++store_idx >= SQEntries)
1233        store_idx = 0;
1234}
1235
1236template <class Impl>
1237inline void
1238LSQUnit<Impl>::decrStIdx(int &store_idx)
1239{
1240    if (--store_idx < 0)
1241        store_idx += SQEntries;
1242}
1243
1244template <class Impl>
1245inline void
1246LSQUnit<Impl>::incrLdIdx(int &load_idx)
1247{
1248    if (++load_idx >= LQEntries)
1249        load_idx = 0;
1250}
1251
1252template <class Impl>
1253inline void
1254LSQUnit<Impl>::decrLdIdx(int &load_idx)
1255{
1256    if (--load_idx < 0)
1257        load_idx += LQEntries;
1258}
1259
1260template <class Impl>
1261void
1262LSQUnit<Impl>::dumpInsts()
1263{
1264    cprintf("Load store queue: Dumping instructions.\n");
1265    cprintf("Load queue size: %i\n", loads);
1266    cprintf("Load queue: ");
1267
1268    int load_idx = loadHead;
1269
1270    while (load_idx != loadTail && loadQueue[load_idx]) {
1271        cprintf("%s ", loadQueue[load_idx]->pcState());
1272
1273        incrLdIdx(load_idx);
1274    }
1275
1276    cprintf("Store queue size: %i\n", stores);
1277    cprintf("Store queue: ");
1278
1279    int store_idx = storeHead;
1280
1281    while (store_idx != storeTail && storeQueue[store_idx].inst) {
1282        cprintf("%s ", storeQueue[store_idx].inst->pcState());
1283
1284        incrStIdx(store_idx);
1285    }
1286
1287    cprintf("\n");
1288}
1289