lsq_unit_impl.hh revision 11356
12568SN/A
22568SN/A/*
32568SN/A * Copyright (c) 2010-2014 ARM Limited
42568SN/A * Copyright (c) 2013 Advanced Micro Devices, Inc.
52568SN/A * All rights reserved
62568SN/A *
72568SN/A * The license below extends only to copyright in the software and shall
82568SN/A * not be construed as granting a license to any other intellectual
92568SN/A * property including but not limited to intellectual property relating
102568SN/A * to a hardware implementation of the functionality of the software
112568SN/A * licensed hereunder.  You may use the software subject to the license
122568SN/A * terms below provided that you ensure that this notice is replicated
132568SN/A * unmodified and in its entirety in all distributions of the software,
142568SN/A * modified or unmodified, in source code or in binary form.
152568SN/A *
162568SN/A * Copyright (c) 2004-2005 The Regents of The University of Michigan
172568SN/A * All rights reserved.
182568SN/A *
192568SN/A * Redistribution and use in source and binary forms, with or without
202568SN/A * modification, are permitted provided that the following conditions are
212568SN/A * met: redistributions of source code must retain the above copyright
222568SN/A * notice, this list of conditions and the following disclaimer;
232568SN/A * redistributions in binary form must reproduce the above copyright
242568SN/A * notice, this list of conditions and the following disclaimer in the
252568SN/A * documentation and/or other materials provided with the distribution;
262568SN/A * neither the name of the copyright holders nor the names of its
272568SN/A * contributors may be used to endorse or promote products derived from
282665Ssaidi@eecs.umich.edu * this software without specific prior written permission.
292665Ssaidi@eecs.umich.edu *
302665Ssaidi@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
312568SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
322568SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
332568SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
342982Sstever@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
352982Sstever@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
362568SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
372568SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
382643Sstever@eecs.umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
392568SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
402568SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
412568SN/A *
424762Snate@binkert.org * Authors: Kevin Lim
432568SN/A *          Korey Sewell
442643Sstever@eecs.umich.edu */
452643Sstever@eecs.umich.edu
464435Ssaidi@eecs.umich.edu#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__
474435Ssaidi@eecs.umich.edu#define __CPU_O3_LSQ_UNIT_IMPL_HH__
482643Sstever@eecs.umich.edu
494435Ssaidi@eecs.umich.edu#include "arch/generic/debugfaults.hh"
504435Ssaidi@eecs.umich.edu#include "arch/locked_mem.hh"
514435Ssaidi@eecs.umich.edu#include "base/str.hh"
522643Sstever@eecs.umich.edu#include "config/the_isa.hh"
532643Sstever@eecs.umich.edu#include "cpu/checker/cpu.hh"
542643Sstever@eecs.umich.edu#include "cpu/o3/lsq.hh"
554435Ssaidi@eecs.umich.edu#include "cpu/o3/lsq_unit.hh"
564435Ssaidi@eecs.umich.edu#include "debug/Activity.hh"
574435Ssaidi@eecs.umich.edu#include "debug/IEW.hh"
584435Ssaidi@eecs.umich.edu#include "debug/LSQUnit.hh"
594435Ssaidi@eecs.umich.edu#include "debug/O3PipeView.hh"
604435Ssaidi@eecs.umich.edu#include "mem/packet.hh"
614435Ssaidi@eecs.umich.edu#include "mem/request.hh"
622643Sstever@eecs.umich.edu
634432Ssaidi@eecs.umich.edutemplate<class Impl>
644432Ssaidi@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
652643Sstever@eecs.umich.edu                                              LSQUnit *lsq_ptr)
662643Sstever@eecs.umich.edu    : Event(Default_Pri, AutoDelete),
672643Sstever@eecs.umich.edu      inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
682738Sstever@eecs.umich.edu{
692643Sstever@eecs.umich.edu}
702643Sstever@eecs.umich.edu
712643Sstever@eecs.umich.edutemplate<class Impl>
722643Sstever@eecs.umich.eduvoid
732643Sstever@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::process()
742643Sstever@eecs.umich.edu{
752643Sstever@eecs.umich.edu    assert(!lsqPtr->cpu->switchedOut());
762643Sstever@eecs.umich.edu
772643Sstever@eecs.umich.edu    lsqPtr->writeback(inst, pkt);
782643Sstever@eecs.umich.edu
792643Sstever@eecs.umich.edu    if (pkt->senderState)
802643Sstever@eecs.umich.edu        delete pkt->senderState;
812643Sstever@eecs.umich.edu
822643Sstever@eecs.umich.edu    delete pkt->req;
832643Sstever@eecs.umich.edu    delete pkt;
842643Sstever@eecs.umich.edu}
852568SN/A
862568SN/Atemplate<class Impl>
872568SN/Aconst char *
882568SN/ALSQUnit<Impl>::WritebackEvent::description() const
892643Sstever@eecs.umich.edu{
904432Ssaidi@eecs.umich.edu    return "Store writeback";
914432Ssaidi@eecs.umich.edu}
924432Ssaidi@eecs.umich.edu
934432Ssaidi@eecs.umich.edutemplate<class Impl>
942568SN/Avoid
952568SN/ALSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
964433Ssaidi@eecs.umich.edu{
974435Ssaidi@eecs.umich.edu    LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
984433Ssaidi@eecs.umich.edu    DynInstPtr inst = state->inst;
994435Ssaidi@eecs.umich.edu    DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
1004435Ssaidi@eecs.umich.edu    DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);
1014435Ssaidi@eecs.umich.edu
1024435Ssaidi@eecs.umich.edu    if (state->cacheBlocked) {
1034435Ssaidi@eecs.umich.edu        // This is the first half of a previous split load,
1044435Ssaidi@eecs.umich.edu        // where the 2nd half blocked, ignore this response
1054435Ssaidi@eecs.umich.edu        DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier "
1064435Ssaidi@eecs.umich.edu                "blocked split load recieved. Ignoring.\n", inst->seqNum);
1074435Ssaidi@eecs.umich.edu        delete state;
1084433Ssaidi@eecs.umich.edu        return;
1092568SN/A    }
1102643Sstever@eecs.umich.edu
1112568SN/A    // If this is a split access, wait until all packets are received.
1122568SN/A    if (TheISA::HasUnalignedMemAcc && !state->complete()) {
1133349Sbinkertn@umich.edu        return;
1142568SN/A    }
1154433Ssaidi@eecs.umich.edu
1164433Ssaidi@eecs.umich.edu    assert(!cpu->switchedOut());
1174433Ssaidi@eecs.umich.edu    if (!inst->isSquashed()) {
1184433Ssaidi@eecs.umich.edu        if (!state->noWB) {
1194433Ssaidi@eecs.umich.edu            if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
1203662Srdreslin@umich.edu                !state->isLoad) {
1212643Sstever@eecs.umich.edu                writeback(inst, pkt);
1224450Ssaidi@eecs.umich.edu            } else {
1234450Ssaidi@eecs.umich.edu                writeback(inst, state->mainPkt);
1244450Ssaidi@eecs.umich.edu            }
1254450Ssaidi@eecs.umich.edu        }
1264450Ssaidi@eecs.umich.edu
1274450Ssaidi@eecs.umich.edu        if (inst->isStore()) {
1284450Ssaidi@eecs.umich.edu            completeStore(state->idx);
1294450Ssaidi@eecs.umich.edu        }
1304433Ssaidi@eecs.umich.edu    }
1314433Ssaidi@eecs.umich.edu
1324433Ssaidi@eecs.umich.edu    if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
1333662Srdreslin@umich.edu        delete state->mainPkt->req;
1344433Ssaidi@eecs.umich.edu        delete state->mainPkt;
1354433Ssaidi@eecs.umich.edu    }
1364435Ssaidi@eecs.umich.edu
1374433Ssaidi@eecs.umich.edu    pkt->req->setAccessLatency();
1384433Ssaidi@eecs.umich.edu    cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));
1394433Ssaidi@eecs.umich.edu
1404433Ssaidi@eecs.umich.edu    delete state;
1414433Ssaidi@eecs.umich.edu}
1424433Ssaidi@eecs.umich.edu
1434433Ssaidi@eecs.umich.edutemplate <class Impl>
1444433Ssaidi@eecs.umich.eduLSQUnit<Impl>::LSQUnit()
1454433Ssaidi@eecs.umich.edu    : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
1464433Ssaidi@eecs.umich.edu      isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false),
1474433Ssaidi@eecs.umich.edu      pendingPkt(nullptr)
1484433Ssaidi@eecs.umich.edu{
1494433Ssaidi@eecs.umich.edu}
1502657Ssaidi@eecs.umich.edu
1512657Ssaidi@eecs.umich.edutemplate<class Impl>
1524433Ssaidi@eecs.umich.eduvoid
1534433Ssaidi@eecs.umich.eduLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
1544433Ssaidi@eecs.umich.edu        LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
1554433Ssaidi@eecs.umich.edu        unsigned id)
1564433Ssaidi@eecs.umich.edu{
1574433Ssaidi@eecs.umich.edu    cpu = cpu_ptr;
1582657Ssaidi@eecs.umich.edu    iewStage = iew_ptr;
1594433Ssaidi@eecs.umich.edu
1604435Ssaidi@eecs.umich.edu    lsq = lsq_ptr;
1614433Ssaidi@eecs.umich.edu
1624435Ssaidi@eecs.umich.edu    lsqID = id;
1634435Ssaidi@eecs.umich.edu
1644433Ssaidi@eecs.umich.edu    DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);
1654435Ssaidi@eecs.umich.edu
1664433Ssaidi@eecs.umich.edu    // Add 1 for the sentinel entry (they are circular queues).
1674435Ssaidi@eecs.umich.edu    LQEntries = maxLQEntries + 1;
1684435Ssaidi@eecs.umich.edu    SQEntries = maxSQEntries + 1;
1694433Ssaidi@eecs.umich.edu
1704435Ssaidi@eecs.umich.edu    //Due to uint8_t index in LSQSenderState
1714435Ssaidi@eecs.umich.edu    assert(LQEntries <= 256);
1724435Ssaidi@eecs.umich.edu    assert(SQEntries <= 256);
1734435Ssaidi@eecs.umich.edu
1744435Ssaidi@eecs.umich.edu    loadQueue.resize(LQEntries);
1754435Ssaidi@eecs.umich.edu    storeQueue.resize(SQEntries);
1764435Ssaidi@eecs.umich.edu
1774435Ssaidi@eecs.umich.edu    depCheckShift = params->LSQDepCheckShift;
1784435Ssaidi@eecs.umich.edu    checkLoads = params->LSQCheckLoads;
1794435Ssaidi@eecs.umich.edu    cachePorts = params->cachePorts;
1804435Ssaidi@eecs.umich.edu    needsTSO = params->needsTSO;
1814435Ssaidi@eecs.umich.edu
1824435Ssaidi@eecs.umich.edu    resetState();
1834435Ssaidi@eecs.umich.edu}
1844435Ssaidi@eecs.umich.edu
1854435Ssaidi@eecs.umich.edu
1864435Ssaidi@eecs.umich.edutemplate<class Impl>
1874435Ssaidi@eecs.umich.eduvoid
1884435Ssaidi@eecs.umich.eduLSQUnit<Impl>::resetState()
1894435Ssaidi@eecs.umich.edu{
1904435Ssaidi@eecs.umich.edu    loads = stores = storesToWB = 0;
1914435Ssaidi@eecs.umich.edu
1924435Ssaidi@eecs.umich.edu    loadHead = loadTail = 0;
1934435Ssaidi@eecs.umich.edu
1944435Ssaidi@eecs.umich.edu    storeHead = storeWBIdx = storeTail = 0;
1954433Ssaidi@eecs.umich.edu
1964433Ssaidi@eecs.umich.edu    usedPorts = 0;
1974433Ssaidi@eecs.umich.edu
1984433Ssaidi@eecs.umich.edu    retryPkt = NULL;
1993349Sbinkertn@umich.edu    memDepViolator = NULL;
2002657Ssaidi@eecs.umich.edu
2014450Ssaidi@eecs.umich.edu    stalled = false;
2022643Sstever@eecs.umich.edu
2032643Sstever@eecs.umich.edu    cacheBlockMask = ~(cpu->cacheLineSize() - 1);
2042643Sstever@eecs.umich.edu}
2052643Sstever@eecs.umich.edu
2062643Sstever@eecs.umich.edutemplate<class Impl>
2072643Sstever@eecs.umich.edustd::string
2082643Sstever@eecs.umich.eduLSQUnit<Impl>::name() const
2092643Sstever@eecs.umich.edu{
2104433Ssaidi@eecs.umich.edu    if (Impl::MaxThreads == 1) {
2114450Ssaidi@eecs.umich.edu        return iewStage->name() + ".lsq";
2124450Ssaidi@eecs.umich.edu    } else {
2134450Ssaidi@eecs.umich.edu        return iewStage->name() + ".lsq.thread" + std::to_string(lsqID);
2144433Ssaidi@eecs.umich.edu    }
2154433Ssaidi@eecs.umich.edu}
2164739Sstever@eecs.umich.edu
2172643Sstever@eecs.umich.edutemplate<class Impl>
2182643Sstever@eecs.umich.eduvoid
2192643Sstever@eecs.umich.eduLSQUnit<Impl>::regStats()
2204450Ssaidi@eecs.umich.edu{
2214450Ssaidi@eecs.umich.edu    lsqForwLoads
2224450Ssaidi@eecs.umich.edu        .name(name() + ".forwLoads")
2234450Ssaidi@eecs.umich.edu        .desc("Number of loads that had data forwarded from stores");
2244450Ssaidi@eecs.umich.edu
2254450Ssaidi@eecs.umich.edu    invAddrLoads
2264450Ssaidi@eecs.umich.edu        .name(name() + ".invAddrLoads")
2272643Sstever@eecs.umich.edu        .desc("Number of loads ignored due to an invalid address");
2282643Sstever@eecs.umich.edu
2292643Sstever@eecs.umich.edu    lsqSquashedLoads
2302643Sstever@eecs.umich.edu        .name(name() + ".squashedLoads")
2312643Sstever@eecs.umich.edu        .desc("Number of loads squashed");
2322643Sstever@eecs.umich.edu
2332643Sstever@eecs.umich.edu    lsqIgnoredResponses
2342643Sstever@eecs.umich.edu        .name(name() + ".ignoredResponses")
2352643Sstever@eecs.umich.edu        .desc("Number of memory responses ignored because the instruction is squashed");
2362568SN/A
2372643Sstever@eecs.umich.edu    lsqMemOrderViolation
2382568SN/A        .name(name() + ".memOrderViolation")
2392568SN/A        .desc("Number of memory ordering violations");
2402568SN/A
2412643Sstever@eecs.umich.edu    lsqSquashedStores
2422568SN/A        .name(name() + ".squashedStores")
2432643Sstever@eecs.umich.edu        .desc("Number of stores squashed");
2442568SN/A
2452643Sstever@eecs.umich.edu    invAddrSwpfs
2462643Sstever@eecs.umich.edu        .name(name() + ".invAddrSwpfs")
2472643Sstever@eecs.umich.edu        .desc("Number of software prefetches ignored due to an invalid address");
2482643Sstever@eecs.umich.edu
2493349Sbinkertn@umich.edu    lsqBlockedLoads
2502643Sstever@eecs.umich.edu        .name(name() + ".blockedLoads")
2514432Ssaidi@eecs.umich.edu        .desc("Number of blocked loads due to partial load-store forwarding");
2524432Ssaidi@eecs.umich.edu
2534454Ssaidi@eecs.umich.edu    lsqRescheduledLoads
2544432Ssaidi@eecs.umich.edu        .name(name() + ".rescheduledLoads")
2554454Ssaidi@eecs.umich.edu        .desc("Number of loads that were rescheduled");
2564454Ssaidi@eecs.umich.edu
2574454Ssaidi@eecs.umich.edu    lsqCacheBlocked
2584454Ssaidi@eecs.umich.edu        .name(name() + ".cacheBlocked")
2594454Ssaidi@eecs.umich.edu        .desc("Number of times an access to memory failed due to the cache being blocked");
2604454Ssaidi@eecs.umich.edu}
2614454Ssaidi@eecs.umich.edu
2624432Ssaidi@eecs.umich.edutemplate<class Impl>
2634432Ssaidi@eecs.umich.eduvoid
2642643Sstever@eecs.umich.eduLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port)
2652643Sstever@eecs.umich.edu{
2662643Sstever@eecs.umich.edu    dcachePort = dcache_port;
2674450Ssaidi@eecs.umich.edu}
2684450Ssaidi@eecs.umich.edu
2694432Ssaidi@eecs.umich.edutemplate<class Impl>
2702643Sstever@eecs.umich.eduvoid
2712643Sstever@eecs.umich.eduLSQUnit<Impl>::clearLQ()
2722643Sstever@eecs.umich.edu{
2732643Sstever@eecs.umich.edu    loadQueue.clear();
2742657Ssaidi@eecs.umich.edu}
2752657Ssaidi@eecs.umich.edu
2764433Ssaidi@eecs.umich.edutemplate<class Impl>
2772657Ssaidi@eecs.umich.eduvoid
2782657Ssaidi@eecs.umich.eduLSQUnit<Impl>::clearSQ()
2792657Ssaidi@eecs.umich.edu{
2802657Ssaidi@eecs.umich.edu    storeQueue.clear();
2812657Ssaidi@eecs.umich.edu}
2822657Ssaidi@eecs.umich.edu
2832657Ssaidi@eecs.umich.edutemplate<class Impl>
2842657Ssaidi@eecs.umich.eduvoid
2854450Ssaidi@eecs.umich.eduLSQUnit<Impl>::drainSanityCheck() const
2864450Ssaidi@eecs.umich.edu{
2874433Ssaidi@eecs.umich.edu    for (int i = 0; i < loadQueue.size(); ++i)
2884450Ssaidi@eecs.umich.edu        assert(!loadQueue[i]);
2894450Ssaidi@eecs.umich.edu
2904450Ssaidi@eecs.umich.edu    assert(storesToWB == 0);
2914433Ssaidi@eecs.umich.edu    assert(!retryPkt);
2922657Ssaidi@eecs.umich.edu}
2932657Ssaidi@eecs.umich.edu
2942657Ssaidi@eecs.umich.edutemplate<class Impl>
2954433Ssaidi@eecs.umich.eduvoid
2962657Ssaidi@eecs.umich.eduLSQUnit<Impl>::takeOverFrom()
2972657Ssaidi@eecs.umich.edu{
2982643Sstever@eecs.umich.edu    resetState();
2992643Sstever@eecs.umich.edu}
3004435Ssaidi@eecs.umich.edu
3012643Sstever@eecs.umich.edutemplate<class Impl>
3024433Ssaidi@eecs.umich.eduvoid
3034433Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeLQ(unsigned size)
3042568SN/A{
3052568SN/A    unsigned size_plus_sentinel = size + 1;
3062568SN/A    assert(size_plus_sentinel >= LQEntries);
3072657Ssaidi@eecs.umich.edu
3082568SN/A    if (size_plus_sentinel > LQEntries) {
3092568SN/A        while (size_plus_sentinel > loadQueue.size()) {
3104435Ssaidi@eecs.umich.edu            DynInstPtr dummy;
3114435Ssaidi@eecs.umich.edu            loadQueue.push_back(dummy);
3124435Ssaidi@eecs.umich.edu            LQEntries++;
3134435Ssaidi@eecs.umich.edu        }
3144435Ssaidi@eecs.umich.edu    } else {
3154435Ssaidi@eecs.umich.edu        LQEntries = size_plus_sentinel;
3162568SN/A    }
3172568SN/A
3182643Sstever@eecs.umich.edu    assert(LQEntries <= 256);
3192568SN/A}
3202568SN/A
3213349Sbinkertn@umich.edutemplate<class Impl>
3222568SN/Avoid
3234450Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeSQ(unsigned size)
3244454Ssaidi@eecs.umich.edu{
3254454Ssaidi@eecs.umich.edu    unsigned size_plus_sentinel = size + 1;
3264450Ssaidi@eecs.umich.edu    if (size_plus_sentinel > SQEntries) {
3274454Ssaidi@eecs.umich.edu        while (size_plus_sentinel > storeQueue.size()) {
3284454Ssaidi@eecs.umich.edu            SQEntry dummy;
3294454Ssaidi@eecs.umich.edu            storeQueue.push_back(dummy);
3304450Ssaidi@eecs.umich.edu            SQEntries++;
3314450Ssaidi@eecs.umich.edu        }
3324454Ssaidi@eecs.umich.edu    } else {
3334450Ssaidi@eecs.umich.edu        SQEntries = size_plus_sentinel;
3344454Ssaidi@eecs.umich.edu    }
3352568SN/A
3362568SN/A    assert(SQEntries <= 256);
3372643Sstever@eecs.umich.edu}
3382568SN/A
3392568SN/Atemplate <class Impl>
3403349Sbinkertn@umich.eduvoid
3412568SN/ALSQUnit<Impl>::insert(DynInstPtr &inst)
3422643Sstever@eecs.umich.edu{
3432568SN/A    assert(inst->isMemRef());
3442568SN/A
3452643Sstever@eecs.umich.edu    assert(inst->isLoad() || inst->isStore());
3462643Sstever@eecs.umich.edu
3472643Sstever@eecs.umich.edu    if (inst->isLoad()) {
3482568SN/A        insertLoad(inst);
3492568SN/A    } else {
3502568SN/A        insertStore(inst);
3512568SN/A    }
3522643Sstever@eecs.umich.edu
3532568SN/A    inst->setInLSQ();
3542568SN/A}
3552568SN/A
3562643Sstever@eecs.umich.edutemplate <class Impl>
3572568SN/Avoid
3582643Sstever@eecs.umich.eduLSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
3592568SN/A{
3602643Sstever@eecs.umich.edu    assert((loadTail + 1) % LQEntries != loadHead);
3612568SN/A    assert(loads < LQEntries);
3622568SN/A
3632568SN/A    DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
3642643Sstever@eecs.umich.edu            load_inst->pcState(), loadTail, load_inst->seqNum);
3654475Sstever@eecs.umich.edu
3662568SN/A    load_inst->lqIdx = loadTail;
3672643Sstever@eecs.umich.edu
3682568SN/A    if (stores == 0) {
3692568SN/A        load_inst->sqIdx = -1;
3704762Snate@binkert.org    } else {
3714762Snate@binkert.org        load_inst->sqIdx = storeTail;
3722568SN/A    }
3734762Snate@binkert.org
3742568SN/A    loadQueue[loadTail] = load_inst;
375
376    incrLdIdx(loadTail);
377
378    ++loads;
379}
380
381template <class Impl>
382void
383LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
384{
385    // Make sure it is not full before inserting an instruction.
386    assert((storeTail + 1) % SQEntries != storeHead);
387    assert(stores < SQEntries);
388
389    DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
390            store_inst->pcState(), storeTail, store_inst->seqNum);
391
392    store_inst->sqIdx = storeTail;
393    store_inst->lqIdx = loadTail;
394
395    storeQueue[storeTail] = SQEntry(store_inst);
396
397    incrStIdx(storeTail);
398
399    ++stores;
400}
401
402template <class Impl>
403typename Impl::DynInstPtr
404LSQUnit<Impl>::getMemDepViolator()
405{
406    DynInstPtr temp = memDepViolator;
407
408    memDepViolator = NULL;
409
410    return temp;
411}
412
413template <class Impl>
414unsigned
415LSQUnit<Impl>::numFreeLoadEntries()
416{
417        //LQ has an extra dummy entry to differentiate
418        //empty/full conditions. Subtract 1 from the free entries.
419        DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads);
420        return LQEntries - loads - 1;
421}
422
423template <class Impl>
424unsigned
425LSQUnit<Impl>::numFreeStoreEntries()
426{
427        //SQ has an extra dummy entry to differentiate
428        //empty/full conditions. Subtract 1 from the free entries.
429        DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores);
430        return SQEntries - stores - 1;
431
432 }
433
434template <class Impl>
435void
436LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
437{
438    int load_idx = loadHead;
439    DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());
440
441    // Only Invalidate packet calls checkSnoop
442    assert(pkt->isInvalidate());
443    for (int x = 0; x < cpu->numContexts(); x++) {
444        ThreadContext *tc = cpu->getContext(x);
445        bool no_squash = cpu->thread[x]->noSquashFromTC;
446        cpu->thread[x]->noSquashFromTC = true;
447        TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask);
448        cpu->thread[x]->noSquashFromTC = no_squash;
449    }
450
451    Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;
452
453    DynInstPtr ld_inst = loadQueue[load_idx];
454    if (ld_inst) {
455        Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
456        Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
457
458        // Check that this snoop didn't just invalidate our lock flag
459        if (ld_inst->effAddrValid() && (load_addr_low == invalidate_addr
460                                        || load_addr_high == invalidate_addr)
461            && ld_inst->memReqFlags & Request::LLSC)
462            TheISA::handleLockedSnoopHit(ld_inst.get());
463    }
464
465    // If this is the only load in the LSQ we don't care
466    if (load_idx == loadTail)
467        return;
468
469    incrLdIdx(load_idx);
470
471    bool force_squash = false;
472
473    while (load_idx != loadTail) {
474        DynInstPtr ld_inst = loadQueue[load_idx];
475
476        if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
477            incrLdIdx(load_idx);
478            continue;
479        }
480
481        Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
482        Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;
483
484        DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
485                    ld_inst->seqNum, load_addr_low, invalidate_addr);
486
487        if ((load_addr_low == invalidate_addr
488             || load_addr_high == invalidate_addr) || force_squash) {
489            if (needsTSO) {
490                // If we have a TSO system, as all loads must be ordered with
491                // all other loads, this load as well as *all* subsequent loads
492                // need to be squashed to prevent possible load reordering.
493                force_squash = true;
494            }
495            if (ld_inst->possibleLoadViolation() || force_squash) {
496                DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
497                        pkt->getAddr(), ld_inst->seqNum);
498
499                // Mark the load for re-execution
500                ld_inst->fault = std::make_shared<ReExec>();
501            } else {
502                DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n",
503                        pkt->getAddr(), ld_inst->seqNum);
504
505                // Make sure that we don't lose a snoop hitting a LOCKED
506                // address since the LOCK* flags don't get updated until
507                // commit.
508                if (ld_inst->memReqFlags & Request::LLSC)
509                    TheISA::handleLockedSnoopHit(ld_inst.get());
510
511                // If a older load checks this and it's true
512                // then we might have missed the snoop
513                // in which case we need to invalidate to be sure
514                ld_inst->hitExternalSnoop(true);
515            }
516        }
517        incrLdIdx(load_idx);
518    }
519    return;
520}
521
522template <class Impl>
523Fault
524LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
525{
526    Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
527    Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;
528
529    /** @todo in theory you only need to check an instruction that has executed
530     * however, there isn't a good way in the pipeline at the moment to check
531     * all instructions that will execute before the store writes back. Thus,
532     * like the implementation that came before it, we're overly conservative.
533     */
534    while (load_idx != loadTail) {
535        DynInstPtr ld_inst = loadQueue[load_idx];
536        if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
537            incrLdIdx(load_idx);
538            continue;
539        }
540
541        Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
542        Addr ld_eff_addr2 =
543            (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;
544
545        if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
546            if (inst->isLoad()) {
547                // If this load is to the same block as an external snoop
548                // invalidate that we've observed then the load needs to be
549                // squashed as it could have newer data
550                if (ld_inst->hitExternalSnoop()) {
551                    if (!memDepViolator ||
552                            ld_inst->seqNum < memDepViolator->seqNum) {
553                        DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
554                                "and [sn:%lli] at address %#x\n",
555                                inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
556                        memDepViolator = ld_inst;
557
558                        ++lsqMemOrderViolation;
559
560                        return std::make_shared<GenericISA::M5PanicFault>(
561                            "Detected fault with inst [sn:%lli] and "
562                            "[sn:%lli] at address %#x\n",
563                            inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
564                    }
565                }
566
567                // Otherwise, mark the load has a possible load violation
568                // and if we see a snoop before it's commited, we need to squash
569                ld_inst->possibleLoadViolation(true);
570                DPRINTF(LSQUnit, "Found possible load violation at addr: %#x"
571                        " between instructions [sn:%lli] and [sn:%lli]\n",
572                        inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
573            } else {
574                // A load/store incorrectly passed this store.
575                // Check if we already have a violator, or if it's newer
576                // squash and refetch.
577                if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
578                    break;
579
580                DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
581                        "[sn:%lli] at address %#x\n",
582                        inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
583                memDepViolator = ld_inst;
584
585                ++lsqMemOrderViolation;
586
587                return std::make_shared<GenericISA::M5PanicFault>(
588                    "Detected fault with "
589                    "inst [sn:%lli] and [sn:%lli] at address %#x\n",
590                    inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
591            }
592        }
593
594        incrLdIdx(load_idx);
595    }
596    return NoFault;
597}
598
599
600
601
602template <class Impl>
603Fault
604LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
605{
606    using namespace TheISA;
607    // Execute a specific load.
608    Fault load_fault = NoFault;
609
610    DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
611            inst->pcState(), inst->seqNum);
612
613    assert(!inst->isSquashed());
614
615    load_fault = inst->initiateAcc();
616
617    if (inst->isTranslationDelayed() &&
618        load_fault == NoFault)
619        return load_fault;
620
621    // If the instruction faulted or predicated false, then we need to send it
622    // along to commit without the instruction completing.
623    if (load_fault != NoFault || !inst->readPredicate()) {
624        // Send this instruction to commit, also make sure iew stage
625        // realizes there is activity.  Mark it as executed unless it
626        // is a strictly ordered load that needs to hit the head of
627        // commit.
628        if (!inst->readPredicate())
629            inst->forwardOldRegs();
630        DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
631                inst->seqNum,
632                (load_fault != NoFault ? "fault" : "predication"));
633        if (!(inst->hasRequest() && inst->strictlyOrdered()) ||
634            inst->isAtCommit()) {
635            inst->setExecuted();
636        }
637        iewStage->instToCommit(inst);
638        iewStage->activityThisCycle();
639    } else {
640        assert(inst->effAddrValid());
641        int load_idx = inst->lqIdx;
642        incrLdIdx(load_idx);
643
644        if (checkLoads)
645            return checkViolations(load_idx, inst);
646    }
647
648    return load_fault;
649}
650
651template <class Impl>
652Fault
653LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
654{
655    using namespace TheISA;
656    // Make sure that a store exists.
657    assert(stores != 0);
658
659    int store_idx = store_inst->sqIdx;
660
661    DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
662            store_inst->pcState(), store_inst->seqNum);
663
664    assert(!store_inst->isSquashed());
665
666    // Check the recently completed loads to see if any match this store's
667    // address.  If so, then we have a memory ordering violation.
668    int load_idx = store_inst->lqIdx;
669
670    Fault store_fault = store_inst->initiateAcc();
671
672    if (store_inst->isTranslationDelayed() &&
673        store_fault == NoFault)
674        return store_fault;
675
676    if (!store_inst->readPredicate())
677        store_inst->forwardOldRegs();
678
679    if (storeQueue[store_idx].size == 0) {
680        DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
681                store_inst->pcState(), store_inst->seqNum);
682
683        return store_fault;
684    } else if (!store_inst->readPredicate()) {
685        DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
686                store_inst->seqNum);
687        return store_fault;
688    }
689
690    assert(store_fault == NoFault);
691
692    if (store_inst->isStoreConditional()) {
693        // Store conditionals need to set themselves as able to
694        // writeback if we haven't had a fault by here.
695        storeQueue[store_idx].canWB = true;
696
697        ++storesToWB;
698    }
699
700    return checkViolations(load_idx, store_inst);
701
702}
703
704template <class Impl>
705void
706LSQUnit<Impl>::commitLoad()
707{
708    assert(loadQueue[loadHead]);
709
710    DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
711            loadQueue[loadHead]->pcState());
712
713    loadQueue[loadHead] = NULL;
714
715    incrLdIdx(loadHead);
716
717    --loads;
718}
719
720template <class Impl>
721void
722LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
723{
724    assert(loads == 0 || loadQueue[loadHead]);
725
726    while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
727        commitLoad();
728    }
729}
730
731template <class Impl>
732void
733LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
734{
735    assert(stores == 0 || storeQueue[storeHead].inst);
736
737    int store_idx = storeHead;
738
739    while (store_idx != storeTail) {
740        assert(storeQueue[store_idx].inst);
741        // Mark any stores that are now committed and have not yet
742        // been marked as able to write back.
743        if (!storeQueue[store_idx].canWB) {
744            if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
745                break;
746            }
747            DPRINTF(LSQUnit, "Marking store as able to write back, PC "
748                    "%s [sn:%lli]\n",
749                    storeQueue[store_idx].inst->pcState(),
750                    storeQueue[store_idx].inst->seqNum);
751
752            storeQueue[store_idx].canWB = true;
753
754            ++storesToWB;
755        }
756
757        incrStIdx(store_idx);
758    }
759}
760
761template <class Impl>
762void
763LSQUnit<Impl>::writebackPendingStore()
764{
765    if (hasPendingPkt) {
766        assert(pendingPkt != NULL);
767
768        // If the cache is blocked, this will store the packet for retry.
769        if (sendStore(pendingPkt)) {
770            storePostSend(pendingPkt);
771        }
772        pendingPkt = NULL;
773        hasPendingPkt = false;
774    }
775}
776
777template <class Impl>
778void
779LSQUnit<Impl>::writebackStores()
780{
781    // First writeback the second packet from any split store that didn't
782    // complete last cycle because there weren't enough cache ports available.
783    if (TheISA::HasUnalignedMemAcc) {
784        writebackPendingStore();
785    }
786
787    while (storesToWB > 0 &&
788           storeWBIdx != storeTail &&
789           storeQueue[storeWBIdx].inst &&
790           storeQueue[storeWBIdx].canWB &&
791           ((!needsTSO) || (!storeInFlight)) &&
792           usedPorts < cachePorts) {
793
794        if (isStoreBlocked) {
795            DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
796                    " is blocked!\n");
797            break;
798        }
799
800        // Store didn't write any data so no need to write it back to
801        // memory.
802        if (storeQueue[storeWBIdx].size == 0) {
803            completeStore(storeWBIdx);
804
805            incrStIdx(storeWBIdx);
806
807            continue;
808        }
809
810        ++usedPorts;
811
812        if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
813            incrStIdx(storeWBIdx);
814
815            continue;
816        }
817
818        assert(storeQueue[storeWBIdx].req);
819        assert(!storeQueue[storeWBIdx].committed);
820
821        if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
822            assert(storeQueue[storeWBIdx].sreqLow);
823            assert(storeQueue[storeWBIdx].sreqHigh);
824        }
825
826        DynInstPtr inst = storeQueue[storeWBIdx].inst;
827
828        Request *req = storeQueue[storeWBIdx].req;
829        RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
830        RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;
831
832        storeQueue[storeWBIdx].committed = true;
833
834        assert(!inst->memData);
835        inst->memData = new uint8_t[req->getSize()];
836
837        if (storeQueue[storeWBIdx].isAllZeros)
838            memset(inst->memData, 0, req->getSize());
839        else
840            memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());
841
842        PacketPtr data_pkt;
843        PacketPtr snd_data_pkt = NULL;
844
845        LSQSenderState *state = new LSQSenderState;
846        state->isLoad = false;
847        state->idx = storeWBIdx;
848        state->inst = inst;
849
850        if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {
851
852            // Build a single data packet if the store isn't split.
853            data_pkt = Packet::createWrite(req);
854            data_pkt->dataStatic(inst->memData);
855            data_pkt->senderState = state;
856        } else {
857            // Create two packets if the store is split in two.
858            data_pkt = Packet::createWrite(sreqLow);
859            snd_data_pkt = Packet::createWrite(sreqHigh);
860
861            data_pkt->dataStatic(inst->memData);
862            snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());
863
864            data_pkt->senderState = state;
865            snd_data_pkt->senderState = state;
866
867            state->isSplit = true;
868            state->outstanding = 2;
869
870            // Can delete the main request now.
871            delete req;
872            req = sreqLow;
873        }
874
875        DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
876                "to Addr:%#x, data:%#x [sn:%lli]\n",
877                storeWBIdx, inst->pcState(),
878                req->getPaddr(), (int)*(inst->memData),
879                inst->seqNum);
880
881        // @todo: Remove this SC hack once the memory system handles it.
882        if (inst->isStoreConditional()) {
883            assert(!storeQueue[storeWBIdx].isSplit);
884            // Disable recording the result temporarily.  Writing to
885            // misc regs normally updates the result, but this is not
886            // the desired behavior when handling store conditionals.
887            inst->recordResult(false);
888            bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask);
889            inst->recordResult(true);
890
891            if (!success) {
892                // Instantly complete this store.
893                DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
894                        "Instantly completing it.\n",
895                        inst->seqNum);
896                WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
897                cpu->schedule(wb, curTick() + 1);
898                if (cpu->checker) {
899                    // Make sure to set the LLSC data for verification
900                    // if checker is loaded
901                    inst->reqToVerify->setExtraData(0);
902                    inst->completeAcc(data_pkt);
903                }
904                completeStore(storeWBIdx);
905                incrStIdx(storeWBIdx);
906                continue;
907            }
908        } else {
909            // Non-store conditionals do not need a writeback.
910            state->noWB = true;
911        }
912
913        bool split =
914            TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;
915
916        ThreadContext *thread = cpu->tcBase(lsqID);
917
918        if (req->isMmappedIpr()) {
919            assert(!inst->isStoreConditional());
920            TheISA::handleIprWrite(thread, data_pkt);
921            delete data_pkt;
922            if (split) {
923                assert(snd_data_pkt->req->isMmappedIpr());
924                TheISA::handleIprWrite(thread, snd_data_pkt);
925                delete snd_data_pkt;
926                delete sreqLow;
927                delete sreqHigh;
928            }
929            delete state;
930            delete req;
931            completeStore(storeWBIdx);
932            incrStIdx(storeWBIdx);
933        } else if (!sendStore(data_pkt)) {
934            DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
935                    "retry later\n",
936                    inst->seqNum);
937
938            // Need to store the second packet, if split.
939            if (split) {
940                state->pktToSend = true;
941                state->pendingPacket = snd_data_pkt;
942            }
943        } else {
944
945            // If split, try to send the second packet too
946            if (split) {
947                assert(snd_data_pkt);
948
949                // Ensure there are enough ports to use.
950                if (usedPorts < cachePorts) {
951                    ++usedPorts;
952                    if (sendStore(snd_data_pkt)) {
953                        storePostSend(snd_data_pkt);
954                    } else {
955                        DPRINTF(IEW, "D-Cache became blocked when writing"
956                                " [sn:%lli] second packet, will retry later\n",
957                                inst->seqNum);
958                    }
959                } else {
960
961                    // Store the packet for when there's free ports.
962                    assert(pendingPkt == NULL);
963                    pendingPkt = snd_data_pkt;
964                    hasPendingPkt = true;
965                }
966            } else {
967
968                // Not a split store.
969                storePostSend(data_pkt);
970            }
971        }
972    }
973
974    // Not sure this should set it to 0.
975    usedPorts = 0;
976
977    assert(stores >= 0 && storesToWB >= 0);
978}
979
980/*template <class Impl>
981void
982LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
983{
984    list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
985                                              mshrSeqNums.end(),
986                                              seqNum);
987
988    if (mshr_it != mshrSeqNums.end()) {
989        mshrSeqNums.erase(mshr_it);
990        DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
991    }
992}*/
993
994template <class Impl>
995void
996LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
997{
998    DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
999            "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);
1000
1001    int load_idx = loadTail;
1002    decrLdIdx(load_idx);
1003
1004    while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
1005        DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
1006                "[sn:%lli]\n",
1007                loadQueue[load_idx]->pcState(),
1008                loadQueue[load_idx]->seqNum);
1009
1010        if (isStalled() && load_idx == stallingLoadIdx) {
1011            stalled = false;
1012            stallingStoreIsn = 0;
1013            stallingLoadIdx = 0;
1014        }
1015
1016        // Clear the smart pointer to make sure it is decremented.
1017        loadQueue[load_idx]->setSquashed();
1018        loadQueue[load_idx] = NULL;
1019        --loads;
1020
1021        // Inefficient!
1022        loadTail = load_idx;
1023
1024        decrLdIdx(load_idx);
1025        ++lsqSquashedLoads;
1026    }
1027
1028    if (memDepViolator && squashed_num < memDepViolator->seqNum) {
1029        memDepViolator = NULL;
1030    }
1031
1032    int store_idx = storeTail;
1033    decrStIdx(store_idx);
1034
1035    while (stores != 0 &&
1036           storeQueue[store_idx].inst->seqNum > squashed_num) {
1037        // Instructions marked as can WB are already committed.
1038        if (storeQueue[store_idx].canWB) {
1039            break;
1040        }
1041
1042        DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
1043                "idx:%i [sn:%lli]\n",
1044                storeQueue[store_idx].inst->pcState(),
1045                store_idx, storeQueue[store_idx].inst->seqNum);
1046
1047        // I don't think this can happen.  It should have been cleared
1048        // by the stalling load.
1049        if (isStalled() &&
1050            storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1051            panic("Is stalled should have been cleared by stalling load!\n");
1052            stalled = false;
1053            stallingStoreIsn = 0;
1054        }
1055
1056        // Clear the smart pointer to make sure it is decremented.
1057        storeQueue[store_idx].inst->setSquashed();
1058        storeQueue[store_idx].inst = NULL;
1059        storeQueue[store_idx].canWB = 0;
1060
1061        // Must delete request now that it wasn't handed off to
1062        // memory.  This is quite ugly.  @todo: Figure out the proper
1063        // place to really handle request deletes.
1064        delete storeQueue[store_idx].req;
1065        if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
1066            delete storeQueue[store_idx].sreqLow;
1067            delete storeQueue[store_idx].sreqHigh;
1068
1069            storeQueue[store_idx].sreqLow = NULL;
1070            storeQueue[store_idx].sreqHigh = NULL;
1071        }
1072
1073        storeQueue[store_idx].req = NULL;
1074        --stores;
1075
1076        // Inefficient!
1077        storeTail = store_idx;
1078
1079        decrStIdx(store_idx);
1080        ++lsqSquashedStores;
1081    }
1082}
1083
1084template <class Impl>
1085void
1086LSQUnit<Impl>::storePostSend(PacketPtr pkt)
1087{
1088    if (isStalled() &&
1089        storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
1090        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1091                "load idx:%i\n",
1092                stallingStoreIsn, stallingLoadIdx);
1093        stalled = false;
1094        stallingStoreIsn = 0;
1095        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1096    }
1097
1098    if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
1099        // The store is basically completed at this time. This
1100        // only works so long as the checker doesn't try to
1101        // verify the value in memory for stores.
1102        storeQueue[storeWBIdx].inst->setCompleted();
1103
1104        if (cpu->checker) {
1105            cpu->checker->verify(storeQueue[storeWBIdx].inst);
1106        }
1107    }
1108
1109    if (needsTSO) {
1110        storeInFlight = true;
1111    }
1112
1113    incrStIdx(storeWBIdx);
1114}
1115
1116template <class Impl>
1117void
1118LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
1119{
1120    iewStage->wakeCPU();
1121
1122    // Squashed instructions do not need to complete their access.
1123    if (inst->isSquashed()) {
1124        assert(!inst->isStore());
1125        ++lsqIgnoredResponses;
1126        return;
1127    }
1128
1129    if (!inst->isExecuted()) {
1130        inst->setExecuted();
1131
1132        if (inst->fault == NoFault) {
1133            // Complete access to copy data to proper place.
1134            inst->completeAcc(pkt);
1135        } else {
1136            // If the instruction has an outstanding fault, we cannot complete
1137            // the access as this discards the current fault.
1138
1139            // If we have an outstanding fault, the fault should only be of
1140            // type ReExec.
1141            assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr);
1142
1143            DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access "
1144                    "due to pending fault.\n", inst->seqNum);
1145        }
1146    }
1147
1148    // Need to insert instruction into queue to commit
1149    iewStage->instToCommit(inst);
1150
1151    iewStage->activityThisCycle();
1152
1153    // see if this load changed the PC
1154    iewStage->checkMisprediction(inst);
1155}
1156
1157template <class Impl>
1158void
1159LSQUnit<Impl>::completeStore(int store_idx)
1160{
1161    assert(storeQueue[store_idx].inst);
1162    storeQueue[store_idx].completed = true;
1163    --storesToWB;
1164    // A bit conservative because a store completion may not free up entries,
1165    // but hopefully avoids two store completions in one cycle from making
1166    // the CPU tick twice.
1167    cpu->wakeCPU();
1168    cpu->activityThisCycle();
1169
1170    if (store_idx == storeHead) {
1171        do {
1172            incrStIdx(storeHead);
1173
1174            --stores;
1175        } while (storeQueue[storeHead].completed &&
1176                 storeHead != storeTail);
1177
1178        iewStage->updateLSQNextCycle = true;
1179    }
1180
1181    DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
1182            "idx:%i\n",
1183            storeQueue[store_idx].inst->seqNum, store_idx, storeHead);
1184
1185#if TRACING_ON
1186    if (DTRACE(O3PipeView)) {
1187        storeQueue[store_idx].inst->storeTick =
1188            curTick() - storeQueue[store_idx].inst->fetchTick;
1189    }
1190#endif
1191
1192    if (isStalled() &&
1193        storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
1194        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
1195                "load idx:%i\n",
1196                stallingStoreIsn, stallingLoadIdx);
1197        stalled = false;
1198        stallingStoreIsn = 0;
1199        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
1200    }
1201
1202    storeQueue[store_idx].inst->setCompleted();
1203
1204    if (needsTSO) {
1205        storeInFlight = false;
1206    }
1207
1208    // Tell the checker we've completed this instruction.  Some stores
1209    // may get reported twice to the checker, but the checker can
1210    // handle that case.
1211    if (cpu->checker) {
1212        cpu->checker->verify(storeQueue[store_idx].inst);
1213    }
1214}
1215
1216template <class Impl>
1217bool
1218LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
1219{
1220    if (!dcachePort->sendTimingReq(data_pkt)) {
1221        // Need to handle becoming blocked on a store.
1222        isStoreBlocked = true;
1223        ++lsqCacheBlocked;
1224        assert(retryPkt == NULL);
1225        retryPkt = data_pkt;
1226        return false;
1227    }
1228    return true;
1229}
1230
1231template <class Impl>
1232void
1233LSQUnit<Impl>::recvRetry()
1234{
1235    if (isStoreBlocked) {
1236        DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
1237        assert(retryPkt != NULL);
1238
1239        LSQSenderState *state =
1240            dynamic_cast<LSQSenderState *>(retryPkt->senderState);
1241
1242        if (dcachePort->sendTimingReq(retryPkt)) {
1243            // Don't finish the store unless this is the last packet.
1244            if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
1245                    state->pendingPacket == retryPkt) {
1246                state->pktToSend = false;
1247                storePostSend(retryPkt);
1248            }
1249            retryPkt = NULL;
1250            isStoreBlocked = false;
1251
1252            // Send any outstanding packet.
1253            if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
1254                assert(state->pendingPacket);
1255                if (sendStore(state->pendingPacket)) {
1256                    storePostSend(state->pendingPacket);
1257                }
1258            }
1259        } else {
1260            // Still blocked!
1261            ++lsqCacheBlocked;
1262        }
1263    }
1264}
1265
1266template <class Impl>
1267inline void
1268LSQUnit<Impl>::incrStIdx(int &store_idx) const
1269{
1270    if (++store_idx >= SQEntries)
1271        store_idx = 0;
1272}
1273
1274template <class Impl>
1275inline void
1276LSQUnit<Impl>::decrStIdx(int &store_idx) const
1277{
1278    if (--store_idx < 0)
1279        store_idx += SQEntries;
1280}
1281
1282template <class Impl>
1283inline void
1284LSQUnit<Impl>::incrLdIdx(int &load_idx) const
1285{
1286    if (++load_idx >= LQEntries)
1287        load_idx = 0;
1288}
1289
1290template <class Impl>
1291inline void
1292LSQUnit<Impl>::decrLdIdx(int &load_idx) const
1293{
1294    if (--load_idx < 0)
1295        load_idx += LQEntries;
1296}
1297
1298template <class Impl>
1299void
1300LSQUnit<Impl>::dumpInsts() const
1301{
1302    cprintf("Load store queue: Dumping instructions.\n");
1303    cprintf("Load queue size: %i\n", loads);
1304    cprintf("Load queue: ");
1305
1306    int load_idx = loadHead;
1307
1308    while (load_idx != loadTail && loadQueue[load_idx]) {
1309        const DynInstPtr &inst(loadQueue[load_idx]);
1310        cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum);
1311
1312        incrLdIdx(load_idx);
1313    }
1314    cprintf("\n");
1315
1316    cprintf("Store queue size: %i\n", stores);
1317    cprintf("Store queue: ");
1318
1319    int store_idx = storeHead;
1320
1321    while (store_idx != storeTail && storeQueue[store_idx].inst) {
1322        const DynInstPtr &inst(storeQueue[store_idx].inst);
1323        cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum);
1324
1325        incrStIdx(store_idx);
1326    }
1327
1328    cprintf("\n");
1329}
1330
1331#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__
1332