lsq_unit_impl.hh revision 11356
12568SN/A 22568SN/A/* 32568SN/A * Copyright (c) 2010-2014 ARM Limited 42568SN/A * Copyright (c) 2013 Advanced Micro Devices, Inc. 52568SN/A * All rights reserved 62568SN/A * 72568SN/A * The license below extends only to copyright in the software and shall 82568SN/A * not be construed as granting a license to any other intellectual 92568SN/A * property including but not limited to intellectual property relating 102568SN/A * to a hardware implementation of the functionality of the software 112568SN/A * licensed hereunder. You may use the software subject to the license 122568SN/A * terms below provided that you ensure that this notice is replicated 132568SN/A * unmodified and in its entirety in all distributions of the software, 142568SN/A * modified or unmodified, in source code or in binary form. 152568SN/A * 162568SN/A * Copyright (c) 2004-2005 The Regents of The University of Michigan 172568SN/A * All rights reserved. 182568SN/A * 192568SN/A * Redistribution and use in source and binary forms, with or without 202568SN/A * modification, are permitted provided that the following conditions are 212568SN/A * met: redistributions of source code must retain the above copyright 222568SN/A * notice, this list of conditions and the following disclaimer; 232568SN/A * redistributions in binary form must reproduce the above copyright 242568SN/A * notice, this list of conditions and the following disclaimer in the 252568SN/A * documentation and/or other materials provided with the distribution; 262568SN/A * neither the name of the copyright holders nor the names of its 272568SN/A * contributors may be used to endorse or promote products derived from 282665Ssaidi@eecs.umich.edu * this software without specific prior written permission. 292665Ssaidi@eecs.umich.edu * 302665Ssaidi@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 312568SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 322568SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 332568SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 342982Sstever@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 352982Sstever@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 362568SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 372568SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 382643Sstever@eecs.umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 392568SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 402568SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 412568SN/A * 424762Snate@binkert.org * Authors: Kevin Lim 432568SN/A * Korey Sewell 442643Sstever@eecs.umich.edu */ 452643Sstever@eecs.umich.edu 464435Ssaidi@eecs.umich.edu#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 474435Ssaidi@eecs.umich.edu#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 482643Sstever@eecs.umich.edu 494435Ssaidi@eecs.umich.edu#include "arch/generic/debugfaults.hh" 504435Ssaidi@eecs.umich.edu#include "arch/locked_mem.hh" 514435Ssaidi@eecs.umich.edu#include "base/str.hh" 522643Sstever@eecs.umich.edu#include "config/the_isa.hh" 532643Sstever@eecs.umich.edu#include "cpu/checker/cpu.hh" 542643Sstever@eecs.umich.edu#include "cpu/o3/lsq.hh" 554435Ssaidi@eecs.umich.edu#include "cpu/o3/lsq_unit.hh" 564435Ssaidi@eecs.umich.edu#include "debug/Activity.hh" 574435Ssaidi@eecs.umich.edu#include "debug/IEW.hh" 584435Ssaidi@eecs.umich.edu#include "debug/LSQUnit.hh" 594435Ssaidi@eecs.umich.edu#include "debug/O3PipeView.hh" 604435Ssaidi@eecs.umich.edu#include "mem/packet.hh" 614435Ssaidi@eecs.umich.edu#include "mem/request.hh" 622643Sstever@eecs.umich.edu 634432Ssaidi@eecs.umich.edutemplate<class Impl> 644432Ssaidi@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt, 652643Sstever@eecs.umich.edu LSQUnit *lsq_ptr) 662643Sstever@eecs.umich.edu : Event(Default_Pri, AutoDelete), 672643Sstever@eecs.umich.edu inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 682738Sstever@eecs.umich.edu{ 692643Sstever@eecs.umich.edu} 702643Sstever@eecs.umich.edu 712643Sstever@eecs.umich.edutemplate<class Impl> 722643Sstever@eecs.umich.eduvoid 732643Sstever@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::process() 742643Sstever@eecs.umich.edu{ 752643Sstever@eecs.umich.edu assert(!lsqPtr->cpu->switchedOut()); 762643Sstever@eecs.umich.edu 772643Sstever@eecs.umich.edu lsqPtr->writeback(inst, pkt); 782643Sstever@eecs.umich.edu 792643Sstever@eecs.umich.edu if (pkt->senderState) 802643Sstever@eecs.umich.edu delete pkt->senderState; 812643Sstever@eecs.umich.edu 822643Sstever@eecs.umich.edu delete pkt->req; 832643Sstever@eecs.umich.edu delete pkt; 842643Sstever@eecs.umich.edu} 852568SN/A 862568SN/Atemplate<class Impl> 872568SN/Aconst char * 882568SN/ALSQUnit<Impl>::WritebackEvent::description() const 892643Sstever@eecs.umich.edu{ 904432Ssaidi@eecs.umich.edu return "Store writeback"; 914432Ssaidi@eecs.umich.edu} 924432Ssaidi@eecs.umich.edu 934432Ssaidi@eecs.umich.edutemplate<class Impl> 942568SN/Avoid 952568SN/ALSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 964433Ssaidi@eecs.umich.edu{ 974435Ssaidi@eecs.umich.edu LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 984433Ssaidi@eecs.umich.edu DynInstPtr inst = state->inst; 994435Ssaidi@eecs.umich.edu DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum); 1004435Ssaidi@eecs.umich.edu DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum); 1014435Ssaidi@eecs.umich.edu 1024435Ssaidi@eecs.umich.edu if (state->cacheBlocked) { 1034435Ssaidi@eecs.umich.edu // This is the first half of a previous split load, 1044435Ssaidi@eecs.umich.edu // where the 2nd half blocked, ignore this response 1054435Ssaidi@eecs.umich.edu DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier " 1064435Ssaidi@eecs.umich.edu "blocked split load recieved. Ignoring.\n", inst->seqNum); 1074435Ssaidi@eecs.umich.edu delete state; 1084433Ssaidi@eecs.umich.edu return; 1092568SN/A } 1102643Sstever@eecs.umich.edu 1112568SN/A // If this is a split access, wait until all packets are received. 1122568SN/A if (TheISA::HasUnalignedMemAcc && !state->complete()) { 1133349Sbinkertn@umich.edu return; 1142568SN/A } 1154433Ssaidi@eecs.umich.edu 1164433Ssaidi@eecs.umich.edu assert(!cpu->switchedOut()); 1174433Ssaidi@eecs.umich.edu if (!inst->isSquashed()) { 1184433Ssaidi@eecs.umich.edu if (!state->noWB) { 1194433Ssaidi@eecs.umich.edu if (!TheISA::HasUnalignedMemAcc || !state->isSplit || 1203662Srdreslin@umich.edu !state->isLoad) { 1212643Sstever@eecs.umich.edu writeback(inst, pkt); 1224450Ssaidi@eecs.umich.edu } else { 1234450Ssaidi@eecs.umich.edu writeback(inst, state->mainPkt); 1244450Ssaidi@eecs.umich.edu } 1254450Ssaidi@eecs.umich.edu } 1264450Ssaidi@eecs.umich.edu 1274450Ssaidi@eecs.umich.edu if (inst->isStore()) { 1284450Ssaidi@eecs.umich.edu completeStore(state->idx); 1294450Ssaidi@eecs.umich.edu } 1304433Ssaidi@eecs.umich.edu } 1314433Ssaidi@eecs.umich.edu 1324433Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) { 1333662Srdreslin@umich.edu delete state->mainPkt->req; 1344433Ssaidi@eecs.umich.edu delete state->mainPkt; 1354433Ssaidi@eecs.umich.edu } 1364435Ssaidi@eecs.umich.edu 1374433Ssaidi@eecs.umich.edu pkt->req->setAccessLatency(); 1384433Ssaidi@eecs.umich.edu cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1394433Ssaidi@eecs.umich.edu 1404433Ssaidi@eecs.umich.edu delete state; 1414433Ssaidi@eecs.umich.edu} 1424433Ssaidi@eecs.umich.edu 1434433Ssaidi@eecs.umich.edutemplate <class Impl> 1444433Ssaidi@eecs.umich.eduLSQUnit<Impl>::LSQUnit() 1454433Ssaidi@eecs.umich.edu : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 1464433Ssaidi@eecs.umich.edu isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false), 1474433Ssaidi@eecs.umich.edu pendingPkt(nullptr) 1484433Ssaidi@eecs.umich.edu{ 1494433Ssaidi@eecs.umich.edu} 1502657Ssaidi@eecs.umich.edu 1512657Ssaidi@eecs.umich.edutemplate<class Impl> 1524433Ssaidi@eecs.umich.eduvoid 1534433Ssaidi@eecs.umich.eduLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 1544433Ssaidi@eecs.umich.edu LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 1554433Ssaidi@eecs.umich.edu unsigned id) 1564433Ssaidi@eecs.umich.edu{ 1574433Ssaidi@eecs.umich.edu cpu = cpu_ptr; 1582657Ssaidi@eecs.umich.edu iewStage = iew_ptr; 1594433Ssaidi@eecs.umich.edu 1604435Ssaidi@eecs.umich.edu lsq = lsq_ptr; 1614433Ssaidi@eecs.umich.edu 1624435Ssaidi@eecs.umich.edu lsqID = id; 1634435Ssaidi@eecs.umich.edu 1644433Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id); 1654435Ssaidi@eecs.umich.edu 1664433Ssaidi@eecs.umich.edu // Add 1 for the sentinel entry (they are circular queues). 1674435Ssaidi@eecs.umich.edu LQEntries = maxLQEntries + 1; 1684435Ssaidi@eecs.umich.edu SQEntries = maxSQEntries + 1; 1694433Ssaidi@eecs.umich.edu 1704435Ssaidi@eecs.umich.edu //Due to uint8_t index in LSQSenderState 1714435Ssaidi@eecs.umich.edu assert(LQEntries <= 256); 1724435Ssaidi@eecs.umich.edu assert(SQEntries <= 256); 1734435Ssaidi@eecs.umich.edu 1744435Ssaidi@eecs.umich.edu loadQueue.resize(LQEntries); 1754435Ssaidi@eecs.umich.edu storeQueue.resize(SQEntries); 1764435Ssaidi@eecs.umich.edu 1774435Ssaidi@eecs.umich.edu depCheckShift = params->LSQDepCheckShift; 1784435Ssaidi@eecs.umich.edu checkLoads = params->LSQCheckLoads; 1794435Ssaidi@eecs.umich.edu cachePorts = params->cachePorts; 1804435Ssaidi@eecs.umich.edu needsTSO = params->needsTSO; 1814435Ssaidi@eecs.umich.edu 1824435Ssaidi@eecs.umich.edu resetState(); 1834435Ssaidi@eecs.umich.edu} 1844435Ssaidi@eecs.umich.edu 1854435Ssaidi@eecs.umich.edu 1864435Ssaidi@eecs.umich.edutemplate<class Impl> 1874435Ssaidi@eecs.umich.eduvoid 1884435Ssaidi@eecs.umich.eduLSQUnit<Impl>::resetState() 1894435Ssaidi@eecs.umich.edu{ 1904435Ssaidi@eecs.umich.edu loads = stores = storesToWB = 0; 1914435Ssaidi@eecs.umich.edu 1924435Ssaidi@eecs.umich.edu loadHead = loadTail = 0; 1934435Ssaidi@eecs.umich.edu 1944435Ssaidi@eecs.umich.edu storeHead = storeWBIdx = storeTail = 0; 1954433Ssaidi@eecs.umich.edu 1964433Ssaidi@eecs.umich.edu usedPorts = 0; 1974433Ssaidi@eecs.umich.edu 1984433Ssaidi@eecs.umich.edu retryPkt = NULL; 1993349Sbinkertn@umich.edu memDepViolator = NULL; 2002657Ssaidi@eecs.umich.edu 2014450Ssaidi@eecs.umich.edu stalled = false; 2022643Sstever@eecs.umich.edu 2032643Sstever@eecs.umich.edu cacheBlockMask = ~(cpu->cacheLineSize() - 1); 2042643Sstever@eecs.umich.edu} 2052643Sstever@eecs.umich.edu 2062643Sstever@eecs.umich.edutemplate<class Impl> 2072643Sstever@eecs.umich.edustd::string 2082643Sstever@eecs.umich.eduLSQUnit<Impl>::name() const 2092643Sstever@eecs.umich.edu{ 2104433Ssaidi@eecs.umich.edu if (Impl::MaxThreads == 1) { 2114450Ssaidi@eecs.umich.edu return iewStage->name() + ".lsq"; 2124450Ssaidi@eecs.umich.edu } else { 2134450Ssaidi@eecs.umich.edu return iewStage->name() + ".lsq.thread" + std::to_string(lsqID); 2144433Ssaidi@eecs.umich.edu } 2154433Ssaidi@eecs.umich.edu} 2164739Sstever@eecs.umich.edu 2172643Sstever@eecs.umich.edutemplate<class Impl> 2182643Sstever@eecs.umich.eduvoid 2192643Sstever@eecs.umich.eduLSQUnit<Impl>::regStats() 2204450Ssaidi@eecs.umich.edu{ 2214450Ssaidi@eecs.umich.edu lsqForwLoads 2224450Ssaidi@eecs.umich.edu .name(name() + ".forwLoads") 2234450Ssaidi@eecs.umich.edu .desc("Number of loads that had data forwarded from stores"); 2244450Ssaidi@eecs.umich.edu 2254450Ssaidi@eecs.umich.edu invAddrLoads 2264450Ssaidi@eecs.umich.edu .name(name() + ".invAddrLoads") 2272643Sstever@eecs.umich.edu .desc("Number of loads ignored due to an invalid address"); 2282643Sstever@eecs.umich.edu 2292643Sstever@eecs.umich.edu lsqSquashedLoads 2302643Sstever@eecs.umich.edu .name(name() + ".squashedLoads") 2312643Sstever@eecs.umich.edu .desc("Number of loads squashed"); 2322643Sstever@eecs.umich.edu 2332643Sstever@eecs.umich.edu lsqIgnoredResponses 2342643Sstever@eecs.umich.edu .name(name() + ".ignoredResponses") 2352643Sstever@eecs.umich.edu .desc("Number of memory responses ignored because the instruction is squashed"); 2362568SN/A 2372643Sstever@eecs.umich.edu lsqMemOrderViolation 2382568SN/A .name(name() + ".memOrderViolation") 2392568SN/A .desc("Number of memory ordering violations"); 2402568SN/A 2412643Sstever@eecs.umich.edu lsqSquashedStores 2422568SN/A .name(name() + ".squashedStores") 2432643Sstever@eecs.umich.edu .desc("Number of stores squashed"); 2442568SN/A 2452643Sstever@eecs.umich.edu invAddrSwpfs 2462643Sstever@eecs.umich.edu .name(name() + ".invAddrSwpfs") 2472643Sstever@eecs.umich.edu .desc("Number of software prefetches ignored due to an invalid address"); 2482643Sstever@eecs.umich.edu 2493349Sbinkertn@umich.edu lsqBlockedLoads 2502643Sstever@eecs.umich.edu .name(name() + ".blockedLoads") 2514432Ssaidi@eecs.umich.edu .desc("Number of blocked loads due to partial load-store forwarding"); 2524432Ssaidi@eecs.umich.edu 2534454Ssaidi@eecs.umich.edu lsqRescheduledLoads 2544432Ssaidi@eecs.umich.edu .name(name() + ".rescheduledLoads") 2554454Ssaidi@eecs.umich.edu .desc("Number of loads that were rescheduled"); 2564454Ssaidi@eecs.umich.edu 2574454Ssaidi@eecs.umich.edu lsqCacheBlocked 2584454Ssaidi@eecs.umich.edu .name(name() + ".cacheBlocked") 2594454Ssaidi@eecs.umich.edu .desc("Number of times an access to memory failed due to the cache being blocked"); 2604454Ssaidi@eecs.umich.edu} 2614454Ssaidi@eecs.umich.edu 2624432Ssaidi@eecs.umich.edutemplate<class Impl> 2634432Ssaidi@eecs.umich.eduvoid 2642643Sstever@eecs.umich.eduLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2652643Sstever@eecs.umich.edu{ 2662643Sstever@eecs.umich.edu dcachePort = dcache_port; 2674450Ssaidi@eecs.umich.edu} 2684450Ssaidi@eecs.umich.edu 2694432Ssaidi@eecs.umich.edutemplate<class Impl> 2702643Sstever@eecs.umich.eduvoid 2712643Sstever@eecs.umich.eduLSQUnit<Impl>::clearLQ() 2722643Sstever@eecs.umich.edu{ 2732643Sstever@eecs.umich.edu loadQueue.clear(); 2742657Ssaidi@eecs.umich.edu} 2752657Ssaidi@eecs.umich.edu 2764433Ssaidi@eecs.umich.edutemplate<class Impl> 2772657Ssaidi@eecs.umich.eduvoid 2782657Ssaidi@eecs.umich.eduLSQUnit<Impl>::clearSQ() 2792657Ssaidi@eecs.umich.edu{ 2802657Ssaidi@eecs.umich.edu storeQueue.clear(); 2812657Ssaidi@eecs.umich.edu} 2822657Ssaidi@eecs.umich.edu 2832657Ssaidi@eecs.umich.edutemplate<class Impl> 2842657Ssaidi@eecs.umich.eduvoid 2854450Ssaidi@eecs.umich.eduLSQUnit<Impl>::drainSanityCheck() const 2864450Ssaidi@eecs.umich.edu{ 2874433Ssaidi@eecs.umich.edu for (int i = 0; i < loadQueue.size(); ++i) 2884450Ssaidi@eecs.umich.edu assert(!loadQueue[i]); 2894450Ssaidi@eecs.umich.edu 2904450Ssaidi@eecs.umich.edu assert(storesToWB == 0); 2914433Ssaidi@eecs.umich.edu assert(!retryPkt); 2922657Ssaidi@eecs.umich.edu} 2932657Ssaidi@eecs.umich.edu 2942657Ssaidi@eecs.umich.edutemplate<class Impl> 2954433Ssaidi@eecs.umich.eduvoid 2962657Ssaidi@eecs.umich.eduLSQUnit<Impl>::takeOverFrom() 2972657Ssaidi@eecs.umich.edu{ 2982643Sstever@eecs.umich.edu resetState(); 2992643Sstever@eecs.umich.edu} 3004435Ssaidi@eecs.umich.edu 3012643Sstever@eecs.umich.edutemplate<class Impl> 3024433Ssaidi@eecs.umich.eduvoid 3034433Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeLQ(unsigned size) 3042568SN/A{ 3052568SN/A unsigned size_plus_sentinel = size + 1; 3062568SN/A assert(size_plus_sentinel >= LQEntries); 3072657Ssaidi@eecs.umich.edu 3082568SN/A if (size_plus_sentinel > LQEntries) { 3092568SN/A while (size_plus_sentinel > loadQueue.size()) { 3104435Ssaidi@eecs.umich.edu DynInstPtr dummy; 3114435Ssaidi@eecs.umich.edu loadQueue.push_back(dummy); 3124435Ssaidi@eecs.umich.edu LQEntries++; 3134435Ssaidi@eecs.umich.edu } 3144435Ssaidi@eecs.umich.edu } else { 3154435Ssaidi@eecs.umich.edu LQEntries = size_plus_sentinel; 3162568SN/A } 3172568SN/A 3182643Sstever@eecs.umich.edu assert(LQEntries <= 256); 3192568SN/A} 3202568SN/A 3213349Sbinkertn@umich.edutemplate<class Impl> 3222568SN/Avoid 3234450Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeSQ(unsigned size) 3244454Ssaidi@eecs.umich.edu{ 3254454Ssaidi@eecs.umich.edu unsigned size_plus_sentinel = size + 1; 3264450Ssaidi@eecs.umich.edu if (size_plus_sentinel > SQEntries) { 3274454Ssaidi@eecs.umich.edu while (size_plus_sentinel > storeQueue.size()) { 3284454Ssaidi@eecs.umich.edu SQEntry dummy; 3294454Ssaidi@eecs.umich.edu storeQueue.push_back(dummy); 3304450Ssaidi@eecs.umich.edu SQEntries++; 3314450Ssaidi@eecs.umich.edu } 3324454Ssaidi@eecs.umich.edu } else { 3334450Ssaidi@eecs.umich.edu SQEntries = size_plus_sentinel; 3344454Ssaidi@eecs.umich.edu } 3352568SN/A 3362568SN/A assert(SQEntries <= 256); 3372643Sstever@eecs.umich.edu} 3382568SN/A 3392568SN/Atemplate <class Impl> 3403349Sbinkertn@umich.eduvoid 3412568SN/ALSQUnit<Impl>::insert(DynInstPtr &inst) 3422643Sstever@eecs.umich.edu{ 3432568SN/A assert(inst->isMemRef()); 3442568SN/A 3452643Sstever@eecs.umich.edu assert(inst->isLoad() || inst->isStore()); 3462643Sstever@eecs.umich.edu 3472643Sstever@eecs.umich.edu if (inst->isLoad()) { 3482568SN/A insertLoad(inst); 3492568SN/A } else { 3502568SN/A insertStore(inst); 3512568SN/A } 3522643Sstever@eecs.umich.edu 3532568SN/A inst->setInLSQ(); 3542568SN/A} 3552568SN/A 3562643Sstever@eecs.umich.edutemplate <class Impl> 3572568SN/Avoid 3582643Sstever@eecs.umich.eduLSQUnit<Impl>::insertLoad(DynInstPtr &load_inst) 3592568SN/A{ 3602643Sstever@eecs.umich.edu assert((loadTail + 1) % LQEntries != loadHead); 3612568SN/A assert(loads < LQEntries); 3622568SN/A 3632568SN/A DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 3642643Sstever@eecs.umich.edu load_inst->pcState(), loadTail, load_inst->seqNum); 3654475Sstever@eecs.umich.edu 3662568SN/A load_inst->lqIdx = loadTail; 3672643Sstever@eecs.umich.edu 3682568SN/A if (stores == 0) { 3692568SN/A load_inst->sqIdx = -1; 3704762Snate@binkert.org } else { 3714762Snate@binkert.org load_inst->sqIdx = storeTail; 3722568SN/A } 3734762Snate@binkert.org 3742568SN/A loadQueue[loadTail] = load_inst; 375 376 incrLdIdx(loadTail); 377 378 ++loads; 379} 380 381template <class Impl> 382void 383LSQUnit<Impl>::insertStore(DynInstPtr &store_inst) 384{ 385 // Make sure it is not full before inserting an instruction. 386 assert((storeTail + 1) % SQEntries != storeHead); 387 assert(stores < SQEntries); 388 389 DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 390 store_inst->pcState(), storeTail, store_inst->seqNum); 391 392 store_inst->sqIdx = storeTail; 393 store_inst->lqIdx = loadTail; 394 395 storeQueue[storeTail] = SQEntry(store_inst); 396 397 incrStIdx(storeTail); 398 399 ++stores; 400} 401 402template <class Impl> 403typename Impl::DynInstPtr 404LSQUnit<Impl>::getMemDepViolator() 405{ 406 DynInstPtr temp = memDepViolator; 407 408 memDepViolator = NULL; 409 410 return temp; 411} 412 413template <class Impl> 414unsigned 415LSQUnit<Impl>::numFreeLoadEntries() 416{ 417 //LQ has an extra dummy entry to differentiate 418 //empty/full conditions. Subtract 1 from the free entries. 419 DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads); 420 return LQEntries - loads - 1; 421} 422 423template <class Impl> 424unsigned 425LSQUnit<Impl>::numFreeStoreEntries() 426{ 427 //SQ has an extra dummy entry to differentiate 428 //empty/full conditions. Subtract 1 from the free entries. 429 DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores); 430 return SQEntries - stores - 1; 431 432 } 433 434template <class Impl> 435void 436LSQUnit<Impl>::checkSnoop(PacketPtr pkt) 437{ 438 int load_idx = loadHead; 439 DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 440 441 // Only Invalidate packet calls checkSnoop 442 assert(pkt->isInvalidate()); 443 for (int x = 0; x < cpu->numContexts(); x++) { 444 ThreadContext *tc = cpu->getContext(x); 445 bool no_squash = cpu->thread[x]->noSquashFromTC; 446 cpu->thread[x]->noSquashFromTC = true; 447 TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 448 cpu->thread[x]->noSquashFromTC = no_squash; 449 } 450 451 Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 452 453 DynInstPtr ld_inst = loadQueue[load_idx]; 454 if (ld_inst) { 455 Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask; 456 Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask; 457 458 // Check that this snoop didn't just invalidate our lock flag 459 if (ld_inst->effAddrValid() && (load_addr_low == invalidate_addr 460 || load_addr_high == invalidate_addr) 461 && ld_inst->memReqFlags & Request::LLSC) 462 TheISA::handleLockedSnoopHit(ld_inst.get()); 463 } 464 465 // If this is the only load in the LSQ we don't care 466 if (load_idx == loadTail) 467 return; 468 469 incrLdIdx(load_idx); 470 471 bool force_squash = false; 472 473 while (load_idx != loadTail) { 474 DynInstPtr ld_inst = loadQueue[load_idx]; 475 476 if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) { 477 incrLdIdx(load_idx); 478 continue; 479 } 480 481 Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask; 482 Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask; 483 484 DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n", 485 ld_inst->seqNum, load_addr_low, invalidate_addr); 486 487 if ((load_addr_low == invalidate_addr 488 || load_addr_high == invalidate_addr) || force_squash) { 489 if (needsTSO) { 490 // If we have a TSO system, as all loads must be ordered with 491 // all other loads, this load as well as *all* subsequent loads 492 // need to be squashed to prevent possible load reordering. 493 force_squash = true; 494 } 495 if (ld_inst->possibleLoadViolation() || force_squash) { 496 DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 497 pkt->getAddr(), ld_inst->seqNum); 498 499 // Mark the load for re-execution 500 ld_inst->fault = std::make_shared<ReExec>(); 501 } else { 502 DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 503 pkt->getAddr(), ld_inst->seqNum); 504 505 // Make sure that we don't lose a snoop hitting a LOCKED 506 // address since the LOCK* flags don't get updated until 507 // commit. 508 if (ld_inst->memReqFlags & Request::LLSC) 509 TheISA::handleLockedSnoopHit(ld_inst.get()); 510 511 // If a older load checks this and it's true 512 // then we might have missed the snoop 513 // in which case we need to invalidate to be sure 514 ld_inst->hitExternalSnoop(true); 515 } 516 } 517 incrLdIdx(load_idx); 518 } 519 return; 520} 521 522template <class Impl> 523Fault 524LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst) 525{ 526 Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 527 Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 528 529 /** @todo in theory you only need to check an instruction that has executed 530 * however, there isn't a good way in the pipeline at the moment to check 531 * all instructions that will execute before the store writes back. Thus, 532 * like the implementation that came before it, we're overly conservative. 533 */ 534 while (load_idx != loadTail) { 535 DynInstPtr ld_inst = loadQueue[load_idx]; 536 if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) { 537 incrLdIdx(load_idx); 538 continue; 539 } 540 541 Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 542 Addr ld_eff_addr2 = 543 (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 544 545 if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 546 if (inst->isLoad()) { 547 // If this load is to the same block as an external snoop 548 // invalidate that we've observed then the load needs to be 549 // squashed as it could have newer data 550 if (ld_inst->hitExternalSnoop()) { 551 if (!memDepViolator || 552 ld_inst->seqNum < memDepViolator->seqNum) { 553 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 554 "and [sn:%lli] at address %#x\n", 555 inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 556 memDepViolator = ld_inst; 557 558 ++lsqMemOrderViolation; 559 560 return std::make_shared<GenericISA::M5PanicFault>( 561 "Detected fault with inst [sn:%lli] and " 562 "[sn:%lli] at address %#x\n", 563 inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 564 } 565 } 566 567 // Otherwise, mark the load has a possible load violation 568 // and if we see a snoop before it's commited, we need to squash 569 ld_inst->possibleLoadViolation(true); 570 DPRINTF(LSQUnit, "Found possible load violation at addr: %#x" 571 " between instructions [sn:%lli] and [sn:%lli]\n", 572 inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 573 } else { 574 // A load/store incorrectly passed this store. 575 // Check if we already have a violator, or if it's newer 576 // squash and refetch. 577 if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 578 break; 579 580 DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 581 "[sn:%lli] at address %#x\n", 582 inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 583 memDepViolator = ld_inst; 584 585 ++lsqMemOrderViolation; 586 587 return std::make_shared<GenericISA::M5PanicFault>( 588 "Detected fault with " 589 "inst [sn:%lli] and [sn:%lli] at address %#x\n", 590 inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 591 } 592 } 593 594 incrLdIdx(load_idx); 595 } 596 return NoFault; 597} 598 599 600 601 602template <class Impl> 603Fault 604LSQUnit<Impl>::executeLoad(DynInstPtr &inst) 605{ 606 using namespace TheISA; 607 // Execute a specific load. 608 Fault load_fault = NoFault; 609 610 DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 611 inst->pcState(), inst->seqNum); 612 613 assert(!inst->isSquashed()); 614 615 load_fault = inst->initiateAcc(); 616 617 if (inst->isTranslationDelayed() && 618 load_fault == NoFault) 619 return load_fault; 620 621 // If the instruction faulted or predicated false, then we need to send it 622 // along to commit without the instruction completing. 623 if (load_fault != NoFault || !inst->readPredicate()) { 624 // Send this instruction to commit, also make sure iew stage 625 // realizes there is activity. Mark it as executed unless it 626 // is a strictly ordered load that needs to hit the head of 627 // commit. 628 if (!inst->readPredicate()) 629 inst->forwardOldRegs(); 630 DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 631 inst->seqNum, 632 (load_fault != NoFault ? "fault" : "predication")); 633 if (!(inst->hasRequest() && inst->strictlyOrdered()) || 634 inst->isAtCommit()) { 635 inst->setExecuted(); 636 } 637 iewStage->instToCommit(inst); 638 iewStage->activityThisCycle(); 639 } else { 640 assert(inst->effAddrValid()); 641 int load_idx = inst->lqIdx; 642 incrLdIdx(load_idx); 643 644 if (checkLoads) 645 return checkViolations(load_idx, inst); 646 } 647 648 return load_fault; 649} 650 651template <class Impl> 652Fault 653LSQUnit<Impl>::executeStore(DynInstPtr &store_inst) 654{ 655 using namespace TheISA; 656 // Make sure that a store exists. 657 assert(stores != 0); 658 659 int store_idx = store_inst->sqIdx; 660 661 DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 662 store_inst->pcState(), store_inst->seqNum); 663 664 assert(!store_inst->isSquashed()); 665 666 // Check the recently completed loads to see if any match this store's 667 // address. If so, then we have a memory ordering violation. 668 int load_idx = store_inst->lqIdx; 669 670 Fault store_fault = store_inst->initiateAcc(); 671 672 if (store_inst->isTranslationDelayed() && 673 store_fault == NoFault) 674 return store_fault; 675 676 if (!store_inst->readPredicate()) 677 store_inst->forwardOldRegs(); 678 679 if (storeQueue[store_idx].size == 0) { 680 DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 681 store_inst->pcState(), store_inst->seqNum); 682 683 return store_fault; 684 } else if (!store_inst->readPredicate()) { 685 DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 686 store_inst->seqNum); 687 return store_fault; 688 } 689 690 assert(store_fault == NoFault); 691 692 if (store_inst->isStoreConditional()) { 693 // Store conditionals need to set themselves as able to 694 // writeback if we haven't had a fault by here. 695 storeQueue[store_idx].canWB = true; 696 697 ++storesToWB; 698 } 699 700 return checkViolations(load_idx, store_inst); 701 702} 703 704template <class Impl> 705void 706LSQUnit<Impl>::commitLoad() 707{ 708 assert(loadQueue[loadHead]); 709 710 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 711 loadQueue[loadHead]->pcState()); 712 713 loadQueue[loadHead] = NULL; 714 715 incrLdIdx(loadHead); 716 717 --loads; 718} 719 720template <class Impl> 721void 722LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 723{ 724 assert(loads == 0 || loadQueue[loadHead]); 725 726 while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) { 727 commitLoad(); 728 } 729} 730 731template <class Impl> 732void 733LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 734{ 735 assert(stores == 0 || storeQueue[storeHead].inst); 736 737 int store_idx = storeHead; 738 739 while (store_idx != storeTail) { 740 assert(storeQueue[store_idx].inst); 741 // Mark any stores that are now committed and have not yet 742 // been marked as able to write back. 743 if (!storeQueue[store_idx].canWB) { 744 if (storeQueue[store_idx].inst->seqNum > youngest_inst) { 745 break; 746 } 747 DPRINTF(LSQUnit, "Marking store as able to write back, PC " 748 "%s [sn:%lli]\n", 749 storeQueue[store_idx].inst->pcState(), 750 storeQueue[store_idx].inst->seqNum); 751 752 storeQueue[store_idx].canWB = true; 753 754 ++storesToWB; 755 } 756 757 incrStIdx(store_idx); 758 } 759} 760 761template <class Impl> 762void 763LSQUnit<Impl>::writebackPendingStore() 764{ 765 if (hasPendingPkt) { 766 assert(pendingPkt != NULL); 767 768 // If the cache is blocked, this will store the packet for retry. 769 if (sendStore(pendingPkt)) { 770 storePostSend(pendingPkt); 771 } 772 pendingPkt = NULL; 773 hasPendingPkt = false; 774 } 775} 776 777template <class Impl> 778void 779LSQUnit<Impl>::writebackStores() 780{ 781 // First writeback the second packet from any split store that didn't 782 // complete last cycle because there weren't enough cache ports available. 783 if (TheISA::HasUnalignedMemAcc) { 784 writebackPendingStore(); 785 } 786 787 while (storesToWB > 0 && 788 storeWBIdx != storeTail && 789 storeQueue[storeWBIdx].inst && 790 storeQueue[storeWBIdx].canWB && 791 ((!needsTSO) || (!storeInFlight)) && 792 usedPorts < cachePorts) { 793 794 if (isStoreBlocked) { 795 DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 796 " is blocked!\n"); 797 break; 798 } 799 800 // Store didn't write any data so no need to write it back to 801 // memory. 802 if (storeQueue[storeWBIdx].size == 0) { 803 completeStore(storeWBIdx); 804 805 incrStIdx(storeWBIdx); 806 807 continue; 808 } 809 810 ++usedPorts; 811 812 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) { 813 incrStIdx(storeWBIdx); 814 815 continue; 816 } 817 818 assert(storeQueue[storeWBIdx].req); 819 assert(!storeQueue[storeWBIdx].committed); 820 821 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) { 822 assert(storeQueue[storeWBIdx].sreqLow); 823 assert(storeQueue[storeWBIdx].sreqHigh); 824 } 825 826 DynInstPtr inst = storeQueue[storeWBIdx].inst; 827 828 Request *req = storeQueue[storeWBIdx].req; 829 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow; 830 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh; 831 832 storeQueue[storeWBIdx].committed = true; 833 834 assert(!inst->memData); 835 inst->memData = new uint8_t[req->getSize()]; 836 837 if (storeQueue[storeWBIdx].isAllZeros) 838 memset(inst->memData, 0, req->getSize()); 839 else 840 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize()); 841 842 PacketPtr data_pkt; 843 PacketPtr snd_data_pkt = NULL; 844 845 LSQSenderState *state = new LSQSenderState; 846 state->isLoad = false; 847 state->idx = storeWBIdx; 848 state->inst = inst; 849 850 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) { 851 852 // Build a single data packet if the store isn't split. 853 data_pkt = Packet::createWrite(req); 854 data_pkt->dataStatic(inst->memData); 855 data_pkt->senderState = state; 856 } else { 857 // Create two packets if the store is split in two. 858 data_pkt = Packet::createWrite(sreqLow); 859 snd_data_pkt = Packet::createWrite(sreqHigh); 860 861 data_pkt->dataStatic(inst->memData); 862 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize()); 863 864 data_pkt->senderState = state; 865 snd_data_pkt->senderState = state; 866 867 state->isSplit = true; 868 state->outstanding = 2; 869 870 // Can delete the main request now. 871 delete req; 872 req = sreqLow; 873 } 874 875 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 876 "to Addr:%#x, data:%#x [sn:%lli]\n", 877 storeWBIdx, inst->pcState(), 878 req->getPaddr(), (int)*(inst->memData), 879 inst->seqNum); 880 881 // @todo: Remove this SC hack once the memory system handles it. 882 if (inst->isStoreConditional()) { 883 assert(!storeQueue[storeWBIdx].isSplit); 884 // Disable recording the result temporarily. Writing to 885 // misc regs normally updates the result, but this is not 886 // the desired behavior when handling store conditionals. 887 inst->recordResult(false); 888 bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask); 889 inst->recordResult(true); 890 891 if (!success) { 892 // Instantly complete this store. 893 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 894 "Instantly completing it.\n", 895 inst->seqNum); 896 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this); 897 cpu->schedule(wb, curTick() + 1); 898 if (cpu->checker) { 899 // Make sure to set the LLSC data for verification 900 // if checker is loaded 901 inst->reqToVerify->setExtraData(0); 902 inst->completeAcc(data_pkt); 903 } 904 completeStore(storeWBIdx); 905 incrStIdx(storeWBIdx); 906 continue; 907 } 908 } else { 909 // Non-store conditionals do not need a writeback. 910 state->noWB = true; 911 } 912 913 bool split = 914 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit; 915 916 ThreadContext *thread = cpu->tcBase(lsqID); 917 918 if (req->isMmappedIpr()) { 919 assert(!inst->isStoreConditional()); 920 TheISA::handleIprWrite(thread, data_pkt); 921 delete data_pkt; 922 if (split) { 923 assert(snd_data_pkt->req->isMmappedIpr()); 924 TheISA::handleIprWrite(thread, snd_data_pkt); 925 delete snd_data_pkt; 926 delete sreqLow; 927 delete sreqHigh; 928 } 929 delete state; 930 delete req; 931 completeStore(storeWBIdx); 932 incrStIdx(storeWBIdx); 933 } else if (!sendStore(data_pkt)) { 934 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will" 935 "retry later\n", 936 inst->seqNum); 937 938 // Need to store the second packet, if split. 939 if (split) { 940 state->pktToSend = true; 941 state->pendingPacket = snd_data_pkt; 942 } 943 } else { 944 945 // If split, try to send the second packet too 946 if (split) { 947 assert(snd_data_pkt); 948 949 // Ensure there are enough ports to use. 950 if (usedPorts < cachePorts) { 951 ++usedPorts; 952 if (sendStore(snd_data_pkt)) { 953 storePostSend(snd_data_pkt); 954 } else { 955 DPRINTF(IEW, "D-Cache became blocked when writing" 956 " [sn:%lli] second packet, will retry later\n", 957 inst->seqNum); 958 } 959 } else { 960 961 // Store the packet for when there's free ports. 962 assert(pendingPkt == NULL); 963 pendingPkt = snd_data_pkt; 964 hasPendingPkt = true; 965 } 966 } else { 967 968 // Not a split store. 969 storePostSend(data_pkt); 970 } 971 } 972 } 973 974 // Not sure this should set it to 0. 975 usedPorts = 0; 976 977 assert(stores >= 0 && storesToWB >= 0); 978} 979 980/*template <class Impl> 981void 982LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum) 983{ 984 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(), 985 mshrSeqNums.end(), 986 seqNum); 987 988 if (mshr_it != mshrSeqNums.end()) { 989 mshrSeqNums.erase(mshr_it); 990 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size()); 991 } 992}*/ 993 994template <class Impl> 995void 996LSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 997{ 998 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 999 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 1000 1001 int load_idx = loadTail; 1002 decrLdIdx(load_idx); 1003 1004 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) { 1005 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 1006 "[sn:%lli]\n", 1007 loadQueue[load_idx]->pcState(), 1008 loadQueue[load_idx]->seqNum); 1009 1010 if (isStalled() && load_idx == stallingLoadIdx) { 1011 stalled = false; 1012 stallingStoreIsn = 0; 1013 stallingLoadIdx = 0; 1014 } 1015 1016 // Clear the smart pointer to make sure it is decremented. 1017 loadQueue[load_idx]->setSquashed(); 1018 loadQueue[load_idx] = NULL; 1019 --loads; 1020 1021 // Inefficient! 1022 loadTail = load_idx; 1023 1024 decrLdIdx(load_idx); 1025 ++lsqSquashedLoads; 1026 } 1027 1028 if (memDepViolator && squashed_num < memDepViolator->seqNum) { 1029 memDepViolator = NULL; 1030 } 1031 1032 int store_idx = storeTail; 1033 decrStIdx(store_idx); 1034 1035 while (stores != 0 && 1036 storeQueue[store_idx].inst->seqNum > squashed_num) { 1037 // Instructions marked as can WB are already committed. 1038 if (storeQueue[store_idx].canWB) { 1039 break; 1040 } 1041 1042 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 1043 "idx:%i [sn:%lli]\n", 1044 storeQueue[store_idx].inst->pcState(), 1045 store_idx, storeQueue[store_idx].inst->seqNum); 1046 1047 // I don't think this can happen. It should have been cleared 1048 // by the stalling load. 1049 if (isStalled() && 1050 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 1051 panic("Is stalled should have been cleared by stalling load!\n"); 1052 stalled = false; 1053 stallingStoreIsn = 0; 1054 } 1055 1056 // Clear the smart pointer to make sure it is decremented. 1057 storeQueue[store_idx].inst->setSquashed(); 1058 storeQueue[store_idx].inst = NULL; 1059 storeQueue[store_idx].canWB = 0; 1060 1061 // Must delete request now that it wasn't handed off to 1062 // memory. This is quite ugly. @todo: Figure out the proper 1063 // place to really handle request deletes. 1064 delete storeQueue[store_idx].req; 1065 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) { 1066 delete storeQueue[store_idx].sreqLow; 1067 delete storeQueue[store_idx].sreqHigh; 1068 1069 storeQueue[store_idx].sreqLow = NULL; 1070 storeQueue[store_idx].sreqHigh = NULL; 1071 } 1072 1073 storeQueue[store_idx].req = NULL; 1074 --stores; 1075 1076 // Inefficient! 1077 storeTail = store_idx; 1078 1079 decrStIdx(store_idx); 1080 ++lsqSquashedStores; 1081 } 1082} 1083 1084template <class Impl> 1085void 1086LSQUnit<Impl>::storePostSend(PacketPtr pkt) 1087{ 1088 if (isStalled() && 1089 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) { 1090 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1091 "load idx:%i\n", 1092 stallingStoreIsn, stallingLoadIdx); 1093 stalled = false; 1094 stallingStoreIsn = 0; 1095 iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 1096 } 1097 1098 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) { 1099 // The store is basically completed at this time. This 1100 // only works so long as the checker doesn't try to 1101 // verify the value in memory for stores. 1102 storeQueue[storeWBIdx].inst->setCompleted(); 1103 1104 if (cpu->checker) { 1105 cpu->checker->verify(storeQueue[storeWBIdx].inst); 1106 } 1107 } 1108 1109 if (needsTSO) { 1110 storeInFlight = true; 1111 } 1112 1113 incrStIdx(storeWBIdx); 1114} 1115 1116template <class Impl> 1117void 1118LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt) 1119{ 1120 iewStage->wakeCPU(); 1121 1122 // Squashed instructions do not need to complete their access. 1123 if (inst->isSquashed()) { 1124 assert(!inst->isStore()); 1125 ++lsqIgnoredResponses; 1126 return; 1127 } 1128 1129 if (!inst->isExecuted()) { 1130 inst->setExecuted(); 1131 1132 if (inst->fault == NoFault) { 1133 // Complete access to copy data to proper place. 1134 inst->completeAcc(pkt); 1135 } else { 1136 // If the instruction has an outstanding fault, we cannot complete 1137 // the access as this discards the current fault. 1138 1139 // If we have an outstanding fault, the fault should only be of 1140 // type ReExec. 1141 assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr); 1142 1143 DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access " 1144 "due to pending fault.\n", inst->seqNum); 1145 } 1146 } 1147 1148 // Need to insert instruction into queue to commit 1149 iewStage->instToCommit(inst); 1150 1151 iewStage->activityThisCycle(); 1152 1153 // see if this load changed the PC 1154 iewStage->checkMisprediction(inst); 1155} 1156 1157template <class Impl> 1158void 1159LSQUnit<Impl>::completeStore(int store_idx) 1160{ 1161 assert(storeQueue[store_idx].inst); 1162 storeQueue[store_idx].completed = true; 1163 --storesToWB; 1164 // A bit conservative because a store completion may not free up entries, 1165 // but hopefully avoids two store completions in one cycle from making 1166 // the CPU tick twice. 1167 cpu->wakeCPU(); 1168 cpu->activityThisCycle(); 1169 1170 if (store_idx == storeHead) { 1171 do { 1172 incrStIdx(storeHead); 1173 1174 --stores; 1175 } while (storeQueue[storeHead].completed && 1176 storeHead != storeTail); 1177 1178 iewStage->updateLSQNextCycle = true; 1179 } 1180 1181 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 1182 "idx:%i\n", 1183 storeQueue[store_idx].inst->seqNum, store_idx, storeHead); 1184 1185#if TRACING_ON 1186 if (DTRACE(O3PipeView)) { 1187 storeQueue[store_idx].inst->storeTick = 1188 curTick() - storeQueue[store_idx].inst->fetchTick; 1189 } 1190#endif 1191 1192 if (isStalled() && 1193 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 1194 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1195 "load idx:%i\n", 1196 stallingStoreIsn, stallingLoadIdx); 1197 stalled = false; 1198 stallingStoreIsn = 0; 1199 iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 1200 } 1201 1202 storeQueue[store_idx].inst->setCompleted(); 1203 1204 if (needsTSO) { 1205 storeInFlight = false; 1206 } 1207 1208 // Tell the checker we've completed this instruction. Some stores 1209 // may get reported twice to the checker, but the checker can 1210 // handle that case. 1211 if (cpu->checker) { 1212 cpu->checker->verify(storeQueue[store_idx].inst); 1213 } 1214} 1215 1216template <class Impl> 1217bool 1218LSQUnit<Impl>::sendStore(PacketPtr data_pkt) 1219{ 1220 if (!dcachePort->sendTimingReq(data_pkt)) { 1221 // Need to handle becoming blocked on a store. 1222 isStoreBlocked = true; 1223 ++lsqCacheBlocked; 1224 assert(retryPkt == NULL); 1225 retryPkt = data_pkt; 1226 return false; 1227 } 1228 return true; 1229} 1230 1231template <class Impl> 1232void 1233LSQUnit<Impl>::recvRetry() 1234{ 1235 if (isStoreBlocked) { 1236 DPRINTF(LSQUnit, "Receiving retry: store blocked\n"); 1237 assert(retryPkt != NULL); 1238 1239 LSQSenderState *state = 1240 dynamic_cast<LSQSenderState *>(retryPkt->senderState); 1241 1242 if (dcachePort->sendTimingReq(retryPkt)) { 1243 // Don't finish the store unless this is the last packet. 1244 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend || 1245 state->pendingPacket == retryPkt) { 1246 state->pktToSend = false; 1247 storePostSend(retryPkt); 1248 } 1249 retryPkt = NULL; 1250 isStoreBlocked = false; 1251 1252 // Send any outstanding packet. 1253 if (TheISA::HasUnalignedMemAcc && state->pktToSend) { 1254 assert(state->pendingPacket); 1255 if (sendStore(state->pendingPacket)) { 1256 storePostSend(state->pendingPacket); 1257 } 1258 } 1259 } else { 1260 // Still blocked! 1261 ++lsqCacheBlocked; 1262 } 1263 } 1264} 1265 1266template <class Impl> 1267inline void 1268LSQUnit<Impl>::incrStIdx(int &store_idx) const 1269{ 1270 if (++store_idx >= SQEntries) 1271 store_idx = 0; 1272} 1273 1274template <class Impl> 1275inline void 1276LSQUnit<Impl>::decrStIdx(int &store_idx) const 1277{ 1278 if (--store_idx < 0) 1279 store_idx += SQEntries; 1280} 1281 1282template <class Impl> 1283inline void 1284LSQUnit<Impl>::incrLdIdx(int &load_idx) const 1285{ 1286 if (++load_idx >= LQEntries) 1287 load_idx = 0; 1288} 1289 1290template <class Impl> 1291inline void 1292LSQUnit<Impl>::decrLdIdx(int &load_idx) const 1293{ 1294 if (--load_idx < 0) 1295 load_idx += LQEntries; 1296} 1297 1298template <class Impl> 1299void 1300LSQUnit<Impl>::dumpInsts() const 1301{ 1302 cprintf("Load store queue: Dumping instructions.\n"); 1303 cprintf("Load queue size: %i\n", loads); 1304 cprintf("Load queue: "); 1305 1306 int load_idx = loadHead; 1307 1308 while (load_idx != loadTail && loadQueue[load_idx]) { 1309 const DynInstPtr &inst(loadQueue[load_idx]); 1310 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1311 1312 incrLdIdx(load_idx); 1313 } 1314 cprintf("\n"); 1315 1316 cprintf("Store queue size: %i\n", stores); 1317 cprintf("Store queue: "); 1318 1319 int store_idx = storeHead; 1320 1321 while (store_idx != storeTail && storeQueue[store_idx].inst) { 1322 const DynInstPtr &inst(storeQueue[store_idx].inst); 1323 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1324 1325 incrStIdx(store_idx); 1326 } 1327 1328 cprintf("\n"); 1329} 1330 1331#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 1332