lsq_unit_impl.hh revision 13734:a57152849a55
16019SN/A 26019SN/A/* 310346Smitch.hayenga@arm.com * Copyright (c) 2010-2014, 2017-2018 ARM Limited 47134Sgblack@eecs.umich.edu * Copyright (c) 2013 Advanced Micro Devices, Inc. 57134Sgblack@eecs.umich.edu * All rights reserved 67134Sgblack@eecs.umich.edu * 77134Sgblack@eecs.umich.edu * The license below extends only to copyright in the software and shall 87134Sgblack@eecs.umich.edu * not be construed as granting a license to any other intellectual 97134Sgblack@eecs.umich.edu * property including but not limited to intellectual property relating 107134Sgblack@eecs.umich.edu * to a hardware implementation of the functionality of the software 117134Sgblack@eecs.umich.edu * licensed hereunder. You may use the software subject to the license 127134Sgblack@eecs.umich.edu * terms below provided that you ensure that this notice is replicated 137134Sgblack@eecs.umich.edu * unmodified and in its entirety in all distributions of the software, 147134Sgblack@eecs.umich.edu * modified or unmodified, in source code or in binary form. 156019SN/A * 166019SN/A * Copyright (c) 2004-2005 The Regents of The University of Michigan 176019SN/A * All rights reserved. 186019SN/A * 196019SN/A * Redistribution and use in source and binary forms, with or without 206019SN/A * modification, are permitted provided that the following conditions are 216019SN/A * met: redistributions of source code must retain the above copyright 226019SN/A * notice, this list of conditions and the following disclaimer; 236019SN/A * redistributions in binary form must reproduce the above copyright 246019SN/A * notice, this list of conditions and the following disclaimer in the 256019SN/A * documentation and/or other materials provided with the distribution; 266019SN/A * neither the name of the copyright holders nor the names of its 276019SN/A * contributors may be used to endorse or promote products derived from 286019SN/A * this software without specific prior written permission. 296019SN/A * 306019SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 316019SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 326019SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 336019SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 346019SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 356019SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 366019SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 376019SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 386019SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 396019SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 406019SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 416019SN/A * 426308SN/A * Authors: Kevin Lim 436308SN/A * Korey Sewell 446309SN/A */ 456309SN/A 466309SN/A#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 476309SN/A#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 486309SN/A 497134Sgblack@eecs.umich.edu#include "arch/generic/debugfaults.hh" 508588Sgblack@eecs.umich.edu#include "arch/locked_mem.hh" 516309SN/A#include "base/str.hh" 526309SN/A#include "config/the_isa.hh" 537296Sgblack@eecs.umich.edu#include "cpu/checker/cpu.hh" 548139SMatt.Horsnell@arm.com#include "cpu/o3/lsq.hh" 556309SN/A#include "cpu/o3/lsq_unit.hh" 566309SN/A#include "debug/Activity.hh" 576309SN/A#include "debug/IEW.hh" 5810346Smitch.hayenga@arm.com#include "debug/LSQUnit.hh" 5910346Smitch.hayenga@arm.com#include "debug/O3PipeView.hh" 6010346Smitch.hayenga@arm.com#include "mem/packet.hh" 6110346Smitch.hayenga@arm.com#include "mem/request.hh" 6210346Smitch.hayenga@arm.com 6310346Smitch.hayenga@arm.comtemplate<class Impl> 6410346Smitch.hayenga@arm.comLSQUnit<Impl>::WritebackEvent::WritebackEvent(const DynInstPtr &_inst, 6510346Smitch.hayenga@arm.com PacketPtr _pkt, LSQUnit *lsq_ptr) 6610346Smitch.hayenga@arm.com : Event(Default_Pri, AutoDelete), 6710346Smitch.hayenga@arm.com inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 6810346Smitch.hayenga@arm.com{ 6910346Smitch.hayenga@arm.com assert(_inst->savedReq); 708588Sgblack@eecs.umich.edu _inst->savedReq->writebackScheduled(); 717174Sgblack@eecs.umich.edu} 727639Sgblack@eecs.umich.edu 737639Sgblack@eecs.umich.edutemplate<class Impl> 747644Sali.saidi@arm.comvoid 758139SMatt.Horsnell@arm.comLSQUnit<Impl>::WritebackEvent::process() 767639Sgblack@eecs.umich.edu{ 777639Sgblack@eecs.umich.edu assert(!lsqPtr->cpu->switchedOut()); 787639Sgblack@eecs.umich.edu 798588Sgblack@eecs.umich.edu lsqPtr->writeback(inst, pkt); 807639Sgblack@eecs.umich.edu 817639Sgblack@eecs.umich.edu assert(inst->savedReq); 827639Sgblack@eecs.umich.edu inst->savedReq->writebackDone(); 837644Sali.saidi@arm.com delete pkt; 848139SMatt.Horsnell@arm.com} 857639Sgblack@eecs.umich.edu 867639Sgblack@eecs.umich.edutemplate<class Impl> 877639Sgblack@eecs.umich.educonst char * 887639Sgblack@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::description() const 897639Sgblack@eecs.umich.edu{ 908588Sgblack@eecs.umich.edu return "Store writeback"; 917639Sgblack@eecs.umich.edu} 927639Sgblack@eecs.umich.edu 937639Sgblack@eecs.umich.edutemplate <class Impl> 947644Sali.saidi@arm.combool 958139SMatt.Horsnell@arm.comLSQUnit<Impl>::recvTimingResp(PacketPtr pkt) 967639Sgblack@eecs.umich.edu{ 977639Sgblack@eecs.umich.edu auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState); 987639Sgblack@eecs.umich.edu LSQRequest* req = senderState->request(); 997639Sgblack@eecs.umich.edu assert(req != nullptr); 1007174Sgblack@eecs.umich.edu bool ret = true; 1018148SAli.Saidi@ARM.com /* Check that the request is still alive before any further action. */ 1028303SAli.Saidi@ARM.com if (senderState->alive()) { 1037400SAli.Saidi@ARM.com ret = req->recvTimingResp(pkt); 1048303SAli.Saidi@ARM.com } else { 1058303SAli.Saidi@ARM.com senderState->outstanding--; 10610037SARM gem5 Developers } 10710037SARM gem5 Developers return ret; 1088303SAli.Saidi@ARM.com 1098303SAli.Saidi@ARM.com} 1108303SAli.Saidi@ARM.com 1118303SAli.Saidi@ARM.comtemplate<class Impl> 1128303SAli.Saidi@ARM.comvoid 1138303SAli.Saidi@ARM.comLSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 1148205SAli.Saidi@ARM.com{ 1157858SMatt.Horsnell@arm.com LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 1168285SPrakash.Ramrakhyani@arm.com DynInstPtr inst = state->inst; 1176754SN/A 1188148SAli.Saidi@ARM.com cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1196754SN/A 1206754SN/A /* Notify the sender state that the access is complete (for ownership 1218148SAli.Saidi@ARM.com * tracking). */ 1228588Sgblack@eecs.umich.edu state->complete(); 1236754SN/A 1248139SMatt.Horsnell@arm.com assert(!cpu->switchedOut()); 1257422Sgblack@eecs.umich.edu if (!inst->isSquashed()) { 1268148SAli.Saidi@ARM.com if (state->needWB) { 1278148SAli.Saidi@ARM.com // Only loads, store conditionals and atomics perform the writeback 1286754SN/A // after receving the response from the memory 1298588Sgblack@eecs.umich.edu assert(inst->isLoad() || inst->isStoreConditional() || 1306309SN/A inst->isAtomic()); 1316309SN/A writeback(inst, state->request()->mainPacket()); 1327296Sgblack@eecs.umich.edu if (inst->isStore() || inst->isAtomic()) { 1337303Sgblack@eecs.umich.edu auto ss = dynamic_cast<SQSenderState*>(state); 1348139SMatt.Horsnell@arm.com ss->writebackDone(); 1356309SN/A completeStore(ss->idx); 1366309SN/A } 1376309SN/A } else if (inst->isStore()) { 1388588Sgblack@eecs.umich.edu // This is a regular store (i.e., not store conditionals and 1397174Sgblack@eecs.umich.edu // atomics), so it can complete without writing back 1407174Sgblack@eecs.umich.edu completeStore(dynamic_cast<SQSenderState*>(state)->idx); 1417296Sgblack@eecs.umich.edu } 1427303Sgblack@eecs.umich.edu } 1437644Sali.saidi@arm.com} 1448139SMatt.Horsnell@arm.com 1457174Sgblack@eecs.umich.edutemplate <class Impl> 1467174Sgblack@eecs.umich.eduLSQUnit<Impl>::LSQUnit(uint32_t lqEntries, uint32_t sqEntries) 1477174Sgblack@eecs.umich.edu : lsqID(-1), storeQueue(sqEntries+1), loadQueue(lqEntries+1), 1488588Sgblack@eecs.umich.edu loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 1497639Sgblack@eecs.umich.edu isStoreBlocked(false), storeInFlight(false), hasPendingRequest(false), 1507639Sgblack@eecs.umich.edu pendingRequest(nullptr) 1517639Sgblack@eecs.umich.edu{ 1527639Sgblack@eecs.umich.edu} 1537644Sali.saidi@arm.com 1548139SMatt.Horsnell@arm.comtemplate<class Impl> 1557639Sgblack@eecs.umich.eduvoid 1567639Sgblack@eecs.umich.eduLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 1577639Sgblack@eecs.umich.edu LSQ *lsq_ptr, unsigned id) 1587639Sgblack@eecs.umich.edu{ 1597639Sgblack@eecs.umich.edu lsqID = id; 1608588Sgblack@eecs.umich.edu 1617639Sgblack@eecs.umich.edu cpu = cpu_ptr; 1627639Sgblack@eecs.umich.edu iewStage = iew_ptr; 1637639Sgblack@eecs.umich.edu 1647639Sgblack@eecs.umich.edu lsq = lsq_ptr; 1657644Sali.saidi@arm.com 1668139SMatt.Horsnell@arm.com DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",lsqID); 1677639Sgblack@eecs.umich.edu 1687639Sgblack@eecs.umich.edu depCheckShift = params->LSQDepCheckShift; 1697639Sgblack@eecs.umich.edu checkLoads = params->LSQCheckLoads; 1707639Sgblack@eecs.umich.edu needsTSO = params->needsTSO; 1717639Sgblack@eecs.umich.edu 1727174Sgblack@eecs.umich.edu resetState(); 1737174Sgblack@eecs.umich.edu} 17410346Smitch.hayenga@arm.com 17510346Smitch.hayenga@arm.com 1767639Sgblack@eecs.umich.edutemplate<class Impl> 1777639Sgblack@eecs.umich.eduvoid 1787174Sgblack@eecs.umich.eduLSQUnit<Impl>::resetState() 1797174Sgblack@eecs.umich.edu{ 1807174Sgblack@eecs.umich.edu loads = stores = storesToWB = 0; 1817174Sgblack@eecs.umich.edu 1827174Sgblack@eecs.umich.edu 1837174Sgblack@eecs.umich.edu storeWBIt = storeQueue.begin(); 1847174Sgblack@eecs.umich.edu 1857174Sgblack@eecs.umich.edu retryPkt = NULL; 1867174Sgblack@eecs.umich.edu memDepViolator = NULL; 1877174Sgblack@eecs.umich.edu 1887174Sgblack@eecs.umich.edu stalled = false; 18910346Smitch.hayenga@arm.com 19010346Smitch.hayenga@arm.com cacheBlockMask = ~(cpu->cacheLineSize() - 1); 19110346Smitch.hayenga@arm.com} 19210346Smitch.hayenga@arm.com 19310346Smitch.hayenga@arm.comtemplate<class Impl> 19410346Smitch.hayenga@arm.comstd::string 1956309SN/ALSQUnit<Impl>::name() const 1966308SN/A{ 1977639Sgblack@eecs.umich.edu if (Impl::MaxThreads == 1) { 1987639Sgblack@eecs.umich.edu return iewStage->name() + ".lsq"; 1997639Sgblack@eecs.umich.edu } else { 20010037SARM gem5 Developers return iewStage->name() + ".lsq.thread" + std::to_string(lsqID); 2017639Sgblack@eecs.umich.edu } 2027639Sgblack@eecs.umich.edu} 2037639Sgblack@eecs.umich.edu 2047639Sgblack@eecs.umich.edutemplate<class Impl> 2057639Sgblack@eecs.umich.eduvoid 2067639Sgblack@eecs.umich.eduLSQUnit<Impl>::regStats() 2077639Sgblack@eecs.umich.edu{ 2087639Sgblack@eecs.umich.edu lsqForwLoads 2097639Sgblack@eecs.umich.edu .name(name() + ".forwLoads") 2107639Sgblack@eecs.umich.edu .desc("Number of loads that had data forwarded from stores"); 2117639Sgblack@eecs.umich.edu 2127639Sgblack@eecs.umich.edu invAddrLoads 2137639Sgblack@eecs.umich.edu .name(name() + ".invAddrLoads") 2147639Sgblack@eecs.umich.edu .desc("Number of loads ignored due to an invalid address"); 2157639Sgblack@eecs.umich.edu 2167639Sgblack@eecs.umich.edu lsqSquashedLoads 2177639Sgblack@eecs.umich.edu .name(name() + ".squashedLoads") 2187639Sgblack@eecs.umich.edu .desc("Number of loads squashed"); 2197639Sgblack@eecs.umich.edu 2207639Sgblack@eecs.umich.edu lsqIgnoredResponses 2217639Sgblack@eecs.umich.edu .name(name() + ".ignoredResponses") 2227639Sgblack@eecs.umich.edu .desc("Number of memory responses ignored because the instruction is squashed"); 2237639Sgblack@eecs.umich.edu 2247639Sgblack@eecs.umich.edu lsqMemOrderViolation 2257639Sgblack@eecs.umich.edu .name(name() + ".memOrderViolation") 2267639Sgblack@eecs.umich.edu .desc("Number of memory ordering violations"); 2277639Sgblack@eecs.umich.edu 2287639Sgblack@eecs.umich.edu lsqSquashedStores 2297639Sgblack@eecs.umich.edu .name(name() + ".squashedStores") 2307639Sgblack@eecs.umich.edu .desc("Number of stores squashed"); 2317639Sgblack@eecs.umich.edu 2327639Sgblack@eecs.umich.edu invAddrSwpfs 2337639Sgblack@eecs.umich.edu .name(name() + ".invAddrSwpfs") 2347639Sgblack@eecs.umich.edu .desc("Number of software prefetches ignored due to an invalid address"); 2357639Sgblack@eecs.umich.edu 2368588Sgblack@eecs.umich.edu lsqBlockedLoads 2377639Sgblack@eecs.umich.edu .name(name() + ".blockedLoads") 2387639Sgblack@eecs.umich.edu .desc("Number of blocked loads due to partial load-store forwarding"); 2397639Sgblack@eecs.umich.edu 2407639Sgblack@eecs.umich.edu lsqRescheduledLoads 2417639Sgblack@eecs.umich.edu .name(name() + ".rescheduledLoads") 2427639Sgblack@eecs.umich.edu .desc("Number of loads that were rescheduled"); 2438588Sgblack@eecs.umich.edu 2447639Sgblack@eecs.umich.edu lsqCacheBlocked 2457639Sgblack@eecs.umich.edu .name(name() + ".cacheBlocked") 2467639Sgblack@eecs.umich.edu .desc("Number of times an access to memory failed due to the cache being blocked"); 2477639Sgblack@eecs.umich.edu} 2487639Sgblack@eecs.umich.edu 2497639Sgblack@eecs.umich.edutemplate<class Impl> 2507639Sgblack@eecs.umich.eduvoid 2517639Sgblack@eecs.umich.eduLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2527639Sgblack@eecs.umich.edu{ 2537639Sgblack@eecs.umich.edu dcachePort = dcache_port; 2547639Sgblack@eecs.umich.edu} 2557644Sali.saidi@arm.com 2567639Sgblack@eecs.umich.edutemplate<class Impl> 2577639Sgblack@eecs.umich.eduvoid 2587639Sgblack@eecs.umich.eduLSQUnit<Impl>::drainSanityCheck() const 2597639Sgblack@eecs.umich.edu{ 2607639Sgblack@eecs.umich.edu for (int i = 0; i < loadQueue.capacity(); ++i) 2617639Sgblack@eecs.umich.edu assert(!loadQueue[i].valid()); 2627639Sgblack@eecs.umich.edu 2637639Sgblack@eecs.umich.edu assert(storesToWB == 0); 2647644Sali.saidi@arm.com assert(!retryPkt); 2657639Sgblack@eecs.umich.edu} 2667639Sgblack@eecs.umich.edu 2677639Sgblack@eecs.umich.edutemplate<class Impl> 2687639Sgblack@eecs.umich.eduvoid 2697639Sgblack@eecs.umich.eduLSQUnit<Impl>::takeOverFrom() 2707639Sgblack@eecs.umich.edu{ 2717639Sgblack@eecs.umich.edu resetState(); 2727639Sgblack@eecs.umich.edu} 2737639Sgblack@eecs.umich.edu 2747639Sgblack@eecs.umich.edutemplate <class Impl> 2757639Sgblack@eecs.umich.eduvoid 2767639Sgblack@eecs.umich.eduLSQUnit<Impl>::insert(const DynInstPtr &inst) 2777639Sgblack@eecs.umich.edu{ 2787639Sgblack@eecs.umich.edu assert(inst->isMemRef()); 2797639Sgblack@eecs.umich.edu 2807639Sgblack@eecs.umich.edu assert(inst->isLoad() || inst->isStore() || inst->isAtomic()); 2817639Sgblack@eecs.umich.edu 2827639Sgblack@eecs.umich.edu if (inst->isLoad()) { 2837639Sgblack@eecs.umich.edu insertLoad(inst); 2847639Sgblack@eecs.umich.edu } else { 2857639Sgblack@eecs.umich.edu insertStore(inst); 2867639Sgblack@eecs.umich.edu } 2877639Sgblack@eecs.umich.edu 2887639Sgblack@eecs.umich.edu inst->setInLSQ(); 2897639Sgblack@eecs.umich.edu} 2907639Sgblack@eecs.umich.edu 2917639Sgblack@eecs.umich.edutemplate <class Impl> 2927639Sgblack@eecs.umich.eduvoid 2937639Sgblack@eecs.umich.eduLSQUnit<Impl>::insertLoad(const DynInstPtr &load_inst) 2947639Sgblack@eecs.umich.edu{ 2957639Sgblack@eecs.umich.edu assert(!loadQueue.full()); 2967639Sgblack@eecs.umich.edu assert(loads < loadQueue.capacity()); 2977639Sgblack@eecs.umich.edu 2987639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 2997639Sgblack@eecs.umich.edu load_inst->pcState(), loadQueue.tail(), load_inst->seqNum); 3007639Sgblack@eecs.umich.edu 3017639Sgblack@eecs.umich.edu /* Grow the queue. */ 3027639Sgblack@eecs.umich.edu loadQueue.advance_tail(); 3037639Sgblack@eecs.umich.edu 3047639Sgblack@eecs.umich.edu load_inst->sqIt = storeQueue.end(); 3057639Sgblack@eecs.umich.edu 3067639Sgblack@eecs.umich.edu assert(!loadQueue.back().valid()); 3077639Sgblack@eecs.umich.edu loadQueue.back().set(load_inst); 3087639Sgblack@eecs.umich.edu load_inst->lqIdx = loadQueue.tail(); 3097639Sgblack@eecs.umich.edu load_inst->lqIt = loadQueue.getIterator(load_inst->lqIdx); 3107639Sgblack@eecs.umich.edu 3117639Sgblack@eecs.umich.edu ++loads; 3127639Sgblack@eecs.umich.edu} 3137639Sgblack@eecs.umich.edu 3147639Sgblack@eecs.umich.edutemplate <class Impl> 3158588Sgblack@eecs.umich.eduvoid 3168588Sgblack@eecs.umich.eduLSQUnit<Impl>::insertStore(const DynInstPtr& store_inst) 3177639Sgblack@eecs.umich.edu{ 3187639Sgblack@eecs.umich.edu // Make sure it is not full before inserting an instruction. 3198588Sgblack@eecs.umich.edu assert(!storeQueue.full()); 3208588Sgblack@eecs.umich.edu assert(stores < storeQueue.capacity()); 3217639Sgblack@eecs.umich.edu 3227639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 3237639Sgblack@eecs.umich.edu store_inst->pcState(), storeQueue.tail(), store_inst->seqNum); 3247639Sgblack@eecs.umich.edu storeQueue.advance_tail(); 3257639Sgblack@eecs.umich.edu 3267639Sgblack@eecs.umich.edu store_inst->sqIdx = storeQueue.tail(); 3277639Sgblack@eecs.umich.edu store_inst->lqIdx = loadQueue.moduloAdd(loadQueue.tail(), 1); 3287639Sgblack@eecs.umich.edu store_inst->lqIt = loadQueue.end(); 3297639Sgblack@eecs.umich.edu 3307639Sgblack@eecs.umich.edu storeQueue.back().set(store_inst); 3317639Sgblack@eecs.umich.edu 3327639Sgblack@eecs.umich.edu ++stores; 3337639Sgblack@eecs.umich.edu} 3347639Sgblack@eecs.umich.edu 3357639Sgblack@eecs.umich.edutemplate <class Impl> 3367639Sgblack@eecs.umich.edutypename Impl::DynInstPtr 3377639Sgblack@eecs.umich.eduLSQUnit<Impl>::getMemDepViolator() 3387639Sgblack@eecs.umich.edu{ 3397639Sgblack@eecs.umich.edu DynInstPtr temp = memDepViolator; 3407639Sgblack@eecs.umich.edu 3417639Sgblack@eecs.umich.edu memDepViolator = NULL; 3427639Sgblack@eecs.umich.edu 3437639Sgblack@eecs.umich.edu return temp; 3447639Sgblack@eecs.umich.edu} 3457639Sgblack@eecs.umich.edu 3467639Sgblack@eecs.umich.edutemplate <class Impl> 3477639Sgblack@eecs.umich.eduunsigned 3487639Sgblack@eecs.umich.eduLSQUnit<Impl>::numFreeLoadEntries() 3497639Sgblack@eecs.umich.edu{ 3507639Sgblack@eecs.umich.edu //LQ has an extra dummy entry to differentiate 3517639Sgblack@eecs.umich.edu //empty/full conditions. Subtract 1 from the free entries. 3527639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", 3537639Sgblack@eecs.umich.edu 1 + loadQueue.capacity(), loads); 3547639Sgblack@eecs.umich.edu return loadQueue.capacity() - loads; 3557639Sgblack@eecs.umich.edu} 3567639Sgblack@eecs.umich.edu 3577639Sgblack@eecs.umich.edutemplate <class Impl> 3587639Sgblack@eecs.umich.eduunsigned 3597639Sgblack@eecs.umich.eduLSQUnit<Impl>::numFreeStoreEntries() 3607639Sgblack@eecs.umich.edu{ 3618588Sgblack@eecs.umich.edu //SQ has an extra dummy entry to differentiate 3628588Sgblack@eecs.umich.edu //empty/full conditions. Subtract 1 from the free entries. 3637639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", 3647639Sgblack@eecs.umich.edu 1 + storeQueue.capacity(), stores); 3658588Sgblack@eecs.umich.edu return storeQueue.capacity() - stores; 3668588Sgblack@eecs.umich.edu 3677639Sgblack@eecs.umich.edu } 3687639Sgblack@eecs.umich.edu 3697639Sgblack@eecs.umich.edutemplate <class Impl> 3707639Sgblack@eecs.umich.eduvoid 3717639Sgblack@eecs.umich.eduLSQUnit<Impl>::checkSnoop(PacketPtr pkt) 3727639Sgblack@eecs.umich.edu{ 3737639Sgblack@eecs.umich.edu // Should only ever get invalidations in here 3747639Sgblack@eecs.umich.edu assert(pkt->isInvalidate()); 3757639Sgblack@eecs.umich.edu 3767639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 3777639Sgblack@eecs.umich.edu 3787639Sgblack@eecs.umich.edu for (int x = 0; x < cpu->numContexts(); x++) { 3797639Sgblack@eecs.umich.edu ThreadContext *tc = cpu->getContext(x); 3807639Sgblack@eecs.umich.edu bool no_squash = cpu->thread[x]->noSquashFromTC; 3817639Sgblack@eecs.umich.edu cpu->thread[x]->noSquashFromTC = true; 3827639Sgblack@eecs.umich.edu TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 3837639Sgblack@eecs.umich.edu cpu->thread[x]->noSquashFromTC = no_squash; 3847639Sgblack@eecs.umich.edu } 3857639Sgblack@eecs.umich.edu 3867639Sgblack@eecs.umich.edu if (loadQueue.empty()) 3877639Sgblack@eecs.umich.edu return; 3887639Sgblack@eecs.umich.edu 3897639Sgblack@eecs.umich.edu auto iter = loadQueue.begin(); 3907639Sgblack@eecs.umich.edu 3917639Sgblack@eecs.umich.edu Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 3927639Sgblack@eecs.umich.edu 3937639Sgblack@eecs.umich.edu DynInstPtr ld_inst = iter->instruction(); 3947639Sgblack@eecs.umich.edu assert(ld_inst); 3957639Sgblack@eecs.umich.edu LSQRequest *req = iter->request(); 3967639Sgblack@eecs.umich.edu 3977639Sgblack@eecs.umich.edu // Check that this snoop didn't just invalidate our lock flag 3987639Sgblack@eecs.umich.edu if (ld_inst->effAddrValid() && 3997639Sgblack@eecs.umich.edu req->isCacheBlockHit(invalidate_addr, cacheBlockMask) 4007639Sgblack@eecs.umich.edu && ld_inst->memReqFlags & Request::LLSC) 4017639Sgblack@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 4027639Sgblack@eecs.umich.edu 4037639Sgblack@eecs.umich.edu bool force_squash = false; 4047639Sgblack@eecs.umich.edu 4057639Sgblack@eecs.umich.edu while (++iter != loadQueue.end()) { 4067639Sgblack@eecs.umich.edu ld_inst = iter->instruction(); 4077639Sgblack@eecs.umich.edu assert(ld_inst); 4087639Sgblack@eecs.umich.edu req = iter->request(); 4097639Sgblack@eecs.umich.edu if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) 4107639Sgblack@eecs.umich.edu continue; 4117639Sgblack@eecs.umich.edu 4127639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "-- inst [sn:%lli] to pktAddr:%#x\n", 4137639Sgblack@eecs.umich.edu ld_inst->seqNum, invalidate_addr); 4147639Sgblack@eecs.umich.edu 4157639Sgblack@eecs.umich.edu if (force_squash || 4167639Sgblack@eecs.umich.edu req->isCacheBlockHit(invalidate_addr, cacheBlockMask)) { 4177639Sgblack@eecs.umich.edu if (needsTSO) { 4187639Sgblack@eecs.umich.edu // If we have a TSO system, as all loads must be ordered with 4197639Sgblack@eecs.umich.edu // all other loads, this load as well as *all* subsequent loads 4207639Sgblack@eecs.umich.edu // need to be squashed to prevent possible load reordering. 4217639Sgblack@eecs.umich.edu force_squash = true; 4227639Sgblack@eecs.umich.edu } 4237639Sgblack@eecs.umich.edu if (ld_inst->possibleLoadViolation() || force_squash) { 4247639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 4257639Sgblack@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 4267639Sgblack@eecs.umich.edu 4278588Sgblack@eecs.umich.edu // Mark the load for re-execution 4288588Sgblack@eecs.umich.edu ld_inst->fault = std::make_shared<ReExec>(); 4297639Sgblack@eecs.umich.edu } else { 4307639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 4317639Sgblack@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 4327639Sgblack@eecs.umich.edu 4337639Sgblack@eecs.umich.edu // Make sure that we don't lose a snoop hitting a LOCKED 4347639Sgblack@eecs.umich.edu // address since the LOCK* flags don't get updated until 4357639Sgblack@eecs.umich.edu // commit. 4368588Sgblack@eecs.umich.edu if (ld_inst->memReqFlags & Request::LLSC) 4378588Sgblack@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 4387639Sgblack@eecs.umich.edu 4397639Sgblack@eecs.umich.edu // If a older load checks this and it's true 4408588Sgblack@eecs.umich.edu // then we might have missed the snoop 4418588Sgblack@eecs.umich.edu // in which case we need to invalidate to be sure 4427639Sgblack@eecs.umich.edu ld_inst->hitExternalSnoop(true); 4437639Sgblack@eecs.umich.edu } 4447639Sgblack@eecs.umich.edu } 4457639Sgblack@eecs.umich.edu } 4467639Sgblack@eecs.umich.edu return; 4477639Sgblack@eecs.umich.edu} 4487639Sgblack@eecs.umich.edu 4497639Sgblack@eecs.umich.edutemplate <class Impl> 4507639Sgblack@eecs.umich.eduFault 4517639Sgblack@eecs.umich.eduLSQUnit<Impl>::checkViolations(typename LoadQueue::iterator& loadIt, 4527639Sgblack@eecs.umich.edu const DynInstPtr& inst) 4537639Sgblack@eecs.umich.edu{ 4547639Sgblack@eecs.umich.edu Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 4557639Sgblack@eecs.umich.edu Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 4567639Sgblack@eecs.umich.edu 4577639Sgblack@eecs.umich.edu /** @todo in theory you only need to check an instruction that has executed 4587639Sgblack@eecs.umich.edu * however, there isn't a good way in the pipeline at the moment to check 4597639Sgblack@eecs.umich.edu * all instructions that will execute before the store writes back. Thus, 4607639Sgblack@eecs.umich.edu * like the implementation that came before it, we're overly conservative. 4617639Sgblack@eecs.umich.edu */ 4627639Sgblack@eecs.umich.edu while (loadIt != loadQueue.end()) { 4637639Sgblack@eecs.umich.edu DynInstPtr ld_inst = loadIt->instruction(); 4647639Sgblack@eecs.umich.edu if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) { 4657639Sgblack@eecs.umich.edu ++loadIt; 4667639Sgblack@eecs.umich.edu continue; 4677639Sgblack@eecs.umich.edu } 4687639Sgblack@eecs.umich.edu 4697639Sgblack@eecs.umich.edu Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 4707639Sgblack@eecs.umich.edu Addr ld_eff_addr2 = 4717639Sgblack@eecs.umich.edu (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 4727639Sgblack@eecs.umich.edu 4737639Sgblack@eecs.umich.edu if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 4747639Sgblack@eecs.umich.edu if (inst->isLoad()) { 4757639Sgblack@eecs.umich.edu // If this load is to the same block as an external snoop 4767639Sgblack@eecs.umich.edu // invalidate that we've observed then the load needs to be 4777639Sgblack@eecs.umich.edu // squashed as it could have newer data 4787639Sgblack@eecs.umich.edu if (ld_inst->hitExternalSnoop()) { 4797639Sgblack@eecs.umich.edu if (!memDepViolator || 4807639Sgblack@eecs.umich.edu ld_inst->seqNum < memDepViolator->seqNum) { 4817639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 4828588Sgblack@eecs.umich.edu "and [sn:%lli] at address %#x\n", 4838588Sgblack@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 4847639Sgblack@eecs.umich.edu memDepViolator = ld_inst; 4857639Sgblack@eecs.umich.edu 4867639Sgblack@eecs.umich.edu ++lsqMemOrderViolation; 4877639Sgblack@eecs.umich.edu 4887639Sgblack@eecs.umich.edu return std::make_shared<GenericISA::M5PanicFault>( 4897639Sgblack@eecs.umich.edu "Detected fault with inst [sn:%lli] and " 4908588Sgblack@eecs.umich.edu "[sn:%lli] at address %#x\n", 4918588Sgblack@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 4927639Sgblack@eecs.umich.edu } 4937639Sgblack@eecs.umich.edu } 4947639Sgblack@eecs.umich.edu 4957639Sgblack@eecs.umich.edu // Otherwise, mark the load has a possible load violation 4967639Sgblack@eecs.umich.edu // and if we see a snoop before it's commited, we need to squash 4977639Sgblack@eecs.umich.edu ld_inst->possibleLoadViolation(true); 4987639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Found possible load violation at addr: %#x" 4997639Sgblack@eecs.umich.edu " between instructions [sn:%lli] and [sn:%lli]\n", 5007639Sgblack@eecs.umich.edu inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 5017639Sgblack@eecs.umich.edu } else { 5027639Sgblack@eecs.umich.edu // A load/store incorrectly passed this store. 5037639Sgblack@eecs.umich.edu // Check if we already have a violator, or if it's newer 5047639Sgblack@eecs.umich.edu // squash and refetch. 5057639Sgblack@eecs.umich.edu if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 5067639Sgblack@eecs.umich.edu break; 5077639Sgblack@eecs.umich.edu 5087639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 5097639Sgblack@eecs.umich.edu "[sn:%lli] at address %#x\n", 5107639Sgblack@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5117639Sgblack@eecs.umich.edu memDepViolator = ld_inst; 5127639Sgblack@eecs.umich.edu 5137639Sgblack@eecs.umich.edu ++lsqMemOrderViolation; 5147639Sgblack@eecs.umich.edu 5157639Sgblack@eecs.umich.edu return std::make_shared<GenericISA::M5PanicFault>( 5167639Sgblack@eecs.umich.edu "Detected fault with " 5177639Sgblack@eecs.umich.edu "inst [sn:%lli] and [sn:%lli] at address %#x\n", 5187639Sgblack@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5197639Sgblack@eecs.umich.edu } 5207639Sgblack@eecs.umich.edu } 5217639Sgblack@eecs.umich.edu 5227639Sgblack@eecs.umich.edu ++loadIt; 5237639Sgblack@eecs.umich.edu } 5247639Sgblack@eecs.umich.edu return NoFault; 5257639Sgblack@eecs.umich.edu} 5267639Sgblack@eecs.umich.edu 5277639Sgblack@eecs.umich.edu 5287639Sgblack@eecs.umich.edu 5297639Sgblack@eecs.umich.edu 5307639Sgblack@eecs.umich.edutemplate <class Impl> 5317639Sgblack@eecs.umich.eduFault 5327639Sgblack@eecs.umich.eduLSQUnit<Impl>::executeLoad(const DynInstPtr &inst) 5338588Sgblack@eecs.umich.edu{ 5348588Sgblack@eecs.umich.edu using namespace TheISA; 5357639Sgblack@eecs.umich.edu // Execute a specific load. 5367639Sgblack@eecs.umich.edu Fault load_fault = NoFault; 5377639Sgblack@eecs.umich.edu 5387639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 5397639Sgblack@eecs.umich.edu inst->pcState(), inst->seqNum); 5407639Sgblack@eecs.umich.edu 5418588Sgblack@eecs.umich.edu assert(!inst->isSquashed()); 5428588Sgblack@eecs.umich.edu 5437639Sgblack@eecs.umich.edu load_fault = inst->initiateAcc(); 5447639Sgblack@eecs.umich.edu 5457639Sgblack@eecs.umich.edu if (inst->isTranslationDelayed() && load_fault == NoFault) 5467639Sgblack@eecs.umich.edu return load_fault; 5477639Sgblack@eecs.umich.edu 5487639Sgblack@eecs.umich.edu // If the instruction faulted or predicated false, then we need to send it 5497639Sgblack@eecs.umich.edu // along to commit without the instruction completing. 5507639Sgblack@eecs.umich.edu if (load_fault != NoFault || !inst->readPredicate()) { 5517639Sgblack@eecs.umich.edu // Send this instruction to commit, also make sure iew stage 5527639Sgblack@eecs.umich.edu // realizes there is activity. Mark it as executed unless it 5537639Sgblack@eecs.umich.edu // is a strictly ordered load that needs to hit the head of 5547639Sgblack@eecs.umich.edu // commit. 5557639Sgblack@eecs.umich.edu if (!inst->readPredicate()) 5567639Sgblack@eecs.umich.edu inst->forwardOldRegs(); 5577639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 5587639Sgblack@eecs.umich.edu inst->seqNum, 5597639Sgblack@eecs.umich.edu (load_fault != NoFault ? "fault" : "predication")); 5607639Sgblack@eecs.umich.edu if (!(inst->hasRequest() && inst->strictlyOrdered()) || 5617639Sgblack@eecs.umich.edu inst->isAtCommit()) { 5627639Sgblack@eecs.umich.edu inst->setExecuted(); 5638309Snate@binkert.org } 5648309Snate@binkert.org iewStage->instToCommit(inst); 5658309Snate@binkert.org iewStage->activityThisCycle(); 5667639Sgblack@eecs.umich.edu } else { 5677639Sgblack@eecs.umich.edu if (inst->effAddrValid()) { 5687639Sgblack@eecs.umich.edu auto it = inst->lqIt; 5697639Sgblack@eecs.umich.edu ++it; 5707639Sgblack@eecs.umich.edu 5717639Sgblack@eecs.umich.edu if (checkLoads) 5727639Sgblack@eecs.umich.edu return checkViolations(it, inst); 5737639Sgblack@eecs.umich.edu } 5747639Sgblack@eecs.umich.edu } 5757639Sgblack@eecs.umich.edu 5767639Sgblack@eecs.umich.edu return load_fault; 5777639Sgblack@eecs.umich.edu} 5787639Sgblack@eecs.umich.edu 5797639Sgblack@eecs.umich.edutemplate <class Impl> 5807639Sgblack@eecs.umich.eduFault 5817639Sgblack@eecs.umich.eduLSQUnit<Impl>::executeStore(const DynInstPtr &store_inst) 5827639Sgblack@eecs.umich.edu{ 5837639Sgblack@eecs.umich.edu using namespace TheISA; 5847639Sgblack@eecs.umich.edu // Make sure that a store exists. 5858607Sgblack@eecs.umich.edu assert(stores != 0); 5867639Sgblack@eecs.umich.edu 5878607Sgblack@eecs.umich.edu int store_idx = store_inst->sqIdx; 5887639Sgblack@eecs.umich.edu 5897639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 5907639Sgblack@eecs.umich.edu store_inst->pcState(), store_inst->seqNum); 5917639Sgblack@eecs.umich.edu 5927639Sgblack@eecs.umich.edu assert(!store_inst->isSquashed()); 5938607Sgblack@eecs.umich.edu 5948607Sgblack@eecs.umich.edu // Check the recently completed loads to see if any match this store's 5957639Sgblack@eecs.umich.edu // address. If so, then we have a memory ordering violation. 5967639Sgblack@eecs.umich.edu typename LoadQueue::iterator loadIt = store_inst->lqIt; 5977639Sgblack@eecs.umich.edu 5986308SN/A Fault store_fault = store_inst->initiateAcc(); 5996308SN/A 6006308SN/A if (store_inst->isTranslationDelayed() && 6016308SN/A store_fault == NoFault) 6026308SN/A return store_fault; 6036308SN/A 6046308SN/A if (!store_inst->readPredicate()) { 6057639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 6068139SMatt.Horsnell@arm.com store_inst->seqNum); 6076308SN/A store_inst->forwardOldRegs(); 6086308SN/A return store_fault; 6096308SN/A } 6108304SAli.Saidi@ARM.com 6118304SAli.Saidi@ARM.com if (storeQueue[store_idx].size() == 0) { 6128304SAli.Saidi@ARM.com DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 6138304SAli.Saidi@ARM.com store_inst->pcState(), store_inst->seqNum); 61410037SARM gem5 Developers 61510037SARM gem5 Developers return store_fault; 61610037SARM gem5 Developers } 61710037SARM gem5 Developers 61810037SARM gem5 Developers assert(store_fault == NoFault); 61910037SARM gem5 Developers 62010037SARM gem5 Developers if (store_inst->isStoreConditional() || store_inst->isAtomic()) { 62110037SARM gem5 Developers // Store conditionals and Atomics need to set themselves as able to 62210037SARM gem5 Developers // writeback if we haven't had a fault by here. 62310474Sandreas.hansson@arm.com storeQueue[store_idx].canWB() = true; 62410037SARM gem5 Developers 62510037SARM gem5 Developers ++storesToWB; 62610037SARM gem5 Developers } 62710037SARM gem5 Developers 62810037SARM gem5 Developers return checkViolations(loadIt, store_inst); 62910037SARM gem5 Developers 63010037SARM gem5 Developers} 63110037SARM gem5 Developers 63210037SARM gem5 Developerstemplate <class Impl> 63310037SARM gem5 Developersvoid 6347639Sgblack@eecs.umich.eduLSQUnit<Impl>::commitLoad() 6357646Sgene.wu@arm.com{ 6368304SAli.Saidi@ARM.com assert(loadQueue.front().valid()); 6378304SAli.Saidi@ARM.com 6387639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 6397639Sgblack@eecs.umich.edu loadQueue.front().instruction()->pcState()); 6406308SN/A 6417639Sgblack@eecs.umich.edu loadQueue.front().clear(); 6428139SMatt.Horsnell@arm.com loadQueue.pop_front(); 6436308SN/A 6446308SN/A --loads; 6456308SN/A} 64610037SARM gem5 Developers 64710037SARM gem5 Developerstemplate <class Impl> 64810037SARM gem5 Developersvoid 64910037SARM gem5 DevelopersLSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 65010037SARM gem5 Developers{ 6518304SAli.Saidi@ARM.com assert(loads == 0 || loadQueue.front().valid()); 6528304SAli.Saidi@ARM.com 6538304SAli.Saidi@ARM.com while (loads != 0 && loadQueue.front().instruction()->seqNum 6547646Sgene.wu@arm.com <= youngest_inst) { 6557646Sgene.wu@arm.com commitLoad(); 6568304SAli.Saidi@ARM.com } 6578304SAli.Saidi@ARM.com} 6587646Sgene.wu@arm.com 6597646Sgene.wu@arm.comtemplate <class Impl> 6607646Sgene.wu@arm.comvoid 6617646Sgene.wu@arm.comLSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 6628139SMatt.Horsnell@arm.com{ 6637646Sgene.wu@arm.com assert(stores == 0 || storeQueue.front().valid()); 6647646Sgene.wu@arm.com 6657646Sgene.wu@arm.com /* Forward iterate the store queue (age order). */ 6668148SAli.Saidi@ARM.com for (auto& x : storeQueue) { 6678148SAli.Saidi@ARM.com assert(x.valid()); 6688148SAli.Saidi@ARM.com // Mark any stores that are now committed and have not yet 6698148SAli.Saidi@ARM.com // been marked as able to write back. 6708148SAli.Saidi@ARM.com if (!x.canWB()) { 6718148SAli.Saidi@ARM.com if (x.instruction()->seqNum > youngest_inst) { 6728148SAli.Saidi@ARM.com break; 6738140SMatt.Horsnell@arm.com } 6748140SMatt.Horsnell@arm.com DPRINTF(LSQUnit, "Marking store as able to write back, PC " 6758140SMatt.Horsnell@arm.com "%s [sn:%lli]\n", 6768140SMatt.Horsnell@arm.com x.instruction()->pcState(), 6778303SAli.Saidi@ARM.com x.instruction()->seqNum); 67810037SARM gem5 Developers 67910037SARM gem5 Developers x.canWB() = true; 6808303SAli.Saidi@ARM.com 6818303SAli.Saidi@ARM.com ++storesToWB; 6828303SAli.Saidi@ARM.com } 6838205SAli.Saidi@ARM.com } 6848140SMatt.Horsnell@arm.com} 6858303SAli.Saidi@ARM.com 6868303SAli.Saidi@ARM.comtemplate <class Impl> 6878303SAli.Saidi@ARM.comvoid 6888303SAli.Saidi@ARM.comLSQUnit<Impl>::writebackBlockedStore() 6898140SMatt.Horsnell@arm.com{ 6908140SMatt.Horsnell@arm.com assert(isStoreBlocked); 6918140SMatt.Horsnell@arm.com storeWBIt->request()->sendPacketToCache(); 6928140SMatt.Horsnell@arm.com if (storeWBIt->request()->isSent()){ 6938140SMatt.Horsnell@arm.com storePostSend(); 6948140SMatt.Horsnell@arm.com } 6958140SMatt.Horsnell@arm.com} 6968140SMatt.Horsnell@arm.com 6977639Sgblack@eecs.umich.edutemplate <class Impl> 69810037SARM gem5 Developersvoid 69910037SARM gem5 DevelopersLSQUnit<Impl>::writebackStores() 7007639Sgblack@eecs.umich.edu{ 70110037SARM gem5 Developers if (isStoreBlocked) { 7027646Sgene.wu@arm.com DPRINTF(LSQUnit, "Writing back blocked store\n"); 7037646Sgene.wu@arm.com writebackBlockedStore(); 70410037SARM gem5 Developers } 7058140SMatt.Horsnell@arm.com 7068148SAli.Saidi@ARM.com while (storesToWB > 0 && 7078140SMatt.Horsnell@arm.com storeWBIt.dereferenceable() && 7087646Sgene.wu@arm.com storeWBIt->valid() && 7097639Sgblack@eecs.umich.edu storeWBIt->canWB() && 71010037SARM gem5 Developers ((!needsTSO) || (!storeInFlight)) && 71110037SARM gem5 Developers lsq->cachePortAvailable(false)) { 7127639Sgblack@eecs.umich.edu 71310037SARM gem5 Developers if (isStoreBlocked) { 7147646Sgene.wu@arm.com DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 7157646Sgene.wu@arm.com " is blocked!\n"); 71610037SARM gem5 Developers break; 7178140SMatt.Horsnell@arm.com } 7188148SAli.Saidi@ARM.com 7198140SMatt.Horsnell@arm.com // Store didn't write any data so no need to write it back to 7207646Sgene.wu@arm.com // memory. 7216308SN/A if (storeWBIt->size() == 0) { 72210037SARM gem5 Developers /* It is important that the preincrement happens at (or before) 72310037SARM gem5 Developers * the call, as the the code of completeStore checks 7247639Sgblack@eecs.umich.edu * storeWBIt. */ 72510037SARM gem5 Developers completeStore(storeWBIt++); 7267646Sgene.wu@arm.com continue; 7277646Sgene.wu@arm.com } 72810037SARM gem5 Developers 7298140SMatt.Horsnell@arm.com if (storeWBIt->instruction()->isDataPrefetch()) { 7308148SAli.Saidi@ARM.com storeWBIt++; 7318140SMatt.Horsnell@arm.com continue; 7328140SMatt.Horsnell@arm.com } 7336308SN/A 7346019SN/A assert(storeWBIt->hasRequest()); 7357134Sgblack@eecs.umich.edu assert(!storeWBIt->committed()); 7367170Sgblack@eecs.umich.edu 7377134Sgblack@eecs.umich.edu DynInstPtr inst = storeWBIt->instruction(); 7387134Sgblack@eecs.umich.edu LSQRequest* req = storeWBIt->request(); 7397179Sgblack@eecs.umich.edu storeWBIt->committed() = true; 74010037SARM gem5 Developers 74110037SARM gem5 Developers assert(!inst->memData); 74210037SARM gem5 Developers inst->memData = new uint8_t[req->_size]; 74310037SARM gem5 Developers 74410037SARM gem5 Developers if (storeWBIt->isAllZeros()) 74510037SARM gem5 Developers memset(inst->memData, 0, req->_size); 74610037SARM gem5 Developers else 74710037SARM gem5 Developers memcpy(inst->memData, storeWBIt->data(), req->_size); 74810037SARM gem5 Developers 74910037SARM gem5 Developers 75010037SARM gem5 Developers if (req->senderState() == nullptr) { 75110037SARM gem5 Developers SQSenderState *state = new SQSenderState(storeWBIt); 75210037SARM gem5 Developers state->isLoad = false; 75310037SARM gem5 Developers state->needWB = false; 75410037SARM gem5 Developers state->inst = inst; 75510037SARM gem5 Developers 75610037SARM gem5 Developers req->senderState(state); 75710037SARM gem5 Developers if (inst->isStoreConditional() || inst->isAtomic()) { 75810037SARM gem5 Developers /* Only store conditionals and atomics need a writeback. */ 7597639Sgblack@eecs.umich.edu state->needWB = true; 7607639Sgblack@eecs.umich.edu } 7617639Sgblack@eecs.umich.edu } 7627639Sgblack@eecs.umich.edu req->buildPackets(); 7637639Sgblack@eecs.umich.edu 7647639Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 7657639Sgblack@eecs.umich.edu "to Addr:%#x, data:%#x [sn:%lli]\n", 7667639Sgblack@eecs.umich.edu storeWBIt.idx(), inst->pcState(), 7677639Sgblack@eecs.umich.edu req->request()->getPaddr(), (int)*(inst->memData), 7687639Sgblack@eecs.umich.edu inst->seqNum); 7697639Sgblack@eecs.umich.edu 7707639Sgblack@eecs.umich.edu // @todo: Remove this SC hack once the memory system handles it. 7717639Sgblack@eecs.umich.edu if (inst->isStoreConditional()) { 7727639Sgblack@eecs.umich.edu // Disable recording the result temporarily. Writing to 7737639Sgblack@eecs.umich.edu // misc regs normally updates the result, but this is not 7747639Sgblack@eecs.umich.edu // the desired behavior when handling store conditionals. 7757179Sgblack@eecs.umich.edu inst->recordResult(false); 7767179Sgblack@eecs.umich.edu bool success = TheISA::handleLockedWrite(inst.get(), 7777179Sgblack@eecs.umich.edu req->request(), cacheBlockMask); 7786019SN/A inst->recordResult(true); 779 req->packetSent(); 780 781 if (!success) { 782 req->complete(); 783 // Instantly complete this store. 784 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 785 "Instantly completing it.\n", 786 inst->seqNum); 787 PacketPtr new_pkt = new Packet(*req->packet()); 788 WritebackEvent *wb = new WritebackEvent(inst, 789 new_pkt, this); 790 cpu->schedule(wb, curTick() + 1); 791 completeStore(storeWBIt); 792 if (!storeQueue.empty()) 793 storeWBIt++; 794 else 795 storeWBIt = storeQueue.end(); 796 continue; 797 } 798 } 799 800 if (req->request()->isMmappedIpr()) { 801 assert(!inst->isStoreConditional()); 802 ThreadContext *thread = cpu->tcBase(lsqID); 803 PacketPtr main_pkt = new Packet(req->mainRequest(), 804 MemCmd::WriteReq); 805 main_pkt->dataStatic(inst->memData); 806 req->handleIprWrite(thread, main_pkt); 807 delete main_pkt; 808 completeStore(storeWBIt); 809 storeWBIt++; 810 continue; 811 } 812 /* Send to cache */ 813 req->sendPacketToCache(); 814 815 /* If successful, do the post send */ 816 if (req->isSent()) { 817 storePostSend(); 818 } else { 819 DPRINTF(LSQUnit, "D-Cache became blocked when writing [sn:%lli], " 820 "will retry later\n", 821 inst->seqNum); 822 } 823 } 824 assert(stores >= 0 && storesToWB >= 0); 825} 826 827template <class Impl> 828void 829LSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 830{ 831 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 832 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 833 834 while (loads != 0 && 835 loadQueue.back().instruction()->seqNum > squashed_num) { 836 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 837 "[sn:%lli]\n", 838 loadQueue.back().instruction()->pcState(), 839 loadQueue.back().instruction()->seqNum); 840 841 if (isStalled() && loadQueue.tail() == stallingLoadIdx) { 842 stalled = false; 843 stallingStoreIsn = 0; 844 stallingLoadIdx = 0; 845 } 846 847 // Clear the smart pointer to make sure it is decremented. 848 loadQueue.back().instruction()->setSquashed(); 849 loadQueue.back().clear(); 850 851 --loads; 852 853 loadQueue.pop_back(); 854 ++lsqSquashedLoads; 855 } 856 857 if (memDepViolator && squashed_num < memDepViolator->seqNum) { 858 memDepViolator = NULL; 859 } 860 861 while (stores != 0 && 862 storeQueue.back().instruction()->seqNum > squashed_num) { 863 // Instructions marked as can WB are already committed. 864 if (storeQueue.back().canWB()) { 865 break; 866 } 867 868 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 869 "idx:%i [sn:%lli]\n", 870 storeQueue.back().instruction()->pcState(), 871 storeQueue.tail(), storeQueue.back().instruction()->seqNum); 872 873 // I don't think this can happen. It should have been cleared 874 // by the stalling load. 875 if (isStalled() && 876 storeQueue.back().instruction()->seqNum == stallingStoreIsn) { 877 panic("Is stalled should have been cleared by stalling load!\n"); 878 stalled = false; 879 stallingStoreIsn = 0; 880 } 881 882 // Clear the smart pointer to make sure it is decremented. 883 storeQueue.back().instruction()->setSquashed(); 884 885 // Must delete request now that it wasn't handed off to 886 // memory. This is quite ugly. @todo: Figure out the proper 887 // place to really handle request deletes. 888 storeQueue.back().clear(); 889 --stores; 890 891 storeQueue.pop_back(); 892 ++lsqSquashedStores; 893 } 894} 895 896template <class Impl> 897void 898LSQUnit<Impl>::storePostSend() 899{ 900 if (isStalled() && 901 storeWBIt->instruction()->seqNum == stallingStoreIsn) { 902 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 903 "load idx:%i\n", 904 stallingStoreIsn, stallingLoadIdx); 905 stalled = false; 906 stallingStoreIsn = 0; 907 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction()); 908 } 909 910 if (!storeWBIt->instruction()->isStoreConditional()) { 911 // The store is basically completed at this time. This 912 // only works so long as the checker doesn't try to 913 // verify the value in memory for stores. 914 storeWBIt->instruction()->setCompleted(); 915 916 if (cpu->checker) { 917 cpu->checker->verify(storeWBIt->instruction()); 918 } 919 } 920 921 if (needsTSO) { 922 storeInFlight = true; 923 } 924 925 storeWBIt++; 926} 927 928template <class Impl> 929void 930LSQUnit<Impl>::writeback(const DynInstPtr &inst, PacketPtr pkt) 931{ 932 iewStage->wakeCPU(); 933 934 // Squashed instructions do not need to complete their access. 935 if (inst->isSquashed()) { 936 assert(!inst->isStore()); 937 ++lsqIgnoredResponses; 938 return; 939 } 940 941 if (!inst->isExecuted()) { 942 inst->setExecuted(); 943 944 if (inst->fault == NoFault) { 945 // Complete access to copy data to proper place. 946 inst->completeAcc(pkt); 947 } else { 948 // If the instruction has an outstanding fault, we cannot complete 949 // the access as this discards the current fault. 950 951 // If we have an outstanding fault, the fault should only be of 952 // type ReExec. 953 assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr); 954 955 DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access " 956 "due to pending fault.\n", inst->seqNum); 957 } 958 } 959 960 // Need to insert instruction into queue to commit 961 iewStage->instToCommit(inst); 962 963 iewStage->activityThisCycle(); 964 965 // see if this load changed the PC 966 iewStage->checkMisprediction(inst); 967} 968 969template <class Impl> 970void 971LSQUnit<Impl>::completeStore(typename StoreQueue::iterator store_idx) 972{ 973 assert(store_idx->valid()); 974 store_idx->completed() = true; 975 --storesToWB; 976 // A bit conservative because a store completion may not free up entries, 977 // but hopefully avoids two store completions in one cycle from making 978 // the CPU tick twice. 979 cpu->wakeCPU(); 980 cpu->activityThisCycle(); 981 982 /* We 'need' a copy here because we may clear the entry from the 983 * store queue. */ 984 DynInstPtr store_inst = store_idx->instruction(); 985 if (store_idx == storeQueue.begin()) { 986 do { 987 storeQueue.front().clear(); 988 storeQueue.pop_front(); 989 --stores; 990 } while (storeQueue.front().completed() && 991 !storeQueue.empty()); 992 993 iewStage->updateLSQNextCycle = true; 994 } 995 996 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 997 "idx:%i\n", 998 store_inst->seqNum, store_idx.idx() - 1, storeQueue.head() - 1); 999 1000#if TRACING_ON 1001 if (DTRACE(O3PipeView)) { 1002 store_inst->storeTick = 1003 curTick() - store_inst->fetchTick; 1004 } 1005#endif 1006 1007 if (isStalled() && 1008 store_inst->seqNum == stallingStoreIsn) { 1009 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1010 "load idx:%i\n", 1011 stallingStoreIsn, stallingLoadIdx); 1012 stalled = false; 1013 stallingStoreIsn = 0; 1014 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction()); 1015 } 1016 1017 store_inst->setCompleted(); 1018 1019 if (needsTSO) { 1020 storeInFlight = false; 1021 } 1022 1023 // Tell the checker we've completed this instruction. Some stores 1024 // may get reported twice to the checker, but the checker can 1025 // handle that case. 1026 // Store conditionals cannot be sent to the checker yet, they have 1027 // to update the misc registers first which should take place 1028 // when they commit 1029 if (cpu->checker && !store_inst->isStoreConditional()) { 1030 cpu->checker->verify(store_inst); 1031 } 1032} 1033 1034template <class Impl> 1035bool 1036LSQUnit<Impl>::trySendPacket(bool isLoad, PacketPtr data_pkt) 1037{ 1038 bool ret = true; 1039 bool cache_got_blocked = false; 1040 1041 auto state = dynamic_cast<LSQSenderState*>(data_pkt->senderState); 1042 1043 if (!lsq->cacheBlocked() && 1044 lsq->cachePortAvailable(isLoad)) { 1045 if (!dcachePort->sendTimingReq(data_pkt)) { 1046 ret = false; 1047 cache_got_blocked = true; 1048 } 1049 } else { 1050 ret = false; 1051 } 1052 1053 if (ret) { 1054 if (!isLoad) { 1055 isStoreBlocked = false; 1056 } 1057 lsq->cachePortBusy(isLoad); 1058 state->outstanding++; 1059 state->request()->packetSent(); 1060 } else { 1061 if (cache_got_blocked) { 1062 lsq->cacheBlocked(true); 1063 ++lsqCacheBlocked; 1064 } 1065 if (!isLoad) { 1066 assert(state->request() == storeWBIt->request()); 1067 isStoreBlocked = true; 1068 } 1069 state->request()->packetNotSent(); 1070 } 1071 return ret; 1072} 1073 1074template <class Impl> 1075void 1076LSQUnit<Impl>::recvRetry() 1077{ 1078 if (isStoreBlocked) { 1079 DPRINTF(LSQUnit, "Receiving retry: blocked store\n"); 1080 writebackBlockedStore(); 1081 } 1082} 1083 1084template <class Impl> 1085void 1086LSQUnit<Impl>::dumpInsts() const 1087{ 1088 cprintf("Load store queue: Dumping instructions.\n"); 1089 cprintf("Load queue size: %i\n", loads); 1090 cprintf("Load queue: "); 1091 1092 for (const auto& e: loadQueue) { 1093 const DynInstPtr &inst(e.instruction()); 1094 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1095 } 1096 cprintf("\n"); 1097 1098 cprintf("Store queue size: %i\n", stores); 1099 cprintf("Store queue: "); 1100 1101 for (const auto& e: storeQueue) { 1102 const DynInstPtr &inst(e.instruction()); 1103 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1104 } 1105 1106 cprintf("\n"); 1107} 1108 1109template <class Impl> 1110unsigned int 1111LSQUnit<Impl>::cacheLineSize() 1112{ 1113 return cpu->cacheLineSize(); 1114} 1115 1116#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 1117