lsq_unit_impl.hh revision 14111
16657Snate@binkert.org 26657Snate@binkert.org/* 36657Snate@binkert.org * Copyright (c) 2010-2014, 2017-2018 ARM Limited 46657Snate@binkert.org * Copyright (c) 2013 Advanced Micro Devices, Inc. 56657Snate@binkert.org * All rights reserved 66657Snate@binkert.org * 76657Snate@binkert.org * The license below extends only to copyright in the software and shall 86657Snate@binkert.org * not be construed as granting a license to any other intellectual 96657Snate@binkert.org * property including but not limited to intellectual property relating 106657Snate@binkert.org * to a hardware implementation of the functionality of the software 116657Snate@binkert.org * licensed hereunder. You may use the software subject to the license 126657Snate@binkert.org * terms below provided that you ensure that this notice is replicated 136657Snate@binkert.org * unmodified and in its entirety in all distributions of the software, 146657Snate@binkert.org * modified or unmodified, in source code or in binary form. 156657Snate@binkert.org * 166657Snate@binkert.org * Copyright (c) 2004-2005 The Regents of The University of Michigan 176657Snate@binkert.org * All rights reserved. 186657Snate@binkert.org * 196657Snate@binkert.org * Redistribution and use in source and binary forms, with or without 206657Snate@binkert.org * modification, are permitted provided that the following conditions are 216657Snate@binkert.org * met: redistributions of source code must retain the above copyright 226657Snate@binkert.org * notice, this list of conditions and the following disclaimer; 236657Snate@binkert.org * redistributions in binary form must reproduce the above copyright 246657Snate@binkert.org * notice, this list of conditions and the following disclaimer in the 256657Snate@binkert.org * documentation and/or other materials provided with the distribution; 266657Snate@binkert.org * neither the name of the copyright holders nor the names of its 276657Snate@binkert.org * contributors may be used to endorse or promote products derived from 286657Snate@binkert.org * this software without specific prior written permission. 296657Snate@binkert.org * 306657Snate@binkert.org * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 316657Snate@binkert.org * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 326657Snate@binkert.org * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 336657Snate@binkert.org * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 346657Snate@binkert.org * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 356657Snate@binkert.org * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 366657Snate@binkert.org * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 376657Snate@binkert.org * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 386657Snate@binkert.org * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 396657Snate@binkert.org * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 406657Snate@binkert.org * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 416657Snate@binkert.org * 426657Snate@binkert.org * Authors: Kevin Lim 436657Snate@binkert.org * Korey Sewell 446657Snate@binkert.org */ 456657Snate@binkert.org 466657Snate@binkert.org#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 476657Snate@binkert.org#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 486657Snate@binkert.org 496657Snate@binkert.org#include "arch/generic/debugfaults.hh" 506657Snate@binkert.org#include "arch/locked_mem.hh" 516657Snate@binkert.org#include "base/str.hh" 526657Snate@binkert.org#include "config/the_isa.hh" 536657Snate@binkert.org#include "cpu/checker/cpu.hh" 546657Snate@binkert.org#include "cpu/o3/lsq.hh" 556657Snate@binkert.org#include "cpu/o3/lsq_unit.hh" 566657Snate@binkert.org#include "debug/Activity.hh" 576657Snate@binkert.org#include "debug/IEW.hh" 586657Snate@binkert.org#include "debug/LSQUnit.hh" 596657Snate@binkert.org#include "debug/O3PipeView.hh" 606657Snate@binkert.org#include "mem/packet.hh" 616657Snate@binkert.org#include "mem/request.hh" 626657Snate@binkert.org 636657Snate@binkert.orgtemplate<class Impl> 646657Snate@binkert.orgLSQUnit<Impl>::WritebackEvent::WritebackEvent(const DynInstPtr &_inst, 656657Snate@binkert.org PacketPtr _pkt, LSQUnit *lsq_ptr) 666657Snate@binkert.org : Event(Default_Pri, AutoDelete), 676657Snate@binkert.org inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 686657Snate@binkert.org{ 696657Snate@binkert.org assert(_inst->savedReq); 706657Snate@binkert.org _inst->savedReq->writebackScheduled(); 716657Snate@binkert.org} 726657Snate@binkert.org 736657Snate@binkert.orgtemplate<class Impl> 746657Snate@binkert.orgvoid 756657Snate@binkert.orgLSQUnit<Impl>::WritebackEvent::process() 766657Snate@binkert.org{ 776657Snate@binkert.org assert(!lsqPtr->cpu->switchedOut()); 786657Snate@binkert.org 796657Snate@binkert.org lsqPtr->writeback(inst, pkt); 806657Snate@binkert.org 816657Snate@binkert.org assert(inst->savedReq); 826657Snate@binkert.org inst->savedReq->writebackDone(); 836657Snate@binkert.org delete pkt; 846657Snate@binkert.org} 856657Snate@binkert.org 866657Snate@binkert.orgtemplate<class Impl> 876657Snate@binkert.orgconst char * 886657Snate@binkert.orgLSQUnit<Impl>::WritebackEvent::description() const 896657Snate@binkert.org{ 906657Snate@binkert.org return "Store writeback"; 916657Snate@binkert.org} 926657Snate@binkert.org 936657Snate@binkert.orgtemplate <class Impl> 946657Snate@binkert.orgbool 956657Snate@binkert.orgLSQUnit<Impl>::recvTimingResp(PacketPtr pkt) 966657Snate@binkert.org{ 976657Snate@binkert.org auto senderState = dynamic_cast<LSQSenderState*>(pkt->senderState); 986657Snate@binkert.org LSQRequest* req = senderState->request(); 996657Snate@binkert.org assert(req != nullptr); 1006657Snate@binkert.org bool ret = true; 1016657Snate@binkert.org /* Check that the request is still alive before any further action. */ 1026657Snate@binkert.org if (senderState->alive()) { 1036657Snate@binkert.org ret = req->recvTimingResp(pkt); 1046657Snate@binkert.org } else { 1056657Snate@binkert.org senderState->outstanding--; 1066657Snate@binkert.org } 1076657Snate@binkert.org return ret; 1086657Snate@binkert.org 1096657Snate@binkert.org} 1106657Snate@binkert.org 1116657Snate@binkert.orgtemplate<class Impl> 1126657Snate@binkert.orgvoid 1136657Snate@binkert.orgLSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 1146657Snate@binkert.org{ 1156657Snate@binkert.org LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 1166657Snate@binkert.org DynInstPtr inst = state->inst; 1176657Snate@binkert.org 1186657Snate@binkert.org cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1196657Snate@binkert.org 1206657Snate@binkert.org /* Notify the sender state that the access is complete (for ownership 1216657Snate@binkert.org * tracking). */ 1226657Snate@binkert.org state->complete(); 1236657Snate@binkert.org 1246657Snate@binkert.org assert(!cpu->switchedOut()); 1256657Snate@binkert.org if (!inst->isSquashed()) { 1266657Snate@binkert.org if (state->needWB) { 1276657Snate@binkert.org // Only loads, store conditionals and atomics perform the writeback 1286657Snate@binkert.org // after receving the response from the memory 1296657Snate@binkert.org assert(inst->isLoad() || inst->isStoreConditional() || 1306657Snate@binkert.org inst->isAtomic()); 1316657Snate@binkert.org writeback(inst, state->request()->mainPacket()); 1326657Snate@binkert.org if (inst->isStore() || inst->isAtomic()) { 1336657Snate@binkert.org auto ss = dynamic_cast<SQSenderState*>(state); 1346657Snate@binkert.org ss->writebackDone(); 1356657Snate@binkert.org completeStore(ss->idx); 1366657Snate@binkert.org } 1376657Snate@binkert.org } else if (inst->isStore()) { 1386657Snate@binkert.org // This is a regular store (i.e., not store conditionals and 1396657Snate@binkert.org // atomics), so it can complete without writing back 1406657Snate@binkert.org completeStore(dynamic_cast<SQSenderState*>(state)->idx); 1416657Snate@binkert.org } 1426657Snate@binkert.org } 1436657Snate@binkert.org} 1446657Snate@binkert.org 1456657Snate@binkert.orgtemplate <class Impl> 1466657Snate@binkert.orgLSQUnit<Impl>::LSQUnit(uint32_t lqEntries, uint32_t sqEntries) 1476657Snate@binkert.org : lsqID(-1), storeQueue(sqEntries+1), loadQueue(lqEntries+1), 1486657Snate@binkert.org loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 1496657Snate@binkert.org isStoreBlocked(false), storeInFlight(false), hasPendingRequest(false), 1506657Snate@binkert.org pendingRequest(nullptr) 1516657Snate@binkert.org{ 1526657Snate@binkert.org} 1536657Snate@binkert.org 1546657Snate@binkert.orgtemplate<class Impl> 1556657Snate@binkert.orgvoid 1566657Snate@binkert.orgLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 1576657Snate@binkert.org LSQ *lsq_ptr, unsigned id) 1586657Snate@binkert.org{ 1596657Snate@binkert.org lsqID = id; 1606657Snate@binkert.org 1616657Snate@binkert.org cpu = cpu_ptr; 1626657Snate@binkert.org iewStage = iew_ptr; 1636657Snate@binkert.org 1646657Snate@binkert.org lsq = lsq_ptr; 1656657Snate@binkert.org 1666657Snate@binkert.org DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",lsqID); 1676657Snate@binkert.org 1686657Snate@binkert.org depCheckShift = params->LSQDepCheckShift; 1696657Snate@binkert.org checkLoads = params->LSQCheckLoads; 1706657Snate@binkert.org needsTSO = params->needsTSO; 1716657Snate@binkert.org 1726657Snate@binkert.org resetState(); 1736657Snate@binkert.org} 1746657Snate@binkert.org 1756657Snate@binkert.org 1766657Snate@binkert.orgtemplate<class Impl> 1776657Snate@binkert.orgvoid 1786657Snate@binkert.orgLSQUnit<Impl>::resetState() 1796657Snate@binkert.org{ 1806657Snate@binkert.org loads = stores = storesToWB = 0; 1816657Snate@binkert.org 1826657Snate@binkert.org 1836657Snate@binkert.org storeWBIt = storeQueue.begin(); 1846657Snate@binkert.org 1856657Snate@binkert.org retryPkt = NULL; 1866657Snate@binkert.org memDepViolator = NULL; 1876657Snate@binkert.org 1886657Snate@binkert.org stalled = false; 1896657Snate@binkert.org 1906657Snate@binkert.org cacheBlockMask = ~(cpu->cacheLineSize() - 1); 1916657Snate@binkert.org} 1926657Snate@binkert.org 1936657Snate@binkert.orgtemplate<class Impl> 1946657Snate@binkert.orgstd::string 1956657Snate@binkert.orgLSQUnit<Impl>::name() const 1966657Snate@binkert.org{ 1976657Snate@binkert.org if (Impl::MaxThreads == 1) { 1986657Snate@binkert.org return iewStage->name() + ".lsq"; 1996657Snate@binkert.org } else { 2006657Snate@binkert.org return iewStage->name() + ".lsq.thread" + std::to_string(lsqID); 2016657Snate@binkert.org } 2026657Snate@binkert.org} 2036657Snate@binkert.org 2046657Snate@binkert.orgtemplate<class Impl> 2056657Snate@binkert.orgvoid 2066657Snate@binkert.orgLSQUnit<Impl>::regStats() 2076657Snate@binkert.org{ 2086657Snate@binkert.org lsqForwLoads 2096657Snate@binkert.org .name(name() + ".forwLoads") 2106657Snate@binkert.org .desc("Number of loads that had data forwarded from stores"); 2116657Snate@binkert.org 2126657Snate@binkert.org invAddrLoads 2136657Snate@binkert.org .name(name() + ".invAddrLoads") 2146657Snate@binkert.org .desc("Number of loads ignored due to an invalid address"); 2156657Snate@binkert.org 2166657Snate@binkert.org lsqSquashedLoads 2176657Snate@binkert.org .name(name() + ".squashedLoads") 2186657Snate@binkert.org .desc("Number of loads squashed"); 2196657Snate@binkert.org 2206657Snate@binkert.org lsqIgnoredResponses 2216657Snate@binkert.org .name(name() + ".ignoredResponses") 2226657Snate@binkert.org .desc("Number of memory responses ignored because the instruction is squashed"); 2236657Snate@binkert.org 2246657Snate@binkert.org lsqMemOrderViolation 2256657Snate@binkert.org .name(name() + ".memOrderViolation") 2266657Snate@binkert.org .desc("Number of memory ordering violations"); 2276657Snate@binkert.org 2286657Snate@binkert.org lsqSquashedStores 2296657Snate@binkert.org .name(name() + ".squashedStores") 2306657Snate@binkert.org .desc("Number of stores squashed"); 2316657Snate@binkert.org 2326657Snate@binkert.org invAddrSwpfs 2336657Snate@binkert.org .name(name() + ".invAddrSwpfs") 2346657Snate@binkert.org .desc("Number of software prefetches ignored due to an invalid address"); 2356657Snate@binkert.org 2366657Snate@binkert.org lsqBlockedLoads 2376657Snate@binkert.org .name(name() + ".blockedLoads") 2386657Snate@binkert.org .desc("Number of blocked loads due to partial load-store forwarding"); 2396657Snate@binkert.org 2406657Snate@binkert.org lsqRescheduledLoads 2416657Snate@binkert.org .name(name() + ".rescheduledLoads") 2426657Snate@binkert.org .desc("Number of loads that were rescheduled"); 2436657Snate@binkert.org 2446657Snate@binkert.org lsqCacheBlocked 2456657Snate@binkert.org .name(name() + ".cacheBlocked") 2466657Snate@binkert.org .desc("Number of times an access to memory failed due to the cache being blocked"); 2476657Snate@binkert.org} 2486657Snate@binkert.org 2496657Snate@binkert.orgtemplate<class Impl> 2506657Snate@binkert.orgvoid 2516657Snate@binkert.orgLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2526657Snate@binkert.org{ 2536657Snate@binkert.org dcachePort = dcache_port; 2546657Snate@binkert.org} 2556657Snate@binkert.org 2566657Snate@binkert.orgtemplate<class Impl> 2576657Snate@binkert.orgvoid 2586657Snate@binkert.orgLSQUnit<Impl>::drainSanityCheck() const 2596657Snate@binkert.org{ 2606657Snate@binkert.org for (int i = 0; i < loadQueue.capacity(); ++i) 2616657Snate@binkert.org assert(!loadQueue[i].valid()); 2626657Snate@binkert.org 2636657Snate@binkert.org assert(storesToWB == 0); 2646657Snate@binkert.org assert(!retryPkt); 2656657Snate@binkert.org} 2666657Snate@binkert.org 2676657Snate@binkert.orgtemplate<class Impl> 2686657Snate@binkert.orgvoid 2696657Snate@binkert.orgLSQUnit<Impl>::takeOverFrom() 2706657Snate@binkert.org{ 2716657Snate@binkert.org resetState(); 2726657Snate@binkert.org} 2736657Snate@binkert.org 2746657Snate@binkert.orgtemplate <class Impl> 2756657Snate@binkert.orgvoid 2766657Snate@binkert.orgLSQUnit<Impl>::insert(const DynInstPtr &inst) 2776657Snate@binkert.org{ 2786657Snate@binkert.org assert(inst->isMemRef()); 2796657Snate@binkert.org 2806657Snate@binkert.org assert(inst->isLoad() || inst->isStore() || inst->isAtomic()); 2816657Snate@binkert.org 2826657Snate@binkert.org if (inst->isLoad()) { 2836657Snate@binkert.org insertLoad(inst); 2846657Snate@binkert.org } else { 2856657Snate@binkert.org insertStore(inst); 2866657Snate@binkert.org } 2876657Snate@binkert.org 2886657Snate@binkert.org inst->setInLSQ(); 2896657Snate@binkert.org} 2906657Snate@binkert.org 2916657Snate@binkert.orgtemplate <class Impl> 2926657Snate@binkert.orgvoid 2936657Snate@binkert.orgLSQUnit<Impl>::insertLoad(const DynInstPtr &load_inst) 2946657Snate@binkert.org{ 2956657Snate@binkert.org assert(!loadQueue.full()); 2966657Snate@binkert.org assert(loads < loadQueue.capacity()); 2976657Snate@binkert.org 2986657Snate@binkert.org DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 2996657Snate@binkert.org load_inst->pcState(), loadQueue.tail(), load_inst->seqNum); 3006657Snate@binkert.org 3016657Snate@binkert.org /* Grow the queue. */ 3026657Snate@binkert.org loadQueue.advance_tail(); 3036657Snate@binkert.org 3046657Snate@binkert.org load_inst->sqIt = storeQueue.end(); 3056657Snate@binkert.org 3066657Snate@binkert.org assert(!loadQueue.back().valid()); 3076657Snate@binkert.org loadQueue.back().set(load_inst); 3086657Snate@binkert.org load_inst->lqIdx = loadQueue.tail(); 3096657Snate@binkert.org load_inst->lqIt = loadQueue.getIterator(load_inst->lqIdx); 3106657Snate@binkert.org 3116657Snate@binkert.org ++loads; 3126657Snate@binkert.org} 3136657Snate@binkert.org 3146657Snate@binkert.orgtemplate <class Impl> 3156657Snate@binkert.orgvoid 3166657Snate@binkert.orgLSQUnit<Impl>::insertStore(const DynInstPtr& store_inst) 3176657Snate@binkert.org{ 3186657Snate@binkert.org // Make sure it is not full before inserting an instruction. 3196657Snate@binkert.org assert(!storeQueue.full()); 3206657Snate@binkert.org assert(stores < storeQueue.capacity()); 3216657Snate@binkert.org 3226657Snate@binkert.org DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 3236657Snate@binkert.org store_inst->pcState(), storeQueue.tail(), store_inst->seqNum); 3246657Snate@binkert.org storeQueue.advance_tail(); 3256657Snate@binkert.org 3266657Snate@binkert.org store_inst->sqIdx = storeQueue.tail(); 3276657Snate@binkert.org store_inst->lqIdx = loadQueue.moduloAdd(loadQueue.tail(), 1); 3286657Snate@binkert.org store_inst->lqIt = loadQueue.end(); 3296657Snate@binkert.org 3306657Snate@binkert.org storeQueue.back().set(store_inst); 3316657Snate@binkert.org 3326657Snate@binkert.org ++stores; 3336657Snate@binkert.org} 3346657Snate@binkert.org 3356657Snate@binkert.orgtemplate <class Impl> 3366657Snate@binkert.orgtypename Impl::DynInstPtr 3376657Snate@binkert.orgLSQUnit<Impl>::getMemDepViolator() 3386657Snate@binkert.org{ 3396657Snate@binkert.org DynInstPtr temp = memDepViolator; 3406657Snate@binkert.org 3416657Snate@binkert.org memDepViolator = NULL; 3426657Snate@binkert.org 3436657Snate@binkert.org return temp; 3446657Snate@binkert.org} 3456657Snate@binkert.org 3466657Snate@binkert.orgtemplate <class Impl> 3476657Snate@binkert.orgunsigned 3486657Snate@binkert.orgLSQUnit<Impl>::numFreeLoadEntries() 3496657Snate@binkert.org{ 3506657Snate@binkert.org //LQ has an extra dummy entry to differentiate 3516657Snate@binkert.org //empty/full conditions. Subtract 1 from the free entries. 3526657Snate@binkert.org DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", 3536657Snate@binkert.org 1 + loadQueue.capacity(), loads); 3546657Snate@binkert.org return loadQueue.capacity() - loads; 3556657Snate@binkert.org} 3566657Snate@binkert.org 3576657Snate@binkert.orgtemplate <class Impl> 3586657Snate@binkert.orgunsigned 3596657Snate@binkert.orgLSQUnit<Impl>::numFreeStoreEntries() 3606657Snate@binkert.org{ 3616657Snate@binkert.org //SQ has an extra dummy entry to differentiate 3626657Snate@binkert.org //empty/full conditions. Subtract 1 from the free entries. 3636657Snate@binkert.org DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", 3646657Snate@binkert.org 1 + storeQueue.capacity(), stores); 3656657Snate@binkert.org return storeQueue.capacity() - stores; 3666657Snate@binkert.org 3676657Snate@binkert.org } 3686657Snate@binkert.org 3696657Snate@binkert.orgtemplate <class Impl> 3706657Snate@binkert.orgvoid 3716657Snate@binkert.orgLSQUnit<Impl>::checkSnoop(PacketPtr pkt) 3726657Snate@binkert.org{ 3736657Snate@binkert.org // Should only ever get invalidations in here 3746657Snate@binkert.org assert(pkt->isInvalidate()); 3756657Snate@binkert.org 3766657Snate@binkert.org DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 3776657Snate@binkert.org 3786657Snate@binkert.org for (int x = 0; x < cpu->numContexts(); x++) { 3796657Snate@binkert.org ThreadContext *tc = cpu->getContext(x); 3806657Snate@binkert.org bool no_squash = cpu->thread[x]->noSquashFromTC; 3816657Snate@binkert.org cpu->thread[x]->noSquashFromTC = true; 3826657Snate@binkert.org TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 3836657Snate@binkert.org cpu->thread[x]->noSquashFromTC = no_squash; 3846657Snate@binkert.org } 3856657Snate@binkert.org 3866657Snate@binkert.org if (loadQueue.empty()) 3876657Snate@binkert.org return; 3886657Snate@binkert.org 3896657Snate@binkert.org auto iter = loadQueue.begin(); 3906657Snate@binkert.org 3916657Snate@binkert.org Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 3926657Snate@binkert.org 3936657Snate@binkert.org DynInstPtr ld_inst = iter->instruction(); 3946657Snate@binkert.org assert(ld_inst); 3956657Snate@binkert.org LSQRequest *req = iter->request(); 3966657Snate@binkert.org 3976657Snate@binkert.org // Check that this snoop didn't just invalidate our lock flag 3986657Snate@binkert.org if (ld_inst->effAddrValid() && 3996657Snate@binkert.org req->isCacheBlockHit(invalidate_addr, cacheBlockMask) 4006657Snate@binkert.org && ld_inst->memReqFlags & Request::LLSC) 4016657Snate@binkert.org TheISA::handleLockedSnoopHit(ld_inst.get()); 4026657Snate@binkert.org 4036657Snate@binkert.org bool force_squash = false; 4046657Snate@binkert.org 4056657Snate@binkert.org while (++iter != loadQueue.end()) { 4066657Snate@binkert.org ld_inst = iter->instruction(); 4076657Snate@binkert.org assert(ld_inst); 4086657Snate@binkert.org req = iter->request(); 4096657Snate@binkert.org if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) 4106657Snate@binkert.org continue; 4116657Snate@binkert.org 4126657Snate@binkert.org DPRINTF(LSQUnit, "-- inst [sn:%lli] to pktAddr:%#x\n", 4136657Snate@binkert.org ld_inst->seqNum, invalidate_addr); 4146657Snate@binkert.org 4156657Snate@binkert.org if (force_squash || 4166657Snate@binkert.org req->isCacheBlockHit(invalidate_addr, cacheBlockMask)) { 4176657Snate@binkert.org if (needsTSO) { 4186657Snate@binkert.org // If we have a TSO system, as all loads must be ordered with 4196657Snate@binkert.org // all other loads, this load as well as *all* subsequent loads 4206657Snate@binkert.org // need to be squashed to prevent possible load reordering. 4216657Snate@binkert.org force_squash = true; 4226657Snate@binkert.org } 4236657Snate@binkert.org if (ld_inst->possibleLoadViolation() || force_squash) { 4246657Snate@binkert.org DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 4256657Snate@binkert.org pkt->getAddr(), ld_inst->seqNum); 4266657Snate@binkert.org 4276657Snate@binkert.org // Mark the load for re-execution 4286657Snate@binkert.org ld_inst->fault = std::make_shared<ReExec>(); 4296657Snate@binkert.org } else { 4306657Snate@binkert.org DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 4316657Snate@binkert.org pkt->getAddr(), ld_inst->seqNum); 4326657Snate@binkert.org 4336862Sdrh5@cs.wisc.edu // Make sure that we don't lose a snoop hitting a LOCKED 4346862Sdrh5@cs.wisc.edu // address since the LOCK* flags don't get updated until 4356862Sdrh5@cs.wisc.edu // commit. 4366862Sdrh5@cs.wisc.edu if (ld_inst->memReqFlags & Request::LLSC) 4376657Snate@binkert.org TheISA::handleLockedSnoopHit(ld_inst.get()); 4386657Snate@binkert.org 4396657Snate@binkert.org // If a older load checks this and it's true 4406657Snate@binkert.org // then we might have missed the snoop 4416657Snate@binkert.org // in which case we need to invalidate to be sure 4426657Snate@binkert.org ld_inst->hitExternalSnoop(true); 4436657Snate@binkert.org } 4446657Snate@binkert.org } 4456657Snate@binkert.org } 4466657Snate@binkert.org return; 4476657Snate@binkert.org} 4486657Snate@binkert.org 4496657Snate@binkert.orgtemplate <class Impl> 4506657Snate@binkert.orgFault 4516657Snate@binkert.orgLSQUnit<Impl>::checkViolations(typename LoadQueue::iterator& loadIt, 4526657Snate@binkert.org const DynInstPtr& inst) 4536657Snate@binkert.org{ 4546657Snate@binkert.org Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 4556657Snate@binkert.org Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 4566657Snate@binkert.org 4576657Snate@binkert.org /** @todo in theory you only need to check an instruction that has executed 4586657Snate@binkert.org * however, there isn't a good way in the pipeline at the moment to check 4596657Snate@binkert.org * all instructions that will execute before the store writes back. Thus, 4606657Snate@binkert.org * like the implementation that came before it, we're overly conservative. 4616657Snate@binkert.org */ 4626657Snate@binkert.org while (loadIt != loadQueue.end()) { 4636657Snate@binkert.org DynInstPtr ld_inst = loadIt->instruction(); 4646657Snate@binkert.org if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) { 4656657Snate@binkert.org ++loadIt; 4666657Snate@binkert.org continue; 4676657Snate@binkert.org } 4686657Snate@binkert.org 4696657Snate@binkert.org Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 4706657Snate@binkert.org Addr ld_eff_addr2 = 4716657Snate@binkert.org (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 4726657Snate@binkert.org 4736657Snate@binkert.org if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 4746657Snate@binkert.org if (inst->isLoad()) { 4756657Snate@binkert.org // If this load is to the same block as an external snoop 4766657Snate@binkert.org // invalidate that we've observed then the load needs to be 4776657Snate@binkert.org // squashed as it could have newer data 4786657Snate@binkert.org if (ld_inst->hitExternalSnoop()) { 4796657Snate@binkert.org if (!memDepViolator || 4806657Snate@binkert.org ld_inst->seqNum < memDepViolator->seqNum) { 4816657Snate@binkert.org DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 4826657Snate@binkert.org "and [sn:%lli] at address %#x\n", 4836657Snate@binkert.org inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 4846657Snate@binkert.org memDepViolator = ld_inst; 4856657Snate@binkert.org 4866657Snate@binkert.org ++lsqMemOrderViolation; 4876657Snate@binkert.org 4886657Snate@binkert.org return std::make_shared<GenericISA::M5PanicFault>( 4896657Snate@binkert.org "Detected fault with inst [sn:%lli] and " 4906657Snate@binkert.org "[sn:%lli] at address %#x\n", 4916657Snate@binkert.org inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 4926657Snate@binkert.org } 4936657Snate@binkert.org } 4946657Snate@binkert.org 4956657Snate@binkert.org // Otherwise, mark the load has a possible load violation 4966657Snate@binkert.org // and if we see a snoop before it's commited, we need to squash 4976657Snate@binkert.org ld_inst->possibleLoadViolation(true); 4986657Snate@binkert.org DPRINTF(LSQUnit, "Found possible load violation at addr: %#x" 4996657Snate@binkert.org " between instructions [sn:%lli] and [sn:%lli]\n", 5006657Snate@binkert.org inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 5016657Snate@binkert.org } else { 5026657Snate@binkert.org // A load/store incorrectly passed this store. 5036657Snate@binkert.org // Check if we already have a violator, or if it's newer 5046657Snate@binkert.org // squash and refetch. 5056657Snate@binkert.org if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 5066657Snate@binkert.org break; 5076657Snate@binkert.org 5086657Snate@binkert.org DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 5096657Snate@binkert.org "[sn:%lli] at address %#x\n", 5106657Snate@binkert.org inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5116657Snate@binkert.org memDepViolator = ld_inst; 5126657Snate@binkert.org 5136657Snate@binkert.org ++lsqMemOrderViolation; 5146657Snate@binkert.org 5156657Snate@binkert.org return std::make_shared<GenericISA::M5PanicFault>( 5166657Snate@binkert.org "Detected fault with " 5176657Snate@binkert.org "inst [sn:%lli] and [sn:%lli] at address %#x\n", 5186657Snate@binkert.org inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5196657Snate@binkert.org } 5206657Snate@binkert.org } 5216657Snate@binkert.org 5226657Snate@binkert.org ++loadIt; 5236657Snate@binkert.org } 5246657Snate@binkert.org return NoFault; 5256657Snate@binkert.org} 5266657Snate@binkert.org 5276657Snate@binkert.org 5286657Snate@binkert.org 5296657Snate@binkert.org 5306657Snate@binkert.orgtemplate <class Impl> 5316657Snate@binkert.orgFault 5326657Snate@binkert.orgLSQUnit<Impl>::executeLoad(const DynInstPtr &inst) 5336657Snate@binkert.org{ 5346657Snate@binkert.org using namespace TheISA; 5356657Snate@binkert.org // Execute a specific load. 5366657Snate@binkert.org Fault load_fault = NoFault; 5376657Snate@binkert.org 5386657Snate@binkert.org DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 5396657Snate@binkert.org inst->pcState(), inst->seqNum); 5406657Snate@binkert.org 5416657Snate@binkert.org assert(!inst->isSquashed()); 5426657Snate@binkert.org 5436657Snate@binkert.org load_fault = inst->initiateAcc(); 5446657Snate@binkert.org 5456657Snate@binkert.org if (load_fault == NoFault && !inst->readMemAccPredicate()) { 5466657Snate@binkert.org assert(inst->readPredicate()); 5476657Snate@binkert.org inst->setExecuted(); 5486657Snate@binkert.org inst->completeAcc(nullptr); 5496657Snate@binkert.org iewStage->instToCommit(inst); 5506657Snate@binkert.org iewStage->activityThisCycle(); 5516657Snate@binkert.org return NoFault; 5526657Snate@binkert.org } 5536657Snate@binkert.org 5546657Snate@binkert.org if (inst->isTranslationDelayed() && load_fault == NoFault) 5556657Snate@binkert.org return load_fault; 5566657Snate@binkert.org 5576657Snate@binkert.org if (load_fault != NoFault && inst->translationCompleted() && 5586657Snate@binkert.org inst->savedReq->isPartialFault() && !inst->savedReq->isComplete()) { 5596657Snate@binkert.org assert(inst->savedReq->isSplit()); 5606657Snate@binkert.org // If we have a partial fault where the mem access is not complete yet 5616657Snate@binkert.org // then the cache must have been blocked. This load will be re-executed 5626657Snate@binkert.org // when the cache gets unblocked. We will handle the fault when the 5636657Snate@binkert.org // mem access is complete. 5646657Snate@binkert.org return NoFault; 5656657Snate@binkert.org } 5666657Snate@binkert.org 5676657Snate@binkert.org // If the instruction faulted or predicated false, then we need to send it 5686657Snate@binkert.org // along to commit without the instruction completing. 5696657Snate@binkert.org if (load_fault != NoFault || !inst->readPredicate()) { 5706657Snate@binkert.org // Send this instruction to commit, also make sure iew stage 5716657Snate@binkert.org // realizes there is activity. Mark it as executed unless it 5726657Snate@binkert.org // is a strictly ordered load that needs to hit the head of 5736657Snate@binkert.org // commit. 5746657Snate@binkert.org if (!inst->readPredicate()) 5756657Snate@binkert.org inst->forwardOldRegs(); 5766657Snate@binkert.org DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 5776657Snate@binkert.org inst->seqNum, 5786657Snate@binkert.org (load_fault != NoFault ? "fault" : "predication")); 5796657Snate@binkert.org if (!(inst->hasRequest() && inst->strictlyOrdered()) || 5806657Snate@binkert.org inst->isAtCommit()) { 5816657Snate@binkert.org inst->setExecuted(); 5826657Snate@binkert.org } 5836657Snate@binkert.org iewStage->instToCommit(inst); 5846657Snate@binkert.org iewStage->activityThisCycle(); 5856657Snate@binkert.org } else { 5866657Snate@binkert.org if (inst->effAddrValid()) { 5876657Snate@binkert.org auto it = inst->lqIt; 5886657Snate@binkert.org ++it; 5896657Snate@binkert.org 5906657Snate@binkert.org if (checkLoads) 5916657Snate@binkert.org return checkViolations(it, inst); 5926657Snate@binkert.org } 5936657Snate@binkert.org } 5946657Snate@binkert.org 5956657Snate@binkert.org return load_fault; 5966657Snate@binkert.org} 5976657Snate@binkert.org 5986657Snate@binkert.orgtemplate <class Impl> 5996657Snate@binkert.orgFault 6006657Snate@binkert.orgLSQUnit<Impl>::executeStore(const DynInstPtr &store_inst) 6016657Snate@binkert.org{ 6026657Snate@binkert.org using namespace TheISA; 6036657Snate@binkert.org // Make sure that a store exists. 6046657Snate@binkert.org assert(stores != 0); 6056657Snate@binkert.org 6066657Snate@binkert.org int store_idx = store_inst->sqIdx; 6076657Snate@binkert.org 6086657Snate@binkert.org DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 6096657Snate@binkert.org store_inst->pcState(), store_inst->seqNum); 6106657Snate@binkert.org 6116657Snate@binkert.org assert(!store_inst->isSquashed()); 6126657Snate@binkert.org 6136657Snate@binkert.org // Check the recently completed loads to see if any match this store's 6146657Snate@binkert.org // address. If so, then we have a memory ordering violation. 6156657Snate@binkert.org typename LoadQueue::iterator loadIt = store_inst->lqIt; 6166657Snate@binkert.org 6176657Snate@binkert.org Fault store_fault = store_inst->initiateAcc(); 6186657Snate@binkert.org 6196657Snate@binkert.org if (store_inst->isTranslationDelayed() && 6206657Snate@binkert.org store_fault == NoFault) 6216657Snate@binkert.org return store_fault; 6226657Snate@binkert.org 6236657Snate@binkert.org if (!store_inst->readPredicate()) { 6246657Snate@binkert.org DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 6256657Snate@binkert.org store_inst->seqNum); 6266657Snate@binkert.org store_inst->forwardOldRegs(); 6276657Snate@binkert.org return store_fault; 6286657Snate@binkert.org } 6296657Snate@binkert.org 6306657Snate@binkert.org if (storeQueue[store_idx].size() == 0) { 6316657Snate@binkert.org DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 6326657Snate@binkert.org store_inst->pcState(), store_inst->seqNum); 6336657Snate@binkert.org 6346657Snate@binkert.org return store_fault; 6356657Snate@binkert.org } 6366657Snate@binkert.org 6376657Snate@binkert.org assert(store_fault == NoFault); 6386657Snate@binkert.org 6396657Snate@binkert.org if (store_inst->isStoreConditional() || store_inst->isAtomic()) { 6406657Snate@binkert.org // Store conditionals and Atomics need to set themselves as able to 6416657Snate@binkert.org // writeback if we haven't had a fault by here. 6426657Snate@binkert.org storeQueue[store_idx].canWB() = true; 6436657Snate@binkert.org 6446657Snate@binkert.org ++storesToWB; 6456657Snate@binkert.org } 6466657Snate@binkert.org 6476657Snate@binkert.org return checkViolations(loadIt, store_inst); 6486657Snate@binkert.org 6496657Snate@binkert.org} 6506657Snate@binkert.org 6516657Snate@binkert.orgtemplate <class Impl> 6526657Snate@binkert.orgvoid 653LSQUnit<Impl>::commitLoad() 654{ 655 assert(loadQueue.front().valid()); 656 657 DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 658 loadQueue.front().instruction()->pcState()); 659 660 loadQueue.front().clear(); 661 loadQueue.pop_front(); 662 663 --loads; 664} 665 666template <class Impl> 667void 668LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 669{ 670 assert(loads == 0 || loadQueue.front().valid()); 671 672 while (loads != 0 && loadQueue.front().instruction()->seqNum 673 <= youngest_inst) { 674 commitLoad(); 675 } 676} 677 678template <class Impl> 679void 680LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 681{ 682 assert(stores == 0 || storeQueue.front().valid()); 683 684 /* Forward iterate the store queue (age order). */ 685 for (auto& x : storeQueue) { 686 assert(x.valid()); 687 // Mark any stores that are now committed and have not yet 688 // been marked as able to write back. 689 if (!x.canWB()) { 690 if (x.instruction()->seqNum > youngest_inst) { 691 break; 692 } 693 DPRINTF(LSQUnit, "Marking store as able to write back, PC " 694 "%s [sn:%lli]\n", 695 x.instruction()->pcState(), 696 x.instruction()->seqNum); 697 698 x.canWB() = true; 699 700 ++storesToWB; 701 } 702 } 703} 704 705template <class Impl> 706void 707LSQUnit<Impl>::writebackBlockedStore() 708{ 709 assert(isStoreBlocked); 710 storeWBIt->request()->sendPacketToCache(); 711 if (storeWBIt->request()->isSent()){ 712 storePostSend(); 713 } 714} 715 716template <class Impl> 717void 718LSQUnit<Impl>::writebackStores() 719{ 720 if (isStoreBlocked) { 721 DPRINTF(LSQUnit, "Writing back blocked store\n"); 722 writebackBlockedStore(); 723 } 724 725 while (storesToWB > 0 && 726 storeWBIt.dereferenceable() && 727 storeWBIt->valid() && 728 storeWBIt->canWB() && 729 ((!needsTSO) || (!storeInFlight)) && 730 lsq->cachePortAvailable(false)) { 731 732 if (isStoreBlocked) { 733 DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 734 " is blocked!\n"); 735 break; 736 } 737 738 // Store didn't write any data so no need to write it back to 739 // memory. 740 if (storeWBIt->size() == 0) { 741 /* It is important that the preincrement happens at (or before) 742 * the call, as the the code of completeStore checks 743 * storeWBIt. */ 744 completeStore(storeWBIt++); 745 continue; 746 } 747 748 if (storeWBIt->instruction()->isDataPrefetch()) { 749 storeWBIt++; 750 continue; 751 } 752 753 assert(storeWBIt->hasRequest()); 754 assert(!storeWBIt->committed()); 755 756 DynInstPtr inst = storeWBIt->instruction(); 757 LSQRequest* req = storeWBIt->request(); 758 storeWBIt->committed() = true; 759 760 assert(!inst->memData); 761 inst->memData = new uint8_t[req->_size]; 762 763 if (storeWBIt->isAllZeros()) 764 memset(inst->memData, 0, req->_size); 765 else 766 memcpy(inst->memData, storeWBIt->data(), req->_size); 767 768 769 if (req->senderState() == nullptr) { 770 SQSenderState *state = new SQSenderState(storeWBIt); 771 state->isLoad = false; 772 state->needWB = false; 773 state->inst = inst; 774 775 req->senderState(state); 776 if (inst->isStoreConditional() || inst->isAtomic()) { 777 /* Only store conditionals and atomics need a writeback. */ 778 state->needWB = true; 779 } 780 } 781 req->buildPackets(); 782 783 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 784 "to Addr:%#x, data:%#x [sn:%lli]\n", 785 storeWBIt.idx(), inst->pcState(), 786 req->request()->getPaddr(), (int)*(inst->memData), 787 inst->seqNum); 788 789 // @todo: Remove this SC hack once the memory system handles it. 790 if (inst->isStoreConditional()) { 791 // Disable recording the result temporarily. Writing to 792 // misc regs normally updates the result, but this is not 793 // the desired behavior when handling store conditionals. 794 inst->recordResult(false); 795 bool success = TheISA::handleLockedWrite(inst.get(), 796 req->request(), cacheBlockMask); 797 inst->recordResult(true); 798 req->packetSent(); 799 800 if (!success) { 801 req->complete(); 802 // Instantly complete this store. 803 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 804 "Instantly completing it.\n", 805 inst->seqNum); 806 PacketPtr new_pkt = new Packet(*req->packet()); 807 WritebackEvent *wb = new WritebackEvent(inst, 808 new_pkt, this); 809 cpu->schedule(wb, curTick() + 1); 810 completeStore(storeWBIt); 811 if (!storeQueue.empty()) 812 storeWBIt++; 813 else 814 storeWBIt = storeQueue.end(); 815 continue; 816 } 817 } 818 819 if (req->request()->isMmappedIpr()) { 820 assert(!inst->isStoreConditional()); 821 ThreadContext *thread = cpu->tcBase(lsqID); 822 PacketPtr main_pkt = new Packet(req->mainRequest(), 823 MemCmd::WriteReq); 824 main_pkt->dataStatic(inst->memData); 825 req->handleIprWrite(thread, main_pkt); 826 delete main_pkt; 827 completeStore(storeWBIt); 828 storeWBIt++; 829 continue; 830 } 831 /* Send to cache */ 832 req->sendPacketToCache(); 833 834 /* If successful, do the post send */ 835 if (req->isSent()) { 836 storePostSend(); 837 } else { 838 DPRINTF(LSQUnit, "D-Cache became blocked when writing [sn:%lli], " 839 "will retry later\n", 840 inst->seqNum); 841 } 842 } 843 assert(stores >= 0 && storesToWB >= 0); 844} 845 846template <class Impl> 847void 848LSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 849{ 850 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 851 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 852 853 while (loads != 0 && 854 loadQueue.back().instruction()->seqNum > squashed_num) { 855 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 856 "[sn:%lli]\n", 857 loadQueue.back().instruction()->pcState(), 858 loadQueue.back().instruction()->seqNum); 859 860 if (isStalled() && loadQueue.tail() == stallingLoadIdx) { 861 stalled = false; 862 stallingStoreIsn = 0; 863 stallingLoadIdx = 0; 864 } 865 866 // Clear the smart pointer to make sure it is decremented. 867 loadQueue.back().instruction()->setSquashed(); 868 loadQueue.back().clear(); 869 870 --loads; 871 872 loadQueue.pop_back(); 873 ++lsqSquashedLoads; 874 } 875 876 if (memDepViolator && squashed_num < memDepViolator->seqNum) { 877 memDepViolator = NULL; 878 } 879 880 while (stores != 0 && 881 storeQueue.back().instruction()->seqNum > squashed_num) { 882 // Instructions marked as can WB are already committed. 883 if (storeQueue.back().canWB()) { 884 break; 885 } 886 887 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 888 "idx:%i [sn:%lli]\n", 889 storeQueue.back().instruction()->pcState(), 890 storeQueue.tail(), storeQueue.back().instruction()->seqNum); 891 892 // I don't think this can happen. It should have been cleared 893 // by the stalling load. 894 if (isStalled() && 895 storeQueue.back().instruction()->seqNum == stallingStoreIsn) { 896 panic("Is stalled should have been cleared by stalling load!\n"); 897 stalled = false; 898 stallingStoreIsn = 0; 899 } 900 901 // Clear the smart pointer to make sure it is decremented. 902 storeQueue.back().instruction()->setSquashed(); 903 904 // Must delete request now that it wasn't handed off to 905 // memory. This is quite ugly. @todo: Figure out the proper 906 // place to really handle request deletes. 907 storeQueue.back().clear(); 908 --stores; 909 910 storeQueue.pop_back(); 911 ++lsqSquashedStores; 912 } 913} 914 915template <class Impl> 916void 917LSQUnit<Impl>::storePostSend() 918{ 919 if (isStalled() && 920 storeWBIt->instruction()->seqNum == stallingStoreIsn) { 921 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 922 "load idx:%i\n", 923 stallingStoreIsn, stallingLoadIdx); 924 stalled = false; 925 stallingStoreIsn = 0; 926 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction()); 927 } 928 929 if (!storeWBIt->instruction()->isStoreConditional()) { 930 // The store is basically completed at this time. This 931 // only works so long as the checker doesn't try to 932 // verify the value in memory for stores. 933 storeWBIt->instruction()->setCompleted(); 934 935 if (cpu->checker) { 936 cpu->checker->verify(storeWBIt->instruction()); 937 } 938 } 939 940 if (needsTSO) { 941 storeInFlight = true; 942 } 943 944 storeWBIt++; 945} 946 947template <class Impl> 948void 949LSQUnit<Impl>::writeback(const DynInstPtr &inst, PacketPtr pkt) 950{ 951 iewStage->wakeCPU(); 952 953 // Squashed instructions do not need to complete their access. 954 if (inst->isSquashed()) { 955 assert(!inst->isStore()); 956 ++lsqIgnoredResponses; 957 return; 958 } 959 960 if (!inst->isExecuted()) { 961 inst->setExecuted(); 962 963 if (inst->fault == NoFault) { 964 // Complete access to copy data to proper place. 965 inst->completeAcc(pkt); 966 } else { 967 // If the instruction has an outstanding fault, we cannot complete 968 // the access as this discards the current fault. 969 970 // If we have an outstanding fault, the fault should only be of 971 // type ReExec or - in case of a SplitRequest - a partial 972 // translation fault 973 assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr || 974 inst->savedReq->isPartialFault()); 975 976 DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access " 977 "due to pending fault.\n", inst->seqNum); 978 } 979 } 980 981 // Need to insert instruction into queue to commit 982 iewStage->instToCommit(inst); 983 984 iewStage->activityThisCycle(); 985 986 // see if this load changed the PC 987 iewStage->checkMisprediction(inst); 988} 989 990template <class Impl> 991void 992LSQUnit<Impl>::completeStore(typename StoreQueue::iterator store_idx) 993{ 994 assert(store_idx->valid()); 995 store_idx->completed() = true; 996 --storesToWB; 997 // A bit conservative because a store completion may not free up entries, 998 // but hopefully avoids two store completions in one cycle from making 999 // the CPU tick twice. 1000 cpu->wakeCPU(); 1001 cpu->activityThisCycle(); 1002 1003 /* We 'need' a copy here because we may clear the entry from the 1004 * store queue. */ 1005 DynInstPtr store_inst = store_idx->instruction(); 1006 if (store_idx == storeQueue.begin()) { 1007 do { 1008 storeQueue.front().clear(); 1009 storeQueue.pop_front(); 1010 --stores; 1011 } while (storeQueue.front().completed() && 1012 !storeQueue.empty()); 1013 1014 iewStage->updateLSQNextCycle = true; 1015 } 1016 1017 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 1018 "idx:%i\n", 1019 store_inst->seqNum, store_idx.idx() - 1, storeQueue.head() - 1); 1020 1021#if TRACING_ON 1022 if (DTRACE(O3PipeView)) { 1023 store_inst->storeTick = 1024 curTick() - store_inst->fetchTick; 1025 } 1026#endif 1027 1028 if (isStalled() && 1029 store_inst->seqNum == stallingStoreIsn) { 1030 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1031 "load idx:%i\n", 1032 stallingStoreIsn, stallingLoadIdx); 1033 stalled = false; 1034 stallingStoreIsn = 0; 1035 iewStage->replayMemInst(loadQueue[stallingLoadIdx].instruction()); 1036 } 1037 1038 store_inst->setCompleted(); 1039 1040 if (needsTSO) { 1041 storeInFlight = false; 1042 } 1043 1044 // Tell the checker we've completed this instruction. Some stores 1045 // may get reported twice to the checker, but the checker can 1046 // handle that case. 1047 // Store conditionals cannot be sent to the checker yet, they have 1048 // to update the misc registers first which should take place 1049 // when they commit 1050 if (cpu->checker && !store_inst->isStoreConditional()) { 1051 cpu->checker->verify(store_inst); 1052 } 1053} 1054 1055template <class Impl> 1056bool 1057LSQUnit<Impl>::trySendPacket(bool isLoad, PacketPtr data_pkt) 1058{ 1059 bool ret = true; 1060 bool cache_got_blocked = false; 1061 1062 auto state = dynamic_cast<LSQSenderState*>(data_pkt->senderState); 1063 1064 if (!lsq->cacheBlocked() && 1065 lsq->cachePortAvailable(isLoad)) { 1066 if (!dcachePort->sendTimingReq(data_pkt)) { 1067 ret = false; 1068 cache_got_blocked = true; 1069 } 1070 } else { 1071 ret = false; 1072 } 1073 1074 if (ret) { 1075 if (!isLoad) { 1076 isStoreBlocked = false; 1077 } 1078 lsq->cachePortBusy(isLoad); 1079 state->outstanding++; 1080 state->request()->packetSent(); 1081 } else { 1082 if (cache_got_blocked) { 1083 lsq->cacheBlocked(true); 1084 ++lsqCacheBlocked; 1085 } 1086 if (!isLoad) { 1087 assert(state->request() == storeWBIt->request()); 1088 isStoreBlocked = true; 1089 } 1090 state->request()->packetNotSent(); 1091 } 1092 return ret; 1093} 1094 1095template <class Impl> 1096void 1097LSQUnit<Impl>::recvRetry() 1098{ 1099 if (isStoreBlocked) { 1100 DPRINTF(LSQUnit, "Receiving retry: blocked store\n"); 1101 writebackBlockedStore(); 1102 } 1103} 1104 1105template <class Impl> 1106void 1107LSQUnit<Impl>::dumpInsts() const 1108{ 1109 cprintf("Load store queue: Dumping instructions.\n"); 1110 cprintf("Load queue size: %i\n", loads); 1111 cprintf("Load queue: "); 1112 1113 for (const auto& e: loadQueue) { 1114 const DynInstPtr &inst(e.instruction()); 1115 cprintf("%s.[sn:%llu] ", inst->pcState(), inst->seqNum); 1116 } 1117 cprintf("\n"); 1118 1119 cprintf("Store queue size: %i\n", stores); 1120 cprintf("Store queue: "); 1121 1122 for (const auto& e: storeQueue) { 1123 const DynInstPtr &inst(e.instruction()); 1124 cprintf("%s.[sn:%llu] ", inst->pcState(), inst->seqNum); 1125 } 1126 1127 cprintf("\n"); 1128} 1129 1130template <class Impl> 1131unsigned int 1132LSQUnit<Impl>::cacheLineSize() 1133{ 1134 return cpu->cacheLineSize(); 1135} 1136 1137#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 1138