lsq_unit_impl.hh revision 10573:3b405d11d6dc
11689SN/A 22329SN/A/* 31689SN/A * Copyright (c) 2010-2014 ARM Limited 41689SN/A * Copyright (c) 2013 Advanced Micro Devices, Inc. 51689SN/A * All rights reserved 61689SN/A * 71689SN/A * The license below extends only to copyright in the software and shall 81689SN/A * not be construed as granting a license to any other intellectual 91689SN/A * property including but not limited to intellectual property relating 101689SN/A * to a hardware implementation of the functionality of the software 111689SN/A * licensed hereunder. You may use the software subject to the license 121689SN/A * terms below provided that you ensure that this notice is replicated 131689SN/A * unmodified and in its entirety in all distributions of the software, 141689SN/A * modified or unmodified, in source code or in binary form. 151689SN/A * 161689SN/A * Copyright (c) 2004-2005 The Regents of The University of Michigan 171689SN/A * All rights reserved. 181689SN/A * 191689SN/A * Redistribution and use in source and binary forms, with or without 201689SN/A * modification, are permitted provided that the following conditions are 211689SN/A * met: redistributions of source code must retain the above copyright 221689SN/A * notice, this list of conditions and the following disclaimer; 231689SN/A * redistributions in binary form must reproduce the above copyright 241689SN/A * notice, this list of conditions and the following disclaimer in the 251689SN/A * documentation and/or other materials provided with the distribution; 261689SN/A * neither the name of the copyright holders nor the names of its 272665Ssaidi@eecs.umich.edu * contributors may be used to endorse or promote products derived from 282665Ssaidi@eecs.umich.edu * this software without specific prior written permission. 292935Sksewell@umich.edu * 301689SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 311689SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 321060SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 331060SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 343773Sgblack@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 353773Sgblack@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 361858SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 371717SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 381060SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 391061SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 402292SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 412292SN/A * 422292SN/A * Authors: Kevin Lim 432292SN/A * Korey Sewell 442292SN/A */ 452292SN/A 463788Sgblack@eecs.umich.edu#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 473798Sgblack@eecs.umich.edu#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 482361SN/A 492361SN/A#include "arch/generic/debugfaults.hh" 501060SN/A#include "arch/locked_mem.hh" 512292SN/A#include "base/str.hh" 522292SN/A#include "config/the_isa.hh" 532292SN/A#include "cpu/checker/cpu.hh" 542292SN/A#include "cpu/o3/lsq.hh" 552292SN/A#include "cpu/o3/lsq_unit.hh" 562292SN/A#include "debug/Activity.hh" 572292SN/A#include "debug/IEW.hh" 582292SN/A#include "debug/LSQUnit.hh" 592292SN/A#include "debug/O3PipeView.hh" 602292SN/A#include "mem/packet.hh" 612292SN/A#include "mem/request.hh" 622301SN/A 632292SN/Atemplate<class Impl> 642292SN/ALSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt, 652292SN/A LSQUnit *lsq_ptr) 662292SN/A : Event(Default_Pri, AutoDelete), 672292SN/A inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 682292SN/A{ 692292SN/A} 702292SN/A 712292SN/Atemplate<class Impl> 722292SN/Avoid 732292SN/ALSQUnit<Impl>::WritebackEvent::process() 742292SN/A{ 752292SN/A assert(!lsqPtr->cpu->switchedOut()); 762292SN/A 772292SN/A lsqPtr->writeback(inst, pkt); 782292SN/A 792292SN/A if (pkt->senderState) 801060SN/A delete pkt->senderState; 811060SN/A 821061SN/A delete pkt->req; 831060SN/A delete pkt; 842292SN/A} 851062SN/A 861062SN/Atemplate<class Impl> 872301SN/Aconst char * 881062SN/ALSQUnit<Impl>::WritebackEvent::description() const 891062SN/A{ 901062SN/A return "Store writeback"; 912301SN/A} 921062SN/A 931062SN/Atemplate<class Impl> 941062SN/Avoid 952301SN/ALSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 961062SN/A{ 971062SN/A LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 982301SN/A DynInstPtr inst = state->inst; 992301SN/A DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum); 1002301SN/A DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum); 1012301SN/A 1022292SN/A if (state->cacheBlocked) { 1032301SN/A // This is the first half of a previous split load, 1042292SN/A // where the 2nd half blocked, ignore this response 1052292SN/A DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier " 1061062SN/A "blocked split load recieved. Ignoring.\n", inst->seqNum); 1072301SN/A delete state; 1081062SN/A return; 1091062SN/A } 1101062SN/A 1112301SN/A // If this is a split access, wait until all packets are received. 1121062SN/A if (TheISA::HasUnalignedMemAcc && !state->complete()) { 1131062SN/A return; 1141062SN/A } 1152301SN/A 1161062SN/A assert(!cpu->switchedOut()); 1171062SN/A if (!inst->isSquashed()) { 1181062SN/A if (!state->noWB) { 1192301SN/A if (!TheISA::HasUnalignedMemAcc || !state->isSplit || 1202292SN/A !state->isLoad) { 1211062SN/A writeback(inst, pkt); 1221062SN/A } else { 1232301SN/A writeback(inst, state->mainPkt); 1242292SN/A } 1251062SN/A } 1262292SN/A 1272301SN/A if (inst->isStore()) { 1282292SN/A completeStore(state->idx); 1292292SN/A } 1301062SN/A } 1312301SN/A 1321062SN/A if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) { 1331062SN/A delete state->mainPkt->req; 1341062SN/A delete state->mainPkt; 1352301SN/A } 1361062SN/A 1371062SN/A pkt->req->setAccessLatency(); 1381062SN/A cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1392301SN/A 1401062SN/A delete state; 1411062SN/A} 1421062SN/A 1432301SN/Atemplate <class Impl> 1441062SN/ALSQUnit<Impl>::LSQUnit() 1451062SN/A : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 1461062SN/A isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false) 1472301SN/A{ 1481062SN/A} 1491062SN/A 1502301SN/Atemplate<class Impl> 1512301SN/Avoid 1522301SN/ALSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 1532301SN/A LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 1542301SN/A unsigned id) 1552301SN/A{ 1562301SN/A cpu = cpu_ptr; 1572301SN/A iewStage = iew_ptr; 1582301SN/A 1592301SN/A lsq = lsq_ptr; 1602307SN/A 1612307SN/A lsqID = id; 1622307SN/A 1632307SN/A DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id); 1642307SN/A 1651062SN/A // Add 1 for the sentinel entry (they are circular queues). 1661062SN/A LQEntries = maxLQEntries + 1; 1671062SN/A SQEntries = maxSQEntries + 1; 1681062SN/A 1692733Sktlim@umich.edu //Due to uint8_t index in LSQSenderState 1701060SN/A assert(LQEntries <= 256); 1712292SN/A assert(SQEntries <= 256); 1721060SN/A 1731060SN/A loadQueue.resize(LQEntries); 1741060SN/A storeQueue.resize(SQEntries); 1751061SN/A 1761060SN/A depCheckShift = params->LSQDepCheckShift; 1772292SN/A checkLoads = params->LSQCheckLoads; 1781060SN/A cachePorts = params->cachePorts; 1792292SN/A needsTSO = params->needsTSO; 1801060SN/A 1811060SN/A resetState(); 1821060SN/A} 1831060SN/A 1841060SN/A 1851060SN/Atemplate<class Impl> 1861060SN/Avoid 1871060SN/ALSQUnit<Impl>::resetState() 1881060SN/A{ 1891060SN/A loads = stores = storesToWB = 0; 1901060SN/A 1911060SN/A loadHead = loadTail = 0; 1921061SN/A 1931060SN/A storeHead = storeWBIdx = storeTail = 0; 1942292SN/A 1951060SN/A usedPorts = 0; 1962292SN/A 1971060SN/A retryPkt = NULL; 1981060SN/A memDepViolator = NULL; 1991060SN/A 2001060SN/A stalled = false; 2011060SN/A 2021060SN/A cacheBlockMask = ~(cpu->cacheLineSize() - 1); 2031061SN/A} 2041060SN/A 2052292SN/Atemplate<class Impl> 2061060SN/Astd::string 2072292SN/ALSQUnit<Impl>::name() const 2081060SN/A{ 2091060SN/A if (Impl::MaxThreads == 1) { 2101060SN/A return iewStage->name() + ".lsq"; 2111060SN/A } else { 2121060SN/A return iewStage->name() + ".lsq.thread" + std::to_string(lsqID); 2131060SN/A } 2141061SN/A} 2151060SN/A 2162292SN/Atemplate<class Impl> 2171060SN/Avoid 2182329SN/ALSQUnit<Impl>::regStats() 2192292SN/A{ 2202292SN/A lsqForwLoads 2212292SN/A .name(name() + ".forwLoads") 2222292SN/A .desc("Number of loads that had data forwarded from stores"); 2232292SN/A 2242292SN/A invAddrLoads 2251060SN/A .name(name() + ".invAddrLoads") 2261060SN/A .desc("Number of loads ignored due to an invalid address"); 2272292SN/A 2282292SN/A lsqSquashedLoads 2292980Sgblack@eecs.umich.edu .name(name() + ".squashedLoads") 2302292SN/A .desc("Number of loads squashed"); 2312292SN/A 2322292SN/A lsqIgnoredResponses 2332292SN/A .name(name() + ".ignoredResponses") 2342292SN/A .desc("Number of memory responses ignored because the instruction is squashed"); 2352292SN/A 2361061SN/A lsqMemOrderViolation 2371060SN/A .name(name() + ".memOrderViolation") 2382292SN/A .desc("Number of memory ordering violations"); 2391060SN/A 2402292SN/A lsqSquashedStores 2411060SN/A .name(name() + ".squashedStores") 2422292SN/A .desc("Number of stores squashed"); 2432292SN/A 2441060SN/A invAddrSwpfs 2451060SN/A .name(name() + ".invAddrSwpfs") 2461060SN/A .desc("Number of software prefetches ignored due to an invalid address"); 2471061SN/A 2481060SN/A lsqBlockedLoads 2492292SN/A .name(name() + ".blockedLoads") 2501060SN/A .desc("Number of blocked loads due to partial load-store forwarding"); 2512292SN/A 2522292SN/A lsqRescheduledLoads 2532292SN/A .name(name() + ".rescheduledLoads") 2541060SN/A .desc("Number of loads that were rescheduled"); 2552292SN/A 2562292SN/A lsqCacheBlocked 2572292SN/A .name(name() + ".cacheBlocked") 2582292SN/A .desc("Number of times an access to memory failed due to the cache being blocked"); 2592292SN/A} 2602292SN/A 2611060SN/Atemplate<class Impl> 2621060SN/Avoid 2631061SN/ALSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2642863Sktlim@umich.edu{ 2652843Sktlim@umich.edu dcachePort = dcache_port; 2661060SN/A} 2672348SN/A 2682843Sktlim@umich.edutemplate<class Impl> 2692863Sktlim@umich.eduvoid 2702316SN/ALSQUnit<Impl>::clearLQ() 2711060SN/A{ 2722316SN/A loadQueue.clear(); 2732316SN/A} 2742843Sktlim@umich.edu 2752316SN/Atemplate<class Impl> 2762348SN/Avoid 2772307SN/ALSQUnit<Impl>::clearSQ() 2782980Sgblack@eecs.umich.edu{ 2792980Sgblack@eecs.umich.edu storeQueue.clear(); 2802307SN/A} 2812307SN/A 2822307SN/Atemplate<class Impl> 2832307SN/Avoid 2842307SN/ALSQUnit<Impl>::drainSanityCheck() const 2852307SN/A{ 2862307SN/A for (int i = 0; i < loadQueue.size(); ++i) 2872307SN/A assert(!loadQueue[i]); 2882307SN/A 2892307SN/A assert(storesToWB == 0); 2902307SN/A assert(!retryPkt); 2912307SN/A} 2922307SN/A 2932307SN/Atemplate<class Impl> 2942361SN/Avoid 2952361SN/ALSQUnit<Impl>::takeOverFrom() 2962361SN/A{ 2972361SN/A resetState(); 2982361SN/A} 2992307SN/A 3002307SN/Atemplate<class Impl> 3012307SN/Avoid 3022307SN/ALSQUnit<Impl>::resizeLQ(unsigned size) 3031060SN/A{ 3041060SN/A unsigned size_plus_sentinel = size + 1; 3051060SN/A assert(size_plus_sentinel >= LQEntries); 3061061SN/A 3071060SN/A if (size_plus_sentinel > LQEntries) { 3082307SN/A while (size_plus_sentinel > loadQueue.size()) { 3091060SN/A DynInstPtr dummy; 3102307SN/A loadQueue.push_back(dummy); 3112307SN/A LQEntries++; 3121060SN/A } 3132329SN/A } else { 3142307SN/A LQEntries = size_plus_sentinel; 3152307SN/A } 3161060SN/A 3172307SN/A assert(LQEntries <= 256); 3182307SN/A} 3192307SN/A 3202307SN/Atemplate<class Impl> 3212307SN/Avoid 3222307SN/ALSQUnit<Impl>::resizeSQ(unsigned size) 3232307SN/A{ 3242307SN/A unsigned size_plus_sentinel = size + 1; 3252307SN/A if (size_plus_sentinel > SQEntries) { 3262307SN/A while (size_plus_sentinel > storeQueue.size()) { 3272307SN/A SQEntry dummy; 3282307SN/A storeQueue.push_back(dummy); 3292307SN/A SQEntries++; 3302307SN/A } 3312935Sksewell@umich.edu } else { 3321858SN/A SQEntries = size_plus_sentinel; 3332292SN/A } 3341858SN/A 3352292SN/A assert(SQEntries <= 256); 3362292SN/A} 3372292SN/A 3382292SN/Atemplate <class Impl> 3393788Sgblack@eecs.umich.eduvoid 3402292SN/ALSQUnit<Impl>::insert(DynInstPtr &inst) 3412698Sktlim@umich.edu{ 3423788Sgblack@eecs.umich.edu assert(inst->isMemRef()); 3432301SN/A 3443788Sgblack@eecs.umich.edu assert(inst->isLoad() || inst->isStore()); 3453788Sgblack@eecs.umich.edu 3463788Sgblack@eecs.umich.edu if (inst->isLoad()) { 3473788Sgblack@eecs.umich.edu insertLoad(inst); 3483788Sgblack@eecs.umich.edu } else { 3493788Sgblack@eecs.umich.edu insertStore(inst); 3503788Sgblack@eecs.umich.edu } 3513788Sgblack@eecs.umich.edu 3523788Sgblack@eecs.umich.edu inst->setInLSQ(); 3533788Sgblack@eecs.umich.edu} 3543788Sgblack@eecs.umich.edu 3552292SN/Atemplate <class Impl> 3562292SN/Avoid 3572292SN/ALSQUnit<Impl>::insertLoad(DynInstPtr &load_inst) 3582292SN/A{ 3592292SN/A assert((loadTail + 1) % LQEntries != loadHead); 3602329SN/A assert(loads < LQEntries); 3612292SN/A 3622292SN/A DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 3632292SN/A load_inst->pcState(), loadTail, load_inst->seqNum); 3642935Sksewell@umich.edu 3652935Sksewell@umich.edu load_inst->lqIdx = loadTail; 3662731Sktlim@umich.edu 3672292SN/A if (stores == 0) { 3682292SN/A load_inst->sqIdx = -1; 3692292SN/A } else { 3702935Sksewell@umich.edu load_inst->sqIdx = storeTail; 3712292SN/A } 3722292SN/A 3732935Sksewell@umich.edu loadQueue[loadTail] = load_inst; 3742935Sksewell@umich.edu 3752935Sksewell@umich.edu incrLdIdx(loadTail); 3762935Sksewell@umich.edu 3772935Sksewell@umich.edu ++loads; 3783093Sksewell@umich.edu} 3792935Sksewell@umich.edu 3802935Sksewell@umich.edutemplate <class Impl> 3812935Sksewell@umich.eduvoid 3822935Sksewell@umich.eduLSQUnit<Impl>::insertStore(DynInstPtr &store_inst) 3832935Sksewell@umich.edu{ 3842935Sksewell@umich.edu // Make sure it is not full before inserting an instruction. 3852935Sksewell@umich.edu assert((storeTail + 1) % SQEntries != storeHead); 3862935Sksewell@umich.edu assert(stores < SQEntries); 3872935Sksewell@umich.edu 3882935Sksewell@umich.edu DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 3892935Sksewell@umich.edu store_inst->pcState(), storeTail, store_inst->seqNum); 3903093Sksewell@umich.edu 3913093Sksewell@umich.edu store_inst->sqIdx = storeTail; 3922935Sksewell@umich.edu store_inst->lqIdx = loadTail; 3932292SN/A 3942292SN/A storeQueue[storeTail] = SQEntry(store_inst); 3952935Sksewell@umich.edu 3962935Sksewell@umich.edu incrStIdx(storeTail); 3973093Sksewell@umich.edu 3982935Sksewell@umich.edu ++stores; 3992935Sksewell@umich.edu} 4002935Sksewell@umich.edu 4012935Sksewell@umich.edutemplate <class Impl> 4022935Sksewell@umich.edutypename Impl::DynInstPtr 4032935Sksewell@umich.eduLSQUnit<Impl>::getMemDepViolator() 4042935Sksewell@umich.edu{ 4052935Sksewell@umich.edu DynInstPtr temp = memDepViolator; 4062935Sksewell@umich.edu 4072935Sksewell@umich.edu memDepViolator = NULL; 4082935Sksewell@umich.edu 4093798Sgblack@eecs.umich.edu return temp; 4103798Sgblack@eecs.umich.edu} 4113798Sgblack@eecs.umich.edu 4123093Sksewell@umich.edutemplate <class Impl> 4133093Sksewell@umich.eduunsigned 4142935Sksewell@umich.eduLSQUnit<Impl>::numFreeLoadEntries() 4152935Sksewell@umich.edu{ 4162292SN/A //LQ has an extra dummy entry to differentiate 4172292SN/A //empty/full conditions. Subtract 1 from the free entries. 4182292SN/A DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads); 4192292SN/A return LQEntries - loads - 1; 4202292SN/A} 4212292SN/A 4222292SN/Atemplate <class Impl> 4232292SN/Aunsigned 4242292SN/ALSQUnit<Impl>::numFreeStoreEntries() 4252292SN/A{ 4262292SN/A //SQ has an extra dummy entry to differentiate 4272292SN/A //empty/full conditions. Subtract 1 from the free entries. 4282292SN/A DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores); 4292292SN/A return SQEntries - stores - 1; 4302292SN/A 4312292SN/A } 4322980Sgblack@eecs.umich.edu 4332292SN/Atemplate <class Impl> 4342292SN/Avoid 4352292SN/ALSQUnit<Impl>::checkSnoop(PacketPtr pkt) 4362292SN/A{ 4372292SN/A int load_idx = loadHead; 4382292SN/A DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 4392292SN/A 4402292SN/A // Unlock the cpu-local monitor when the CPU sees a snoop to a locked 4412292SN/A // address. The CPU can speculatively execute a LL operation after a pending 4422292SN/A // SC operation in the pipeline and that can make the cache monitor the CPU 4432292SN/A // is connected to valid while it really shouldn't be. 4442292SN/A for (int x = 0; x < cpu->numContexts(); x++) { 4452292SN/A ThreadContext *tc = cpu->getContext(x); 4462292SN/A bool no_squash = cpu->thread[x]->noSquashFromTC; 4472292SN/A cpu->thread[x]->noSquashFromTC = true; 4482292SN/A TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 4492292SN/A cpu->thread[x]->noSquashFromTC = no_squash; 4502292SN/A } 4512292SN/A 4522292SN/A Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 4532292SN/A 4542292SN/A DynInstPtr ld_inst = loadQueue[load_idx]; 4552292SN/A if (ld_inst) { 4562292SN/A Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 4572292SN/A // Check that this snoop didn't just invalidate our lock flag 4582292SN/A if (ld_inst->effAddrValid() && load_addr == invalidate_addr && 4592292SN/A ld_inst->memReqFlags & Request::LLSC) 4602292SN/A TheISA::handleLockedSnoopHit(ld_inst.get()); 4612292SN/A } 4622292SN/A 4632292SN/A // If this is the only load in the LSQ we don't care 4642292SN/A if (load_idx == loadTail) 4652292SN/A return; 4662292SN/A 4672292SN/A incrLdIdx(load_idx); 4682292SN/A 4692292SN/A bool force_squash = false; 4702292SN/A 4712292SN/A while (load_idx != loadTail) { 4722292SN/A DynInstPtr ld_inst = loadQueue[load_idx]; 4732292SN/A 4742292SN/A if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 4752292SN/A incrLdIdx(load_idx); 4762292SN/A continue; 4772292SN/A } 4782292SN/A 4792292SN/A Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 4802292SN/A DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n", 4812292SN/A ld_inst->seqNum, load_addr, invalidate_addr); 4822292SN/A 4832292SN/A if (load_addr == invalidate_addr || force_squash) { 4842292SN/A if (needsTSO) { 4852292SN/A // If we have a TSO system, as all loads must be ordered with 4862292SN/A // all other loads, this load as well as *all* subsequent loads 4872292SN/A // need to be squashed to prevent possible load reordering. 4882292SN/A force_squash = true; 4892292SN/A } 4902292SN/A if (ld_inst->possibleLoadViolation() || force_squash) { 4912292SN/A DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 4922292SN/A pkt->getAddr(), ld_inst->seqNum); 4932301SN/A 4942301SN/A // Mark the load for re-execution 4953788Sgblack@eecs.umich.edu ld_inst->fault = std::make_shared<ReExec>(); 4963788Sgblack@eecs.umich.edu } else { 4973788Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 4983788Sgblack@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 4993788Sgblack@eecs.umich.edu 5003788Sgblack@eecs.umich.edu // Make sure that we don't lose a snoop hitting a LOCKED 5013788Sgblack@eecs.umich.edu // address since the LOCK* flags don't get updated until 5023788Sgblack@eecs.umich.edu // commit. 5033798Sgblack@eecs.umich.edu if (ld_inst->memReqFlags & Request::LLSC) 5043798Sgblack@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 5053798Sgblack@eecs.umich.edu 5063798Sgblack@eecs.umich.edu // If a older load checks this and it's true 5073798Sgblack@eecs.umich.edu // then we might have missed the snoop 5083798Sgblack@eecs.umich.edu // in which case we need to invalidate to be sure 5092292SN/A ld_inst->hitExternalSnoop(true); 5102292SN/A } 5112292SN/A } 5122292SN/A incrLdIdx(load_idx); 5132292SN/A } 5142292SN/A return; 5152292SN/A} 5162292SN/A 5172292SN/Atemplate <class Impl> 5182292SN/AFault 5192292SN/ALSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst) 5202292SN/A{ 5212292SN/A Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 5222292SN/A Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 5232292SN/A 5242292SN/A /** @todo in theory you only need to check an instruction that has executed 5252292SN/A * however, there isn't a good way in the pipeline at the moment to check 5262292SN/A * all instructions that will execute before the store writes back. Thus, 5272292SN/A * like the implementation that came before it, we're overly conservative. 5282292SN/A */ 5291858SN/A while (load_idx != loadTail) { 5301858SN/A DynInstPtr ld_inst = loadQueue[load_idx]; 5311858SN/A if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 5321858SN/A incrLdIdx(load_idx); 5331858SN/A continue; 5342292SN/A } 5351858SN/A 5362292SN/A Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 5372292SN/A Addr ld_eff_addr2 = 5382292SN/A (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 5392292SN/A 5401858SN/A if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 5412292SN/A if (inst->isLoad()) { 5422292SN/A // If this load is to the same block as an external snoop 5432292SN/A // invalidate that we've observed then the load needs to be 5442292SN/A // squashed as it could have newer data 5452292SN/A if (ld_inst->hitExternalSnoop()) { 5462292SN/A if (!memDepViolator || 5472292SN/A ld_inst->seqNum < memDepViolator->seqNum) { 5482292SN/A DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 5492292SN/A "and [sn:%lli] at address %#x\n", 5502292SN/A inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5512292SN/A memDepViolator = ld_inst; 5522292SN/A 5532292SN/A ++lsqMemOrderViolation; 5541858SN/A 5552292SN/A return std::make_shared<GenericISA::M5PanicFault>( 5562292SN/A "Detected fault with inst [sn:%lli] and " 5572292SN/A "[sn:%lli] at address %#x\n", 5582292SN/A inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5592292SN/A } 5602292SN/A } 5612292SN/A 5622292SN/A // Otherwise, mark the load has a possible load violation 5632292SN/A // and if we see a snoop before it's commited, we need to squash 5642292SN/A ld_inst->possibleLoadViolation(true); 5652292SN/A DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x" 5662292SN/A " between instructions [sn:%lli] and [sn:%lli]\n", 5672292SN/A inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 5682292SN/A } else { 5692292SN/A // A load/store incorrectly passed this store. 5702292SN/A // Check if we already have a violator, or if it's newer 5712292SN/A // squash and refetch. 5722292SN/A if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 5732292SN/A break; 5742292SN/A 5752292SN/A DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 5762292SN/A "[sn:%lli] at address %#x\n", 5772292SN/A inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5782292SN/A memDepViolator = ld_inst; 5792292SN/A 5802292SN/A ++lsqMemOrderViolation; 5812292SN/A 5822292SN/A return std::make_shared<GenericISA::M5PanicFault>( 5832292SN/A "Detected fault with " 5842292SN/A "inst [sn:%lli] and [sn:%lli] at address %#x\n", 5852292SN/A inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5862292SN/A } 5872292SN/A } 5882292SN/A 5892292SN/A incrLdIdx(load_idx); 5902292SN/A } 5912292SN/A return NoFault; 5922292SN/A} 5932292SN/A 5942292SN/A 5952292SN/A 5962292SN/A 5972292SN/Atemplate <class Impl> 5982292SN/AFault 5992292SN/ALSQUnit<Impl>::executeLoad(DynInstPtr &inst) 6002292SN/A{ 6012292SN/A using namespace TheISA; 6022292SN/A // Execute a specific load. 6032292SN/A Fault load_fault = NoFault; 6042292SN/A 6052292SN/A DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 6062292SN/A inst->pcState(), inst->seqNum); 6072292SN/A 6082292SN/A assert(!inst->isSquashed()); 6092292SN/A 6102292SN/A load_fault = inst->initiateAcc(); 6112292SN/A 6122292SN/A if (inst->isTranslationDelayed() && 6132292SN/A load_fault == NoFault) 6142292SN/A return load_fault; 6152292SN/A 6162292SN/A // If the instruction faulted or predicated false, then we need to send it 6172292SN/A // along to commit without the instruction completing. 6182292SN/A if (load_fault != NoFault || !inst->readPredicate()) { 6192292SN/A // Send this instruction to commit, also make sure iew stage 6202292SN/A // realizes there is activity. 6212292SN/A // Mark it as executed unless it is an uncached load that 6222292SN/A // needs to hit the head of commit. 6232292SN/A if (!inst->readPredicate()) 6242292SN/A inst->forwardOldRegs(); 6252292SN/A DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 6262292SN/A inst->seqNum, 6272292SN/A (load_fault != NoFault ? "fault" : "predication")); 6282292SN/A if (!(inst->hasRequest() && inst->uncacheable()) || 6292292SN/A inst->isAtCommit()) { 6302292SN/A inst->setExecuted(); 6312292SN/A } 6322292SN/A iewStage->instToCommit(inst); 6332292SN/A iewStage->activityThisCycle(); 6342292SN/A } else { 6352292SN/A assert(inst->effAddrValid()); 6362292SN/A int load_idx = inst->lqIdx; 6372292SN/A incrLdIdx(load_idx); 6382292SN/A 6392292SN/A if (checkLoads) 6402292SN/A return checkViolations(load_idx, inst); 6412292SN/A } 6422292SN/A 6432292SN/A return load_fault; 6442292SN/A} 6452292SN/A 6462292SN/Atemplate <class Impl> 6472292SN/AFault 6482935Sksewell@umich.eduLSQUnit<Impl>::executeStore(DynInstPtr &store_inst) 6492292SN/A{ 6502292SN/A using namespace TheISA; 6512292SN/A // Make sure that a store exists. 6522292SN/A assert(stores != 0); 6532292SN/A 6542292SN/A int store_idx = store_inst->sqIdx; 6552292SN/A 6562292SN/A DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 6572292SN/A store_inst->pcState(), store_inst->seqNum); 6582292SN/A 6592292SN/A assert(!store_inst->isSquashed()); 6602292SN/A 6612292SN/A // Check the recently completed loads to see if any match this store's 6622292SN/A // address. If so, then we have a memory ordering violation. 6632292SN/A int load_idx = store_inst->lqIdx; 6642292SN/A 6652292SN/A Fault store_fault = store_inst->initiateAcc(); 6662336SN/A 6672336SN/A if (store_inst->isTranslationDelayed() && 6682336SN/A store_fault == NoFault) 6692336SN/A return store_fault; 6702336SN/A 6712336SN/A if (!store_inst->readPredicate()) 6722336SN/A store_inst->forwardOldRegs(); 6732336SN/A 6742292SN/A if (storeQueue[store_idx].size == 0) { 6752292SN/A DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 6762301SN/A store_inst->pcState(), store_inst->seqNum); 6772301SN/A 6782292SN/A return store_fault; 6792301SN/A } else if (!store_inst->readPredicate()) { 6802301SN/A DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 6812301SN/A store_inst->seqNum); 6822292SN/A return store_fault; 6832301SN/A } 6842292SN/A 6852301SN/A assert(store_fault == NoFault); 6862292SN/A 6872301SN/A if (store_inst->isStoreConditional()) { 6882292SN/A // Store conditionals need to set themselves as able to 6892292SN/A // writeback if we haven't had a fault by here. 6902292SN/A storeQueue[store_idx].canWB = true; 6912292SN/A 6922336SN/A ++storesToWB; 6932336SN/A } 6942292SN/A 6952292SN/A return checkViolations(load_idx, store_inst); 6962307SN/A 6972307SN/A} 6982292SN/A 6992292SN/Atemplate <class Impl> 7002292SN/Avoid 7012292SN/ALSQUnit<Impl>::commitLoad() 7022292SN/A{ 7032292SN/A assert(loadQueue[loadHead]); 7042292SN/A 7052292SN/A DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 7062292SN/A loadQueue[loadHead]->pcState()); 7072292SN/A 7082292SN/A loadQueue[loadHead] = NULL; 7092292SN/A 7102292SN/A incrLdIdx(loadHead); 7112292SN/A 7122292SN/A --loads; 7132292SN/A} 7142292SN/A 7152292SN/Atemplate <class Impl> 7162292SN/Avoid 7172292SN/ALSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 7182292SN/A{ 7192292SN/A assert(loads == 0 || loadQueue[loadHead]); 7202292SN/A 7212292SN/A while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) { 7222292SN/A commitLoad(); 7232292SN/A } 7242292SN/A} 7252292SN/A 7262292SN/Atemplate <class Impl> 7272292SN/Avoid 7282292SN/ALSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 7292292SN/A{ 7302292SN/A assert(stores == 0 || storeQueue[storeHead].inst); 7312292SN/A 7322292SN/A int store_idx = storeHead; 7332307SN/A 7342292SN/A while (store_idx != storeTail) { 7352292SN/A assert(storeQueue[store_idx].inst); 7362292SN/A // Mark any stores that are now committed and have not yet 7372292SN/A // been marked as able to write back. 7382292SN/A if (!storeQueue[store_idx].canWB) { 7392292SN/A if (storeQueue[store_idx].inst->seqNum > youngest_inst) { 7402292SN/A break; 7412292SN/A } 7422292SN/A DPRINTF(LSQUnit, "Marking store as able to write back, PC " 7432292SN/A "%s [sn:%lli]\n", 7442292SN/A storeQueue[store_idx].inst->pcState(), 7452292SN/A storeQueue[store_idx].inst->seqNum); 7462292SN/A 7472292SN/A storeQueue[store_idx].canWB = true; 7482292SN/A 7492292SN/A ++storesToWB; 7502292SN/A } 7512292SN/A 7522292SN/A incrStIdx(store_idx); 7532292SN/A } 7542292SN/A} 7552292SN/A 7562292SN/Atemplate <class Impl> 7572292SN/Avoid 7582292SN/ALSQUnit<Impl>::writebackPendingStore() 7592292SN/A{ 7602292SN/A if (hasPendingPkt) { 7612292SN/A assert(pendingPkt != NULL); 7622292SN/A 7632292SN/A // If the cache is blocked, this will store the packet for retry. 7642292SN/A if (sendStore(pendingPkt)) { 7652292SN/A storePostSend(pendingPkt); 7662292SN/A } 7672292SN/A pendingPkt = NULL; 7682307SN/A hasPendingPkt = false; 7692307SN/A } 7702292SN/A} 7712292SN/A 7722292SN/Atemplate <class Impl> 7732292SN/Avoid 7743798Sgblack@eecs.umich.eduLSQUnit<Impl>::writebackStores() 7753798Sgblack@eecs.umich.edu{ 7763798Sgblack@eecs.umich.edu // First writeback the second packet from any split store that didn't 7773798Sgblack@eecs.umich.edu // complete last cycle because there weren't enough cache ports available. 7783798Sgblack@eecs.umich.edu if (TheISA::HasUnalignedMemAcc) { 7793798Sgblack@eecs.umich.edu writebackPendingStore(); 7803798Sgblack@eecs.umich.edu } 7813798Sgblack@eecs.umich.edu 7823798Sgblack@eecs.umich.edu while (storesToWB > 0 && 7832292SN/A storeWBIdx != storeTail && 7843798Sgblack@eecs.umich.edu storeQueue[storeWBIdx].inst && 7852292SN/A storeQueue[storeWBIdx].canWB && 7862292SN/A ((!needsTSO) || (!storeInFlight)) && 7872292SN/A usedPorts < cachePorts) { 7882292SN/A 7892292SN/A if (isStoreBlocked) { 7902292SN/A DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 7912292SN/A " is blocked!\n"); 7922329SN/A break; 7933093Sksewell@umich.edu } 7942292SN/A 7952292SN/A // Store didn't write any data so no need to write it back to 7962329SN/A // memory. 7972935Sksewell@umich.edu if (storeQueue[storeWBIdx].size == 0) { 7982292SN/A completeStore(storeWBIdx); 7992292SN/A 8002292SN/A incrStIdx(storeWBIdx); 8012292SN/A 8022292SN/A continue; 8032292SN/A } 8042292SN/A 8052292SN/A ++usedPorts; 8062292SN/A 8072292SN/A if (storeQueue[storeWBIdx].inst->isDataPrefetch()) { 8082980Sgblack@eecs.umich.edu incrStIdx(storeWBIdx); 8092292SN/A 8102292SN/A continue; 8112292SN/A } 8122292SN/A 8132292SN/A assert(storeQueue[storeWBIdx].req); 8142292SN/A assert(!storeQueue[storeWBIdx].committed); 8152292SN/A 8162292SN/A if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) { 8172292SN/A assert(storeQueue[storeWBIdx].sreqLow); 8182292SN/A assert(storeQueue[storeWBIdx].sreqHigh); 8192292SN/A } 8202292SN/A 8212292SN/A DynInstPtr inst = storeQueue[storeWBIdx].inst; 8222292SN/A 8232292SN/A Request *req = storeQueue[storeWBIdx].req; 8242980Sgblack@eecs.umich.edu RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow; 8252292SN/A RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh; 8262292SN/A 8272292SN/A storeQueue[storeWBIdx].committed = true; 8282292SN/A 8292292SN/A assert(!inst->memData); 8302292SN/A inst->memData = new uint8_t[req->getSize()]; 8312292SN/A 8322292SN/A if (storeQueue[storeWBIdx].isAllZeros) 8332292SN/A memset(inst->memData, 0, req->getSize()); 8342292SN/A else 8352292SN/A memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize()); 8362292SN/A 8372292SN/A PacketPtr data_pkt; 8382292SN/A PacketPtr snd_data_pkt = NULL; 8392292SN/A 8402292SN/A LSQSenderState *state = new LSQSenderState; 8412292SN/A state->isLoad = false; 8422292SN/A state->idx = storeWBIdx; 8432292SN/A state->inst = inst; 8442733Sktlim@umich.edu 8452292SN/A if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) { 8462292SN/A 8472292SN/A // Build a single data packet if the store isn't split. 8482292SN/A data_pkt = Packet::createWrite(req); 8492292SN/A data_pkt->dataStatic(inst->memData); 8502292SN/A data_pkt->senderState = state; 8512292SN/A } else { 8522292SN/A // Create two packets if the store is split in two. 8532733Sktlim@umich.edu data_pkt = Packet::createWrite(sreqLow); 8542292SN/A snd_data_pkt = Packet::createWrite(sreqHigh); 8552292SN/A 8562292SN/A data_pkt->dataStatic(inst->memData); 8572292SN/A snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize()); 8582292SN/A 8592292SN/A data_pkt->senderState = state; 8602292SN/A snd_data_pkt->senderState = state; 8612292SN/A 8622292SN/A state->isSplit = true; 8632292SN/A state->outstanding = 2; 8642292SN/A 8652292SN/A // Can delete the main request now. 8662292SN/A delete req; 8672292SN/A req = sreqLow; 8682292SN/A } 8692292SN/A 8702292SN/A DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 8713798Sgblack@eecs.umich.edu "to Addr:%#x, data:%#x [sn:%lli]\n", 8723798Sgblack@eecs.umich.edu storeWBIdx, inst->pcState(), 8733798Sgblack@eecs.umich.edu req->getPaddr(), (int)*(inst->memData), 8743798Sgblack@eecs.umich.edu inst->seqNum); 8752292SN/A 8762292SN/A // @todo: Remove this SC hack once the memory system handles it. 8772292SN/A if (inst->isStoreConditional()) { 8782292SN/A assert(!storeQueue[storeWBIdx].isSplit); 8792292SN/A // Disable recording the result temporarily. Writing to 8802329SN/A // misc regs normally updates the result, but this is not 8812329SN/A // the desired behavior when handling store conditionals. 8822301SN/A inst->recordResult(false); 8832292SN/A bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask); 8842292SN/A inst->recordResult(true); 8852292SN/A 8862292SN/A if (!success) { 8872292SN/A // Instantly complete this store. 8882292SN/A DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 8892292SN/A "Instantly completing it.\n", 8902292SN/A inst->seqNum); 8912292SN/A WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this); 8922292SN/A cpu->schedule(wb, curTick() + 1); 8932292SN/A if (cpu->checker) { 8942292SN/A // Make sure to set the LLSC data for verification 8952292SN/A // if checker is loaded 8962292SN/A inst->reqToVerify->setExtraData(0); 8972292SN/A inst->completeAcc(data_pkt); 8982292SN/A } 8992301SN/A completeStore(storeWBIdx); 9002292SN/A incrStIdx(storeWBIdx); 9012292SN/A continue; 9022292SN/A } 9032292SN/A } else { 9042292SN/A // Non-store conditionals do not need a writeback. 9052292SN/A state->noWB = true; 9062292SN/A } 9072292SN/A 9082292SN/A bool split = 9092292SN/A TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit; 9102292SN/A 9112292SN/A ThreadContext *thread = cpu->tcBase(lsqID); 9122292SN/A 9132292SN/A if (req->isMmappedIpr()) { 9142292SN/A assert(!inst->isStoreConditional()); 9152935Sksewell@umich.edu TheISA::handleIprWrite(thread, data_pkt); 9162292SN/A delete data_pkt; 9172980Sgblack@eecs.umich.edu if (split) { 9182980Sgblack@eecs.umich.edu assert(snd_data_pkt->req->isMmappedIpr()); 9192292SN/A TheISA::handleIprWrite(thread, snd_data_pkt); 9201060SN/A delete snd_data_pkt; 9211060SN/A delete sreqLow; 9222292SN/A delete sreqHigh; 9231060SN/A } 9241060SN/A delete state; 9251060SN/A delete req; 9261060SN/A completeStore(storeWBIdx); 9271060SN/A incrStIdx(storeWBIdx); 9282292SN/A } else if (!sendStore(data_pkt)) { 9292292SN/A DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will" 9302292SN/A "retry later\n", 9311062SN/A inst->seqNum); 9322292SN/A 9332292SN/A // Need to store the second packet, if split. 9341060SN/A if (split) { 9352292SN/A state->pktToSend = true; 9362292SN/A state->pendingPacket = snd_data_pkt; 9372292SN/A } 9381060SN/A } else { 9392292SN/A 9402292SN/A // If split, try to send the second packet too 9411062SN/A if (split) { 9422367SN/A assert(snd_data_pkt); 9432367SN/A 9442367SN/A // Ensure there are enough ports to use. 9452367SN/A if (usedPorts < cachePorts) { 9462367SN/A ++usedPorts; 9472292SN/A if (sendStore(snd_data_pkt)) { 9481061SN/A storePostSend(snd_data_pkt); 9491062SN/A } else { 9501060SN/A DPRINTF(IEW, "D-Cache became blocked when writing" 9511060SN/A " [sn:%lli] second packet, will retry later\n", 9521060SN/A inst->seqNum); 9531060SN/A } 9541060SN/A } else { 9552292SN/A 9561060SN/A // Store the packet for when there's free ports. 9572292SN/A assert(pendingPkt == NULL); 9582292SN/A pendingPkt = snd_data_pkt; 9592292SN/A hasPendingPkt = true; 9602292SN/A } 9612980Sgblack@eecs.umich.edu } else { 9622980Sgblack@eecs.umich.edu 9631060SN/A // Not a split store. 9641061SN/A storePostSend(data_pkt); 9651060SN/A } 9662292SN/A } 9672292SN/A } 9682292SN/A 9692292SN/A // Not sure this should set it to 0. 9702292SN/A usedPorts = 0; 9712292SN/A 9721060SN/A assert(stores >= 0 && storesToWB >= 0); 9731060SN/A} 9741060SN/A 9752292SN/A/*template <class Impl> 9762292SN/Avoid 9772292SN/ALSQUnit<Impl>::removeMSHR(InstSeqNum seqNum) 9782292SN/A{ 9792292SN/A list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(), 9802292SN/A mshrSeqNums.end(), 9812292SN/A seqNum); 9821060SN/A 9832329SN/A if (mshr_it != mshrSeqNums.end()) { 9842329SN/A mshrSeqNums.erase(mshr_it); 9852292SN/A DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size()); 9861061SN/A } 9872292SN/A}*/ 9882292SN/A 9891061SN/Atemplate <class Impl> 9902292SN/Avoid 9911060SN/ALSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 9921060SN/A{ 9931060SN/A DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 9941061SN/A "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 9951061SN/A 9962292SN/A int load_idx = loadTail; 9971061SN/A decrLdIdx(load_idx); 9982292SN/A 9992292SN/A while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) { 10001061SN/A DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 10011061SN/A "[sn:%lli]\n", 10021061SN/A loadQueue[load_idx]->pcState(), 10031061SN/A loadQueue[load_idx]->seqNum); 10041061SN/A 10052292SN/A if (isStalled() && load_idx == stallingLoadIdx) { 10061061SN/A stalled = false; 10073773Sgblack@eecs.umich.edu stallingStoreIsn = 0; 10083773Sgblack@eecs.umich.edu stallingLoadIdx = 0; 10093773Sgblack@eecs.umich.edu } 10103773Sgblack@eecs.umich.edu 10113773Sgblack@eecs.umich.edu // Clear the smart pointer to make sure it is decremented. 10123773Sgblack@eecs.umich.edu loadQueue[load_idx]->setSquashed(); 10131061SN/A loadQueue[load_idx] = NULL; 10141061SN/A --loads; 10151061SN/A 10163773Sgblack@eecs.umich.edu // Inefficient! 10171061SN/A loadTail = load_idx; 10182292SN/A 10193773Sgblack@eecs.umich.edu decrLdIdx(load_idx); 10202292SN/A ++lsqSquashedLoads; 10211061SN/A } 10221061SN/A 10231061SN/A if (memDepViolator && squashed_num < memDepViolator->seqNum) { 10242292SN/A memDepViolator = NULL; 10252292SN/A } 10262292SN/A 10271061SN/A int store_idx = storeTail; 10281061SN/A decrStIdx(store_idx); 10291061SN/A 10301062SN/A while (stores != 0 && 10311062SN/A storeQueue[store_idx].inst->seqNum > squashed_num) { 10321061SN/A // Instructions marked as can WB are already committed. 10331061SN/A if (storeQueue[store_idx].canWB) { 10341061SN/A break; 10351061SN/A } 10361061SN/A 10372292SN/A DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 10381061SN/A "idx:%i [sn:%lli]\n", 10392292SN/A storeQueue[store_idx].inst->pcState(), 10401061SN/A store_idx, storeQueue[store_idx].inst->seqNum); 10411061SN/A 10421061SN/A // I don't think this can happen. It should have been cleared 10432292SN/A // by the stalling load. 10442292SN/A if (isStalled() && 10452292SN/A storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 10463773Sgblack@eecs.umich.edu panic("Is stalled should have been cleared by stalling load!\n"); 10473773Sgblack@eecs.umich.edu stalled = false; 10483773Sgblack@eecs.umich.edu stallingStoreIsn = 0; 10493773Sgblack@eecs.umich.edu } 10503773Sgblack@eecs.umich.edu 10513773Sgblack@eecs.umich.edu // Clear the smart pointer to make sure it is decremented. 10523773Sgblack@eecs.umich.edu storeQueue[store_idx].inst->setSquashed(); 10531061SN/A storeQueue[store_idx].inst = NULL; 10542292SN/A storeQueue[store_idx].canWB = 0; 10552292SN/A 10563773Sgblack@eecs.umich.edu // Must delete request now that it wasn't handed off to 10571061SN/A // memory. This is quite ugly. @todo: Figure out the proper 10582292SN/A // place to really handle request deletes. 10592292SN/A delete storeQueue[store_idx].req; 10601062SN/A if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) { 10612292SN/A delete storeQueue[store_idx].sreqLow; 10623773Sgblack@eecs.umich.edu delete storeQueue[store_idx].sreqHigh; 10632292SN/A 10641062SN/A storeQueue[store_idx].sreqLow = NULL; 10652292SN/A storeQueue[store_idx].sreqHigh = NULL; 10663773Sgblack@eecs.umich.edu } 10672292SN/A 10682292SN/A storeQueue[store_idx].req = NULL; 10691062SN/A --stores; 10702292SN/A 10711062SN/A // Inefficient! 10722935Sksewell@umich.edu storeTail = store_idx; 10732935Sksewell@umich.edu 10742935Sksewell@umich.edu decrStIdx(store_idx); 10752292SN/A ++lsqSquashedStores; 10761062SN/A } 10772292SN/A} 10782292SN/A 10792292SN/Atemplate <class Impl> 10802292SN/Avoid 10812292SN/ALSQUnit<Impl>::storePostSend(PacketPtr pkt) 10822292SN/A{ 10832292SN/A if (isStalled() && 10842292SN/A storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) { 10851062SN/A DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 10862292SN/A "load idx:%i\n", 10871061SN/A stallingStoreIsn, stallingLoadIdx); 10881061SN/A stalled = false; 10891061SN/A stallingStoreIsn = 0; 10901061SN/A iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 10911061SN/A } 10922292SN/A 10931061SN/A if (!storeQueue[storeWBIdx].inst->isStoreConditional()) { 10942292SN/A // The store is basically completed at this time. This 10952292SN/A // only works so long as the checker doesn't try to 10962292SN/A // verify the value in memory for stores. 10972292SN/A storeQueue[storeWBIdx].inst->setCompleted(); 10982292SN/A 10992292SN/A if (cpu->checker) { 11001061SN/A cpu->checker->verify(storeQueue[storeWBIdx].inst); 11011061SN/A } 11021061SN/A } 11031061SN/A 11042292SN/A if (needsTSO) { 11051061SN/A storeInFlight = true; 11062292SN/A } 11072292SN/A 11082292SN/A incrStIdx(storeWBIdx); 11092292SN/A} 11102292SN/A 11112292SN/Atemplate <class Impl> 11122292SN/Avoid 11132292SN/ALSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt) 11142292SN/A{ 11152292SN/A iewStage->wakeCPU(); 11162292SN/A 11172292SN/A // Squashed instructions do not need to complete their access. 11182292SN/A if (inst->isSquashed()) { 11192292SN/A assert(!inst->isStore()); 11202292SN/A ++lsqIgnoredResponses; 11212292SN/A return; 11222292SN/A } 11232292SN/A 11242292SN/A if (!inst->isExecuted()) { 11252292SN/A inst->setExecuted(); 11262292SN/A 11272292SN/A // Complete access to copy data to proper place. 11282292SN/A inst->completeAcc(pkt); 11292292SN/A } 11302292SN/A 11312292SN/A // Need to insert instruction into queue to commit 11322292SN/A iewStage->instToCommit(inst); 11332731Sktlim@umich.edu 11342292SN/A iewStage->activityThisCycle(); 11352292SN/A 11362292SN/A // see if this load changed the PC 11372292SN/A iewStage->checkMisprediction(inst); 11382292SN/A} 11392292SN/A 11402292SN/Atemplate <class Impl> 11412292SN/Avoid 11422292SN/ALSQUnit<Impl>::completeStore(int store_idx) 11432292SN/A{ 11442292SN/A assert(storeQueue[store_idx].inst); 11452292SN/A storeQueue[store_idx].completed = true; 11462292SN/A --storesToWB; 11472292SN/A // A bit conservative because a store completion may not free up entries, 11482292SN/A // but hopefully avoids two store completions in one cycle from making 11492292SN/A // the CPU tick twice. 11502292SN/A cpu->wakeCPU(); 11512292SN/A cpu->activityThisCycle(); 11522292SN/A 11532292SN/A if (store_idx == storeHead) { 11542292SN/A do { 11552292SN/A incrStIdx(storeHead); 11562292SN/A 11572292SN/A --stores; 11582292SN/A } while (storeQueue[storeHead].completed && 11592292SN/A storeHead != storeTail); 11602292SN/A 11612292SN/A iewStage->updateLSQNextCycle = true; 11622292SN/A } 11632292SN/A 11642292SN/A DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 11652292SN/A "idx:%i\n", 11662292SN/A storeQueue[store_idx].inst->seqNum, store_idx, storeHead); 11672292SN/A 11682292SN/A#if TRACING_ON 11692292SN/A if (DTRACE(O3PipeView)) { 11702292SN/A storeQueue[store_idx].inst->storeTick = 11712292SN/A curTick() - storeQueue[store_idx].inst->fetchTick; 11722292SN/A } 11732292SN/A#endif 11742292SN/A 11752292SN/A if (isStalled() && 11762292SN/A storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 11772292SN/A DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 11782292SN/A "load idx:%i\n", 11792292SN/A stallingStoreIsn, stallingLoadIdx); 11802292SN/A stalled = false; 11812292SN/A stallingStoreIsn = 0; 11822292SN/A iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 11832292SN/A } 11842292SN/A 11852292SN/A storeQueue[store_idx].inst->setCompleted(); 11862292SN/A 11872301SN/A if (needsTSO) { 11882292SN/A storeInFlight = false; 11892301SN/A } 11902292SN/A 11912292SN/A // Tell the checker we've completed this instruction. Some stores 11922292SN/A // may get reported twice to the checker, but the checker can 11932292SN/A // handle that case. 11942292SN/A if (cpu->checker) { 11952292SN/A cpu->checker->verify(storeQueue[store_idx].inst); 11962292SN/A } 11972292SN/A} 11982292SN/A 11992292SN/Atemplate <class Impl> 12002292SN/Abool 12012292SN/ALSQUnit<Impl>::sendStore(PacketPtr data_pkt) 12022292SN/A{ 12032292SN/A if (!dcachePort->sendTimingReq(data_pkt)) { 12042292SN/A // Need to handle becoming blocked on a store. 12052292SN/A isStoreBlocked = true; 12062292SN/A ++lsqCacheBlocked; 12072292SN/A assert(retryPkt == NULL); 12082292SN/A retryPkt = data_pkt; 12092292SN/A return false; 12102292SN/A } 12112292SN/A return true; 12122292SN/A} 12132292SN/A 12142292SN/Atemplate <class Impl> 12152292SN/Avoid 12162292SN/ALSQUnit<Impl>::recvRetry() 12172292SN/A{ 12182292SN/A if (isStoreBlocked) { 12192292SN/A DPRINTF(LSQUnit, "Receiving retry: store blocked\n"); 12202292SN/A assert(retryPkt != NULL); 12212292SN/A 12222292SN/A LSQSenderState *state = 12232292SN/A dynamic_cast<LSQSenderState *>(retryPkt->senderState); 12242292SN/A 12252292SN/A if (dcachePort->sendTimingReq(retryPkt)) { 12262292SN/A // Don't finish the store unless this is the last packet. 12272292SN/A if (!TheISA::HasUnalignedMemAcc || !state->pktToSend || 12282292SN/A state->pendingPacket == retryPkt) { 12292292SN/A state->pktToSend = false; 12302292SN/A storePostSend(retryPkt); 12312292SN/A } 12322292SN/A retryPkt = NULL; 12332292SN/A isStoreBlocked = false; 12342292SN/A 12352292SN/A // Send any outstanding packet. 12362292SN/A if (TheISA::HasUnalignedMemAcc && state->pktToSend) { 12372292SN/A assert(state->pendingPacket); 12382292SN/A if (sendStore(state->pendingPacket)) { 12392292SN/A storePostSend(state->pendingPacket); 12402292SN/A } 12412292SN/A } 12422292SN/A } else { 12432301SN/A // Still blocked! 12442292SN/A ++lsqCacheBlocked; 12452292SN/A } 12462292SN/A } 12472292SN/A} 12482292SN/A 12492292SN/Atemplate <class Impl> 12502292SN/Ainline void 12512292SN/ALSQUnit<Impl>::incrStIdx(int &store_idx) const 12522292SN/A{ 12533093Sksewell@umich.edu if (++store_idx >= SQEntries) 12543093Sksewell@umich.edu store_idx = 0; 12553093Sksewell@umich.edu} 12562935Sksewell@umich.edu 12572935Sksewell@umich.edutemplate <class Impl> 12582935Sksewell@umich.eduinline void 12592935Sksewell@umich.eduLSQUnit<Impl>::decrStIdx(int &store_idx) const 12602292SN/A{ 12612292SN/A if (--store_idx < 0) 12622292SN/A store_idx += SQEntries; 12632292SN/A} 12642292SN/A 12652292SN/Atemplate <class Impl> 12662292SN/Ainline void 12672292SN/ALSQUnit<Impl>::incrLdIdx(int &load_idx) const 12682292SN/A{ 12692292SN/A if (++load_idx >= LQEntries) 12702292SN/A load_idx = 0; 12712292SN/A} 12722292SN/A 12732292SN/Atemplate <class Impl> 12742292SN/Ainline void 12752292SN/ALSQUnit<Impl>::decrLdIdx(int &load_idx) const 12762292SN/A{ 12772292SN/A if (--load_idx < 0) 12782292SN/A load_idx += LQEntries; 12792292SN/A} 12802292SN/A 12812292SN/Atemplate <class Impl> 12822292SN/Avoid 12832292SN/ALSQUnit<Impl>::dumpInsts() const 12842292SN/A{ 12852292SN/A cprintf("Load store queue: Dumping instructions.\n"); 12862292SN/A cprintf("Load queue size: %i\n", loads); 12872292SN/A cprintf("Load queue: "); 12882292SN/A 12892292SN/A int load_idx = loadHead; 12903798Sgblack@eecs.umich.edu 12913798Sgblack@eecs.umich.edu while (load_idx != loadTail && loadQueue[load_idx]) { 12923798Sgblack@eecs.umich.edu const DynInstPtr &inst(loadQueue[load_idx]); 12933798Sgblack@eecs.umich.edu cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 12943798Sgblack@eecs.umich.edu 12953798Sgblack@eecs.umich.edu incrLdIdx(load_idx); 12963798Sgblack@eecs.umich.edu } 12973798Sgblack@eecs.umich.edu cprintf("\n"); 12983798Sgblack@eecs.umich.edu 12993798Sgblack@eecs.umich.edu cprintf("Store queue size: %i\n", stores); 13003798Sgblack@eecs.umich.edu cprintf("Store queue: "); 13013798Sgblack@eecs.umich.edu 13023788Sgblack@eecs.umich.edu int store_idx = storeHead; 13033788Sgblack@eecs.umich.edu 13042292SN/A while (store_idx != storeTail && storeQueue[store_idx].inst) { 13053788Sgblack@eecs.umich.edu const DynInstPtr &inst(storeQueue[store_idx].inst); 13063788Sgblack@eecs.umich.edu cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 13073788Sgblack@eecs.umich.edu 13082292SN/A incrStIdx(store_idx); 13092292SN/A } 13102301SN/A 13112292SN/A cprintf("\n"); 13122301SN/A} 13132292SN/A 13142292SN/A#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 13152301SN/A