lsq_unit_impl.hh revision 10239:592f0bb6bd6f
13569Sgblack@eecs.umich.edu 23569Sgblack@eecs.umich.edu/* 33569Sgblack@eecs.umich.edu * Copyright (c) 2010-2013 ARM Limited 43569Sgblack@eecs.umich.edu * Copyright (c) 2013 Advanced Micro Devices, Inc. 53569Sgblack@eecs.umich.edu * All rights reserved 63569Sgblack@eecs.umich.edu * 73569Sgblack@eecs.umich.edu * The license below extends only to copyright in the software and shall 83569Sgblack@eecs.umich.edu * not be construed as granting a license to any other intellectual 93569Sgblack@eecs.umich.edu * property including but not limited to intellectual property relating 103569Sgblack@eecs.umich.edu * to a hardware implementation of the functionality of the software 113569Sgblack@eecs.umich.edu * licensed hereunder. You may use the software subject to the license 123569Sgblack@eecs.umich.edu * terms below provided that you ensure that this notice is replicated 133569Sgblack@eecs.umich.edu * unmodified and in its entirety in all distributions of the software, 143569Sgblack@eecs.umich.edu * modified or unmodified, in source code or in binary form. 153569Sgblack@eecs.umich.edu * 163569Sgblack@eecs.umich.edu * Copyright (c) 2004-2005 The Regents of The University of Michigan 173569Sgblack@eecs.umich.edu * All rights reserved. 183569Sgblack@eecs.umich.edu * 193569Sgblack@eecs.umich.edu * Redistribution and use in source and binary forms, with or without 203569Sgblack@eecs.umich.edu * modification, are permitted provided that the following conditions are 213569Sgblack@eecs.umich.edu * met: redistributions of source code must retain the above copyright 223569Sgblack@eecs.umich.edu * notice, this list of conditions and the following disclaimer; 233569Sgblack@eecs.umich.edu * redistributions in binary form must reproduce the above copyright 243569Sgblack@eecs.umich.edu * notice, this list of conditions and the following disclaimer in the 253569Sgblack@eecs.umich.edu * documentation and/or other materials provided with the distribution; 263569Sgblack@eecs.umich.edu * neither the name of the copyright holders nor the names of its 273569Sgblack@eecs.umich.edu * contributors may be used to endorse or promote products derived from 283804Ssaidi@eecs.umich.edu * this software without specific prior written permission. 293569Sgblack@eecs.umich.edu * 303569Sgblack@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 313804Ssaidi@eecs.umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 323811Ssaidi@eecs.umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 333569Sgblack@eecs.umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 343824Ssaidi@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 353811Ssaidi@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 363811Ssaidi@eecs.umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 373823Ssaidi@eecs.umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 383823Ssaidi@eecs.umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 393823Ssaidi@eecs.umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 403569Sgblack@eecs.umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 413569Sgblack@eecs.umich.edu * 423804Ssaidi@eecs.umich.edu * Authors: Kevin Lim 433804Ssaidi@eecs.umich.edu * Korey Sewell 443569Sgblack@eecs.umich.edu */ 453569Sgblack@eecs.umich.edu 463569Sgblack@eecs.umich.edu#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 473804Ssaidi@eecs.umich.edu#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 483881Ssaidi@eecs.umich.edu 493881Ssaidi@eecs.umich.edu#include "arch/generic/debugfaults.hh" 503804Ssaidi@eecs.umich.edu#include "arch/locked_mem.hh" 513804Ssaidi@eecs.umich.edu#include "base/str.hh" 523804Ssaidi@eecs.umich.edu#include "config/the_isa.hh" 533804Ssaidi@eecs.umich.edu#include "cpu/checker/cpu.hh" 543569Sgblack@eecs.umich.edu#include "cpu/o3/lsq.hh" 553804Ssaidi@eecs.umich.edu#include "cpu/o3/lsq_unit.hh" 563804Ssaidi@eecs.umich.edu#include "debug/Activity.hh" 573881Ssaidi@eecs.umich.edu#include "debug/IEW.hh" 583881Ssaidi@eecs.umich.edu#include "debug/LSQUnit.hh" 593881Ssaidi@eecs.umich.edu#include "debug/O3PipeView.hh" 603804Ssaidi@eecs.umich.edu#include "mem/packet.hh" 613569Sgblack@eecs.umich.edu#include "mem/request.hh" 623804Ssaidi@eecs.umich.edu 633804Ssaidi@eecs.umich.edutemplate<class Impl> 643804Ssaidi@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt, 653804Ssaidi@eecs.umich.edu LSQUnit *lsq_ptr) 663881Ssaidi@eecs.umich.edu : Event(Default_Pri, AutoDelete), 673804Ssaidi@eecs.umich.edu inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 683804Ssaidi@eecs.umich.edu{ 693804Ssaidi@eecs.umich.edu} 703804Ssaidi@eecs.umich.edu 713804Ssaidi@eecs.umich.edutemplate<class Impl> 723804Ssaidi@eecs.umich.eduvoid 733804Ssaidi@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::process() 743569Sgblack@eecs.umich.edu{ 753569Sgblack@eecs.umich.edu assert(!lsqPtr->cpu->switchedOut()); 763804Ssaidi@eecs.umich.edu 773804Ssaidi@eecs.umich.edu lsqPtr->writeback(inst, pkt); 783826Ssaidi@eecs.umich.edu 793804Ssaidi@eecs.umich.edu if (pkt->senderState) 803569Sgblack@eecs.umich.edu delete pkt->senderState; 813569Sgblack@eecs.umich.edu 823804Ssaidi@eecs.umich.edu delete pkt->req; 833826Ssaidi@eecs.umich.edu delete pkt; 843881Ssaidi@eecs.umich.edu} 853826Ssaidi@eecs.umich.edu 863811Ssaidi@eecs.umich.edutemplate<class Impl> 873836Ssaidi@eecs.umich.educonst char * 883881Ssaidi@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::description() const 893881Ssaidi@eecs.umich.edu{ 903881Ssaidi@eecs.umich.edu return "Store writeback"; 913881Ssaidi@eecs.umich.edu} 923881Ssaidi@eecs.umich.edu 933836Ssaidi@eecs.umich.edutemplate<class Impl> 943881Ssaidi@eecs.umich.eduvoid 953881Ssaidi@eecs.umich.eduLSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 963881Ssaidi@eecs.umich.edu{ 973881Ssaidi@eecs.umich.edu LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 983881Ssaidi@eecs.umich.edu DynInstPtr inst = state->inst; 993881Ssaidi@eecs.umich.edu DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum); 1003881Ssaidi@eecs.umich.edu DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum); 1013881Ssaidi@eecs.umich.edu 1023881Ssaidi@eecs.umich.edu //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum); 1033881Ssaidi@eecs.umich.edu 1043881Ssaidi@eecs.umich.edu // If this is a split access, wait until all packets are received. 1053881Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && !state->complete()) { 1063881Ssaidi@eecs.umich.edu delete pkt->req; 1073881Ssaidi@eecs.umich.edu delete pkt; 1083881Ssaidi@eecs.umich.edu return; 1093881Ssaidi@eecs.umich.edu } 1103881Ssaidi@eecs.umich.edu 1113881Ssaidi@eecs.umich.edu assert(!cpu->switchedOut()); 1123811Ssaidi@eecs.umich.edu if (inst->isSquashed()) { 1133826Ssaidi@eecs.umich.edu iewStage->decrWb(inst->seqNum); 1143826Ssaidi@eecs.umich.edu } else { 1153826Ssaidi@eecs.umich.edu if (!state->noWB) { 1163826Ssaidi@eecs.umich.edu if (!TheISA::HasUnalignedMemAcc || !state->isSplit || 1173881Ssaidi@eecs.umich.edu !state->isLoad) { 1183881Ssaidi@eecs.umich.edu writeback(inst, pkt); 1193881Ssaidi@eecs.umich.edu } else { 1203881Ssaidi@eecs.umich.edu writeback(inst, state->mainPkt); 1213881Ssaidi@eecs.umich.edu } 1223881Ssaidi@eecs.umich.edu } 1233881Ssaidi@eecs.umich.edu 1243881Ssaidi@eecs.umich.edu if (inst->isStore()) { 1253881Ssaidi@eecs.umich.edu completeStore(state->idx); 1263881Ssaidi@eecs.umich.edu } 1273881Ssaidi@eecs.umich.edu } 1283881Ssaidi@eecs.umich.edu 1293881Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) { 1303881Ssaidi@eecs.umich.edu delete state->mainPkt->req; 1313881Ssaidi@eecs.umich.edu delete state->mainPkt; 1323881Ssaidi@eecs.umich.edu } 1333826Ssaidi@eecs.umich.edu 1343826Ssaidi@eecs.umich.edu pkt->req->setAccessLatency(); 1353826Ssaidi@eecs.umich.edu cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1363826Ssaidi@eecs.umich.edu 1373826Ssaidi@eecs.umich.edu delete state; 1383881Ssaidi@eecs.umich.edu delete pkt->req; 1393569Sgblack@eecs.umich.edu delete pkt; 1403569Sgblack@eecs.umich.edu} 1413881Ssaidi@eecs.umich.edu 1423804Ssaidi@eecs.umich.edutemplate <class Impl> 1433881Ssaidi@eecs.umich.eduLSQUnit<Impl>::LSQUnit() 1443826Ssaidi@eecs.umich.edu : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 1453881Ssaidi@eecs.umich.edu isStoreBlocked(false), isLoadBlocked(false), 1463881Ssaidi@eecs.umich.edu loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false) 1473881Ssaidi@eecs.umich.edu{ 1483881Ssaidi@eecs.umich.edu} 1493881Ssaidi@eecs.umich.edu 1503569Sgblack@eecs.umich.edutemplate<class Impl> 1513804Ssaidi@eecs.umich.eduvoid 1523804Ssaidi@eecs.umich.eduLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 1533881Ssaidi@eecs.umich.edu LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 1543804Ssaidi@eecs.umich.edu unsigned id) 1553804Ssaidi@eecs.umich.edu{ 1563804Ssaidi@eecs.umich.edu cpu = cpu_ptr; 1573804Ssaidi@eecs.umich.edu iewStage = iew_ptr; 1583804Ssaidi@eecs.umich.edu 1593804Ssaidi@eecs.umich.edu lsq = lsq_ptr; 1603804Ssaidi@eecs.umich.edu 1613569Sgblack@eecs.umich.edu lsqID = id; 1623569Sgblack@eecs.umich.edu 1633569Sgblack@eecs.umich.edu DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id); 1643863Ssaidi@eecs.umich.edu 1653863Ssaidi@eecs.umich.edu // Add 1 for the sentinel entry (they are circular queues). 1663804Ssaidi@eecs.umich.edu LQEntries = maxLQEntries + 1; 1673804Ssaidi@eecs.umich.edu SQEntries = maxSQEntries + 1; 1683804Ssaidi@eecs.umich.edu 1693804Ssaidi@eecs.umich.edu //Due to uint8_t index in LSQSenderState 1703804Ssaidi@eecs.umich.edu assert(LQEntries <= 256); 1713804Ssaidi@eecs.umich.edu assert(SQEntries <= 256); 1723804Ssaidi@eecs.umich.edu 1733804Ssaidi@eecs.umich.edu loadQueue.resize(LQEntries); 1743804Ssaidi@eecs.umich.edu storeQueue.resize(SQEntries); 1753569Sgblack@eecs.umich.edu 1763804Ssaidi@eecs.umich.edu depCheckShift = params->LSQDepCheckShift; 1773804Ssaidi@eecs.umich.edu checkLoads = params->LSQCheckLoads; 1783804Ssaidi@eecs.umich.edu cachePorts = params->cachePorts; 1793804Ssaidi@eecs.umich.edu needsTSO = params->needsTSO; 1803804Ssaidi@eecs.umich.edu 1813804Ssaidi@eecs.umich.edu resetState(); 1823804Ssaidi@eecs.umich.edu} 1833804Ssaidi@eecs.umich.edu 1843804Ssaidi@eecs.umich.edu 1853811Ssaidi@eecs.umich.edutemplate<class Impl> 1863811Ssaidi@eecs.umich.eduvoid 1873804Ssaidi@eecs.umich.eduLSQUnit<Impl>::resetState() 1883804Ssaidi@eecs.umich.edu{ 1893863Ssaidi@eecs.umich.edu loads = stores = storesToWB = 0; 1903804Ssaidi@eecs.umich.edu 1913804Ssaidi@eecs.umich.edu loadHead = loadTail = 0; 1923804Ssaidi@eecs.umich.edu 1933804Ssaidi@eecs.umich.edu storeHead = storeWBIdx = storeTail = 0; 1943804Ssaidi@eecs.umich.edu 1953804Ssaidi@eecs.umich.edu usedPorts = 0; 1963804Ssaidi@eecs.umich.edu 1973811Ssaidi@eecs.umich.edu retryPkt = NULL; 1983804Ssaidi@eecs.umich.edu memDepViolator = NULL; 1993804Ssaidi@eecs.umich.edu 2003804Ssaidi@eecs.umich.edu blockedLoadSeqNum = 0; 2013804Ssaidi@eecs.umich.edu 2023804Ssaidi@eecs.umich.edu stalled = false; 2033826Ssaidi@eecs.umich.edu isLoadBlocked = false; 2043826Ssaidi@eecs.umich.edu loadBlockedHandled = false; 2053804Ssaidi@eecs.umich.edu 2063804Ssaidi@eecs.umich.edu cacheBlockMask = ~(cpu->cacheLineSize() - 1); 2073804Ssaidi@eecs.umich.edu} 2083804Ssaidi@eecs.umich.edu 2093804Ssaidi@eecs.umich.edutemplate<class Impl> 2103804Ssaidi@eecs.umich.edustd::string 2113804Ssaidi@eecs.umich.eduLSQUnit<Impl>::name() const 2123804Ssaidi@eecs.umich.edu{ 2133804Ssaidi@eecs.umich.edu if (Impl::MaxThreads == 1) { 2143804Ssaidi@eecs.umich.edu return iewStage->name() + ".lsq"; 2153804Ssaidi@eecs.umich.edu } else { 2163804Ssaidi@eecs.umich.edu return iewStage->name() + ".lsq.thread" + to_string(lsqID); 2173804Ssaidi@eecs.umich.edu } 2183826Ssaidi@eecs.umich.edu} 2193826Ssaidi@eecs.umich.edu 2203826Ssaidi@eecs.umich.edutemplate<class Impl> 2213863Ssaidi@eecs.umich.eduvoid 2223826Ssaidi@eecs.umich.eduLSQUnit<Impl>::regStats() 2233826Ssaidi@eecs.umich.edu{ 2243826Ssaidi@eecs.umich.edu lsqForwLoads 2253826Ssaidi@eecs.umich.edu .name(name() + ".forwLoads") 2263826Ssaidi@eecs.umich.edu .desc("Number of loads that had data forwarded from stores"); 2273826Ssaidi@eecs.umich.edu 2283826Ssaidi@eecs.umich.edu invAddrLoads 2293826Ssaidi@eecs.umich.edu .name(name() + ".invAddrLoads") 2303826Ssaidi@eecs.umich.edu .desc("Number of loads ignored due to an invalid address"); 2313804Ssaidi@eecs.umich.edu 2323804Ssaidi@eecs.umich.edu lsqSquashedLoads 2333804Ssaidi@eecs.umich.edu .name(name() + ".squashedLoads") 2343804Ssaidi@eecs.umich.edu .desc("Number of loads squashed"); 2353804Ssaidi@eecs.umich.edu 2363804Ssaidi@eecs.umich.edu lsqIgnoredResponses 2373804Ssaidi@eecs.umich.edu .name(name() + ".ignoredResponses") 2383863Ssaidi@eecs.umich.edu .desc("Number of memory responses ignored because the instruction is squashed"); 2393863Ssaidi@eecs.umich.edu 2403863Ssaidi@eecs.umich.edu lsqMemOrderViolation 2413836Ssaidi@eecs.umich.edu .name(name() + ".memOrderViolation") 2423836Ssaidi@eecs.umich.edu .desc("Number of memory ordering violations"); 2433804Ssaidi@eecs.umich.edu 2443804Ssaidi@eecs.umich.edu lsqSquashedStores 2453863Ssaidi@eecs.umich.edu .name(name() + ".squashedStores") 2463804Ssaidi@eecs.umich.edu .desc("Number of stores squashed"); 2473804Ssaidi@eecs.umich.edu 2483804Ssaidi@eecs.umich.edu invAddrSwpfs 2493804Ssaidi@eecs.umich.edu .name(name() + ".invAddrSwpfs") 2503804Ssaidi@eecs.umich.edu .desc("Number of software prefetches ignored due to an invalid address"); 2513804Ssaidi@eecs.umich.edu 2523804Ssaidi@eecs.umich.edu lsqBlockedLoads 2533863Ssaidi@eecs.umich.edu .name(name() + ".blockedLoads") 2543804Ssaidi@eecs.umich.edu .desc("Number of blocked loads due to partial load-store forwarding"); 2553804Ssaidi@eecs.umich.edu 2563804Ssaidi@eecs.umich.edu lsqRescheduledLoads 2573804Ssaidi@eecs.umich.edu .name(name() + ".rescheduledLoads") 2583804Ssaidi@eecs.umich.edu .desc("Number of loads that were rescheduled"); 2593881Ssaidi@eecs.umich.edu 2603881Ssaidi@eecs.umich.edu lsqCacheBlocked 2613804Ssaidi@eecs.umich.edu .name(name() + ".cacheBlocked") 2623804Ssaidi@eecs.umich.edu .desc("Number of times an access to memory failed due to the cache being blocked"); 2633804Ssaidi@eecs.umich.edu} 2643804Ssaidi@eecs.umich.edu 2653804Ssaidi@eecs.umich.edutemplate<class Impl> 2663804Ssaidi@eecs.umich.eduvoid 2673804Ssaidi@eecs.umich.eduLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2683804Ssaidi@eecs.umich.edu{ 2693863Ssaidi@eecs.umich.edu dcachePort = dcache_port; 2703863Ssaidi@eecs.umich.edu} 2713836Ssaidi@eecs.umich.edu 2723804Ssaidi@eecs.umich.edutemplate<class Impl> 2733804Ssaidi@eecs.umich.eduvoid 2743804Ssaidi@eecs.umich.eduLSQUnit<Impl>::clearLQ() 2753881Ssaidi@eecs.umich.edu{ 2763881Ssaidi@eecs.umich.edu loadQueue.clear(); 2773881Ssaidi@eecs.umich.edu} 2783881Ssaidi@eecs.umich.edu 2793804Ssaidi@eecs.umich.edutemplate<class Impl> 2803804Ssaidi@eecs.umich.eduvoid 2813804Ssaidi@eecs.umich.eduLSQUnit<Impl>::clearSQ() 2823804Ssaidi@eecs.umich.edu{ 2833804Ssaidi@eecs.umich.edu storeQueue.clear(); 2843804Ssaidi@eecs.umich.edu} 2853804Ssaidi@eecs.umich.edu 2863804Ssaidi@eecs.umich.edutemplate<class Impl> 2873804Ssaidi@eecs.umich.eduvoid 2883804Ssaidi@eecs.umich.eduLSQUnit<Impl>::drainSanityCheck() const 2893804Ssaidi@eecs.umich.edu{ 2903804Ssaidi@eecs.umich.edu for (int i = 0; i < loadQueue.size(); ++i) 2913804Ssaidi@eecs.umich.edu assert(!loadQueue[i]); 2923804Ssaidi@eecs.umich.edu 2933863Ssaidi@eecs.umich.edu assert(storesToWB == 0); 2943836Ssaidi@eecs.umich.edu assert(!retryPkt); 2953804Ssaidi@eecs.umich.edu} 2963804Ssaidi@eecs.umich.edu 2973881Ssaidi@eecs.umich.edutemplate<class Impl> 2983881Ssaidi@eecs.umich.eduvoid 2993881Ssaidi@eecs.umich.eduLSQUnit<Impl>::takeOverFrom() 3003881Ssaidi@eecs.umich.edu{ 3013804Ssaidi@eecs.umich.edu resetState(); 3023804Ssaidi@eecs.umich.edu} 3033804Ssaidi@eecs.umich.edu 3043804Ssaidi@eecs.umich.edutemplate<class Impl> 3053804Ssaidi@eecs.umich.eduvoid 3063804Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeLQ(unsigned size) 3073804Ssaidi@eecs.umich.edu{ 3083804Ssaidi@eecs.umich.edu unsigned size_plus_sentinel = size + 1; 3093804Ssaidi@eecs.umich.edu assert(size_plus_sentinel >= LQEntries); 3103804Ssaidi@eecs.umich.edu 3113804Ssaidi@eecs.umich.edu if (size_plus_sentinel > LQEntries) { 3123804Ssaidi@eecs.umich.edu while (size_plus_sentinel > loadQueue.size()) { 3133804Ssaidi@eecs.umich.edu DynInstPtr dummy; 3143804Ssaidi@eecs.umich.edu loadQueue.push_back(dummy); 3153836Ssaidi@eecs.umich.edu LQEntries++; 3163836Ssaidi@eecs.umich.edu } 3173881Ssaidi@eecs.umich.edu } else { 3183804Ssaidi@eecs.umich.edu LQEntries = size_plus_sentinel; 3193881Ssaidi@eecs.umich.edu } 3203881Ssaidi@eecs.umich.edu 3213804Ssaidi@eecs.umich.edu assert(LQEntries <= 256); 3223804Ssaidi@eecs.umich.edu} 3233804Ssaidi@eecs.umich.edu 3243804Ssaidi@eecs.umich.edutemplate<class Impl> 3253804Ssaidi@eecs.umich.eduvoid 3263804Ssaidi@eecs.umich.eduLSQUnit<Impl>::resizeSQ(unsigned size) 3273804Ssaidi@eecs.umich.edu{ 3283881Ssaidi@eecs.umich.edu unsigned size_plus_sentinel = size + 1; 3293881Ssaidi@eecs.umich.edu if (size_plus_sentinel > SQEntries) { 3303881Ssaidi@eecs.umich.edu while (size_plus_sentinel > storeQueue.size()) { 3313804Ssaidi@eecs.umich.edu SQEntry dummy; 3323881Ssaidi@eecs.umich.edu storeQueue.push_back(dummy); 3333881Ssaidi@eecs.umich.edu SQEntries++; 3343881Ssaidi@eecs.umich.edu } 3353881Ssaidi@eecs.umich.edu } else { 3363804Ssaidi@eecs.umich.edu SQEntries = size_plus_sentinel; 3373804Ssaidi@eecs.umich.edu } 3383804Ssaidi@eecs.umich.edu 3393804Ssaidi@eecs.umich.edu assert(SQEntries <= 256); 3403804Ssaidi@eecs.umich.edu} 3413804Ssaidi@eecs.umich.edu 3423881Ssaidi@eecs.umich.edutemplate <class Impl> 3433881Ssaidi@eecs.umich.eduvoid 3443804Ssaidi@eecs.umich.eduLSQUnit<Impl>::insert(DynInstPtr &inst) 3453881Ssaidi@eecs.umich.edu{ 3463881Ssaidi@eecs.umich.edu assert(inst->isMemRef()); 3473881Ssaidi@eecs.umich.edu 3483804Ssaidi@eecs.umich.edu assert(inst->isLoad() || inst->isStore()); 3493804Ssaidi@eecs.umich.edu 3503804Ssaidi@eecs.umich.edu if (inst->isLoad()) { 3513804Ssaidi@eecs.umich.edu insertLoad(inst); 3523804Ssaidi@eecs.umich.edu } else { 3533804Ssaidi@eecs.umich.edu insertStore(inst); 3543804Ssaidi@eecs.umich.edu } 3553804Ssaidi@eecs.umich.edu 3563804Ssaidi@eecs.umich.edu inst->setInLSQ(); 3573804Ssaidi@eecs.umich.edu} 3583804Ssaidi@eecs.umich.edu 3593804Ssaidi@eecs.umich.edutemplate <class Impl> 3603804Ssaidi@eecs.umich.eduvoid 3613804Ssaidi@eecs.umich.eduLSQUnit<Impl>::insertLoad(DynInstPtr &load_inst) 3623804Ssaidi@eecs.umich.edu{ 3633804Ssaidi@eecs.umich.edu assert((loadTail + 1) % LQEntries != loadHead); 3643804Ssaidi@eecs.umich.edu assert(loads < LQEntries); 3653804Ssaidi@eecs.umich.edu 3663804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 3673804Ssaidi@eecs.umich.edu load_inst->pcState(), loadTail, load_inst->seqNum); 3683804Ssaidi@eecs.umich.edu 3693804Ssaidi@eecs.umich.edu load_inst->lqIdx = loadTail; 3703804Ssaidi@eecs.umich.edu 3713804Ssaidi@eecs.umich.edu if (stores == 0) { 3723804Ssaidi@eecs.umich.edu load_inst->sqIdx = -1; 3733804Ssaidi@eecs.umich.edu } else { 3743804Ssaidi@eecs.umich.edu load_inst->sqIdx = storeTail; 3753804Ssaidi@eecs.umich.edu } 3763804Ssaidi@eecs.umich.edu 3773804Ssaidi@eecs.umich.edu loadQueue[loadTail] = load_inst; 3783804Ssaidi@eecs.umich.edu 3793804Ssaidi@eecs.umich.edu incrLdIdx(loadTail); 3803804Ssaidi@eecs.umich.edu 3813804Ssaidi@eecs.umich.edu ++loads; 3823826Ssaidi@eecs.umich.edu} 3833804Ssaidi@eecs.umich.edu 3843804Ssaidi@eecs.umich.edutemplate <class Impl> 3853826Ssaidi@eecs.umich.eduvoid 3863826Ssaidi@eecs.umich.eduLSQUnit<Impl>::insertStore(DynInstPtr &store_inst) 3873826Ssaidi@eecs.umich.edu{ 3883826Ssaidi@eecs.umich.edu // Make sure it is not full before inserting an instruction. 3893826Ssaidi@eecs.umich.edu assert((storeTail + 1) % SQEntries != storeHead); 3903804Ssaidi@eecs.umich.edu assert(stores < SQEntries); 3913804Ssaidi@eecs.umich.edu 3923804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 3933804Ssaidi@eecs.umich.edu store_inst->pcState(), storeTail, store_inst->seqNum); 3943804Ssaidi@eecs.umich.edu 3953811Ssaidi@eecs.umich.edu store_inst->sqIdx = storeTail; 3963811Ssaidi@eecs.umich.edu store_inst->lqIdx = loadTail; 3973804Ssaidi@eecs.umich.edu 3983804Ssaidi@eecs.umich.edu storeQueue[storeTail] = SQEntry(store_inst); 3993804Ssaidi@eecs.umich.edu 4003804Ssaidi@eecs.umich.edu incrStIdx(storeTail); 4013826Ssaidi@eecs.umich.edu 4023826Ssaidi@eecs.umich.edu ++stores; 4033826Ssaidi@eecs.umich.edu} 4043826Ssaidi@eecs.umich.edu 4053826Ssaidi@eecs.umich.edutemplate <class Impl> 4063826Ssaidi@eecs.umich.edutypename Impl::DynInstPtr 4073804Ssaidi@eecs.umich.eduLSQUnit<Impl>::getMemDepViolator() 4083804Ssaidi@eecs.umich.edu{ 4093804Ssaidi@eecs.umich.edu DynInstPtr temp = memDepViolator; 4103811Ssaidi@eecs.umich.edu 4113811Ssaidi@eecs.umich.edu memDepViolator = NULL; 4123804Ssaidi@eecs.umich.edu 4133826Ssaidi@eecs.umich.edu return temp; 4143804Ssaidi@eecs.umich.edu} 4153804Ssaidi@eecs.umich.edu 4163836Ssaidi@eecs.umich.edutemplate <class Impl> 4173826Ssaidi@eecs.umich.eduunsigned 4183826Ssaidi@eecs.umich.eduLSQUnit<Impl>::numFreeLoadEntries() 4193826Ssaidi@eecs.umich.edu{ 4203826Ssaidi@eecs.umich.edu //LQ has an extra dummy entry to differentiate 4213826Ssaidi@eecs.umich.edu //empty/full conditions. Subtract 1 from the free entries. 4223826Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads); 4233804Ssaidi@eecs.umich.edu return LQEntries - loads - 1; 4243804Ssaidi@eecs.umich.edu} 4253804Ssaidi@eecs.umich.edu 4263804Ssaidi@eecs.umich.edutemplate <class Impl> 4273833Ssaidi@eecs.umich.eduunsigned 4283833Ssaidi@eecs.umich.eduLSQUnit<Impl>::numFreeStoreEntries() 4293836Ssaidi@eecs.umich.edu{ 4303836Ssaidi@eecs.umich.edu //SQ has an extra dummy entry to differentiate 4313836Ssaidi@eecs.umich.edu //empty/full conditions. Subtract 1 from the free entries. 4323836Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores); 4333836Ssaidi@eecs.umich.edu return SQEntries - stores - 1; 4343836Ssaidi@eecs.umich.edu 4353836Ssaidi@eecs.umich.edu } 4363836Ssaidi@eecs.umich.edu 4373836Ssaidi@eecs.umich.edutemplate <class Impl> 4383836Ssaidi@eecs.umich.eduvoid 4393836Ssaidi@eecs.umich.eduLSQUnit<Impl>::checkSnoop(PacketPtr pkt) 4403836Ssaidi@eecs.umich.edu{ 4413836Ssaidi@eecs.umich.edu int load_idx = loadHead; 4423836Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 4433836Ssaidi@eecs.umich.edu 4443836Ssaidi@eecs.umich.edu // Unlock the cpu-local monitor when the CPU sees a snoop to a locked 4453836Ssaidi@eecs.umich.edu // address. The CPU can speculatively execute a LL operation after a pending 4463836Ssaidi@eecs.umich.edu // SC operation in the pipeline and that can make the cache monitor the CPU 4473836Ssaidi@eecs.umich.edu // is connected to valid while it really shouldn't be. 4483836Ssaidi@eecs.umich.edu for (int x = 0; x < cpu->numContexts(); x++) { 4493836Ssaidi@eecs.umich.edu ThreadContext *tc = cpu->getContext(x); 4503836Ssaidi@eecs.umich.edu bool no_squash = cpu->thread[x]->noSquashFromTC; 4513836Ssaidi@eecs.umich.edu cpu->thread[x]->noSquashFromTC = true; 4523833Ssaidi@eecs.umich.edu TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 4533833Ssaidi@eecs.umich.edu cpu->thread[x]->noSquashFromTC = no_squash; 4543833Ssaidi@eecs.umich.edu } 4553833Ssaidi@eecs.umich.edu 4563833Ssaidi@eecs.umich.edu Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 4573833Ssaidi@eecs.umich.edu 4583833Ssaidi@eecs.umich.edu DynInstPtr ld_inst = loadQueue[load_idx]; 4593833Ssaidi@eecs.umich.edu if (ld_inst) { 4603833Ssaidi@eecs.umich.edu Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 4613804Ssaidi@eecs.umich.edu // Check that this snoop didn't just invalidate our lock flag 4623804Ssaidi@eecs.umich.edu if (ld_inst->effAddrValid() && load_addr == invalidate_addr && 4633804Ssaidi@eecs.umich.edu ld_inst->memReqFlags & Request::LLSC) 4643804Ssaidi@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 4653804Ssaidi@eecs.umich.edu } 4663833Ssaidi@eecs.umich.edu 4673833Ssaidi@eecs.umich.edu // If this is the only load in the LSQ we don't care 4683811Ssaidi@eecs.umich.edu if (load_idx == loadTail) 4693804Ssaidi@eecs.umich.edu return; 4703804Ssaidi@eecs.umich.edu 4713804Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 4723804Ssaidi@eecs.umich.edu 4733804Ssaidi@eecs.umich.edu bool force_squash = false; 4743804Ssaidi@eecs.umich.edu 4753804Ssaidi@eecs.umich.edu while (load_idx != loadTail) { 4763833Ssaidi@eecs.umich.edu DynInstPtr ld_inst = loadQueue[load_idx]; 4773804Ssaidi@eecs.umich.edu 4783804Ssaidi@eecs.umich.edu if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 4793833Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 4803836Ssaidi@eecs.umich.edu continue; 4813836Ssaidi@eecs.umich.edu } 4823836Ssaidi@eecs.umich.edu 4833836Ssaidi@eecs.umich.edu Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 4843804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n", 4853804Ssaidi@eecs.umich.edu ld_inst->seqNum, load_addr, invalidate_addr); 4863804Ssaidi@eecs.umich.edu 4873836Ssaidi@eecs.umich.edu if (load_addr == invalidate_addr || force_squash) { 4883836Ssaidi@eecs.umich.edu if (needsTSO) { 4893804Ssaidi@eecs.umich.edu // If we have a TSO system, as all loads must be ordered with 4903804Ssaidi@eecs.umich.edu // all other loads, this load as well as *all* subsequent loads 4913804Ssaidi@eecs.umich.edu // need to be squashed to prevent possible load reordering. 4923804Ssaidi@eecs.umich.edu force_squash = true; 4933804Ssaidi@eecs.umich.edu } 4943804Ssaidi@eecs.umich.edu if (ld_inst->possibleLoadViolation() || force_squash) { 4953804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 4963804Ssaidi@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 4973804Ssaidi@eecs.umich.edu 4983804Ssaidi@eecs.umich.edu // Mark the load for re-execution 4993804Ssaidi@eecs.umich.edu ld_inst->fault = new ReExec; 5003804Ssaidi@eecs.umich.edu } else { 5013833Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 5023836Ssaidi@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 5033804Ssaidi@eecs.umich.edu 5043804Ssaidi@eecs.umich.edu // Make sure that we don't lose a snoop hitting a LOCKED 5053804Ssaidi@eecs.umich.edu // address since the LOCK* flags don't get updated until 5063804Ssaidi@eecs.umich.edu // commit. 5073804Ssaidi@eecs.umich.edu if (ld_inst->memReqFlags & Request::LLSC) 5083804Ssaidi@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 5093804Ssaidi@eecs.umich.edu 5103804Ssaidi@eecs.umich.edu // If a older load checks this and it's true 5113804Ssaidi@eecs.umich.edu // then we might have missed the snoop 5123804Ssaidi@eecs.umich.edu // in which case we need to invalidate to be sure 5133804Ssaidi@eecs.umich.edu ld_inst->hitExternalSnoop(true); 5143804Ssaidi@eecs.umich.edu } 5153804Ssaidi@eecs.umich.edu } 5163804Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 5173804Ssaidi@eecs.umich.edu } 5183804Ssaidi@eecs.umich.edu return; 5193804Ssaidi@eecs.umich.edu} 5203804Ssaidi@eecs.umich.edu 5213804Ssaidi@eecs.umich.edutemplate <class Impl> 5223804Ssaidi@eecs.umich.eduFault 5233804Ssaidi@eecs.umich.eduLSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst) 5243836Ssaidi@eecs.umich.edu{ 5253836Ssaidi@eecs.umich.edu Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 5263836Ssaidi@eecs.umich.edu Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 5273836Ssaidi@eecs.umich.edu 5283836Ssaidi@eecs.umich.edu /** @todo in theory you only need to check an instruction that has executed 5293826Ssaidi@eecs.umich.edu * however, there isn't a good way in the pipeline at the moment to check 5303836Ssaidi@eecs.umich.edu * all instructions that will execute before the store writes back. Thus, 5313836Ssaidi@eecs.umich.edu * like the implementation that came before it, we're overly conservative. 5323804Ssaidi@eecs.umich.edu */ 5333804Ssaidi@eecs.umich.edu while (load_idx != loadTail) { 5343804Ssaidi@eecs.umich.edu DynInstPtr ld_inst = loadQueue[load_idx]; 5353804Ssaidi@eecs.umich.edu if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 5363804Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 5373804Ssaidi@eecs.umich.edu continue; 5383804Ssaidi@eecs.umich.edu } 5393804Ssaidi@eecs.umich.edu 5403804Ssaidi@eecs.umich.edu Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 5413833Ssaidi@eecs.umich.edu Addr ld_eff_addr2 = 5423836Ssaidi@eecs.umich.edu (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 5433836Ssaidi@eecs.umich.edu 5443836Ssaidi@eecs.umich.edu if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 5453836Ssaidi@eecs.umich.edu if (inst->isLoad()) { 5463836Ssaidi@eecs.umich.edu // If this load is to the same block as an external snoop 5473836Ssaidi@eecs.umich.edu // invalidate that we've observed then the load needs to be 5483833Ssaidi@eecs.umich.edu // squashed as it could have newer data 5493836Ssaidi@eecs.umich.edu if (ld_inst->hitExternalSnoop()) { 5503836Ssaidi@eecs.umich.edu if (!memDepViolator || 5513836Ssaidi@eecs.umich.edu ld_inst->seqNum < memDepViolator->seqNum) { 5523836Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 5533836Ssaidi@eecs.umich.edu "and [sn:%lli] at address %#x\n", 5543836Ssaidi@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5553836Ssaidi@eecs.umich.edu memDepViolator = ld_inst; 5563836Ssaidi@eecs.umich.edu 5573836Ssaidi@eecs.umich.edu ++lsqMemOrderViolation; 5583836Ssaidi@eecs.umich.edu 5593836Ssaidi@eecs.umich.edu return new GenericISA::M5PanicFault( 5603836Ssaidi@eecs.umich.edu "Detected fault with inst [sn:%lli] and " 5613836Ssaidi@eecs.umich.edu "[sn:%lli] at address %#x\n", 5623836Ssaidi@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5633881Ssaidi@eecs.umich.edu } 5643836Ssaidi@eecs.umich.edu } 5653836Ssaidi@eecs.umich.edu 5663836Ssaidi@eecs.umich.edu // Otherwise, mark the load has a possible load violation 5673836Ssaidi@eecs.umich.edu // and if we see a snoop before it's commited, we need to squash 5683836Ssaidi@eecs.umich.edu ld_inst->possibleLoadViolation(true); 5693881Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x" 5703836Ssaidi@eecs.umich.edu " between instructions [sn:%lli] and [sn:%lli]\n", 5713836Ssaidi@eecs.umich.edu inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 5723836Ssaidi@eecs.umich.edu } else { 5733836Ssaidi@eecs.umich.edu // A load/store incorrectly passed this store. 5743836Ssaidi@eecs.umich.edu // Check if we already have a violator, or if it's newer 5753836Ssaidi@eecs.umich.edu // squash and refetch. 5763833Ssaidi@eecs.umich.edu if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 5773833Ssaidi@eecs.umich.edu break; 5783833Ssaidi@eecs.umich.edu 5793833Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 5803833Ssaidi@eecs.umich.edu "[sn:%lli] at address %#x\n", 5813833Ssaidi@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5823833Ssaidi@eecs.umich.edu memDepViolator = ld_inst; 5833833Ssaidi@eecs.umich.edu 5843833Ssaidi@eecs.umich.edu ++lsqMemOrderViolation; 5853833Ssaidi@eecs.umich.edu 5863804Ssaidi@eecs.umich.edu return new GenericISA::M5PanicFault("Detected fault with " 5873832Ssaidi@eecs.umich.edu "inst [sn:%lli] and [sn:%lli] at address %#x\n", 5883832Ssaidi@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5893804Ssaidi@eecs.umich.edu } 5903804Ssaidi@eecs.umich.edu } 5913804Ssaidi@eecs.umich.edu 5923833Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 5933833Ssaidi@eecs.umich.edu } 5943804Ssaidi@eecs.umich.edu return NoFault; 5953804Ssaidi@eecs.umich.edu} 5963804Ssaidi@eecs.umich.edu 5973804Ssaidi@eecs.umich.edu 5983804Ssaidi@eecs.umich.edu 5993804Ssaidi@eecs.umich.edu 6003804Ssaidi@eecs.umich.edutemplate <class Impl> 6013804Ssaidi@eecs.umich.eduFault 6023804Ssaidi@eecs.umich.eduLSQUnit<Impl>::executeLoad(DynInstPtr &inst) 6033833Ssaidi@eecs.umich.edu{ 6043804Ssaidi@eecs.umich.edu using namespace TheISA; 6053804Ssaidi@eecs.umich.edu // Execute a specific load. 6063823Ssaidi@eecs.umich.edu Fault load_fault = NoFault; 6073804Ssaidi@eecs.umich.edu 6083804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 6093804Ssaidi@eecs.umich.edu inst->pcState(), inst->seqNum); 6103804Ssaidi@eecs.umich.edu 6113833Ssaidi@eecs.umich.edu assert(!inst->isSquashed()); 6123804Ssaidi@eecs.umich.edu 6133833Ssaidi@eecs.umich.edu load_fault = inst->initiateAcc(); 6143804Ssaidi@eecs.umich.edu 6153804Ssaidi@eecs.umich.edu if (inst->isTranslationDelayed() && 6163804Ssaidi@eecs.umich.edu load_fault == NoFault) 6173804Ssaidi@eecs.umich.edu return load_fault; 6183804Ssaidi@eecs.umich.edu 6193804Ssaidi@eecs.umich.edu // If the instruction faulted or predicated false, then we need to send it 6203804Ssaidi@eecs.umich.edu // along to commit without the instruction completing. 6213804Ssaidi@eecs.umich.edu if (load_fault != NoFault || !inst->readPredicate()) { 6223804Ssaidi@eecs.umich.edu // Send this instruction to commit, also make sure iew stage 6233804Ssaidi@eecs.umich.edu // realizes there is activity. 6243804Ssaidi@eecs.umich.edu // Mark it as executed unless it is an uncached load that 6253804Ssaidi@eecs.umich.edu // needs to hit the head of commit. 6263804Ssaidi@eecs.umich.edu if (!inst->readPredicate()) 6273804Ssaidi@eecs.umich.edu inst->forwardOldRegs(); 6283902Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 6293902Ssaidi@eecs.umich.edu inst->seqNum, 6303902Ssaidi@eecs.umich.edu (load_fault != NoFault ? "fault" : "predication")); 6313902Ssaidi@eecs.umich.edu if (!(inst->hasRequest() && inst->uncacheable()) || 6323902Ssaidi@eecs.umich.edu inst->isAtCommit()) { 6333902Ssaidi@eecs.umich.edu inst->setExecuted(); 6343804Ssaidi@eecs.umich.edu } 6353804Ssaidi@eecs.umich.edu iewStage->instToCommit(inst); 6363804Ssaidi@eecs.umich.edu iewStage->activityThisCycle(); 6373804Ssaidi@eecs.umich.edu } else if (!loadBlocked()) { 6383804Ssaidi@eecs.umich.edu assert(inst->effAddrValid()); 6393804Ssaidi@eecs.umich.edu int load_idx = inst->lqIdx; 6403804Ssaidi@eecs.umich.edu incrLdIdx(load_idx); 6413804Ssaidi@eecs.umich.edu 6423856Ssaidi@eecs.umich.edu if (checkLoads) 6433856Ssaidi@eecs.umich.edu return checkViolations(load_idx, inst); 6443856Ssaidi@eecs.umich.edu } 6453804Ssaidi@eecs.umich.edu 6463804Ssaidi@eecs.umich.edu return load_fault; 6473804Ssaidi@eecs.umich.edu} 6483804Ssaidi@eecs.umich.edu 6493824Ssaidi@eecs.umich.edutemplate <class Impl> 6503824Ssaidi@eecs.umich.eduFault 6513823Ssaidi@eecs.umich.eduLSQUnit<Impl>::executeStore(DynInstPtr &store_inst) 6523804Ssaidi@eecs.umich.edu{ 6533804Ssaidi@eecs.umich.edu using namespace TheISA; 6543804Ssaidi@eecs.umich.edu // Make sure that a store exists. 6553804Ssaidi@eecs.umich.edu assert(stores != 0); 6563824Ssaidi@eecs.umich.edu 6573824Ssaidi@eecs.umich.edu int store_idx = store_inst->sqIdx; 6583825Ssaidi@eecs.umich.edu 6593825Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 6603823Ssaidi@eecs.umich.edu store_inst->pcState(), store_inst->seqNum); 6613823Ssaidi@eecs.umich.edu 6623823Ssaidi@eecs.umich.edu assert(!store_inst->isSquashed()); 6633804Ssaidi@eecs.umich.edu 6643804Ssaidi@eecs.umich.edu // Check the recently completed loads to see if any match this store's 6653826Ssaidi@eecs.umich.edu // address. If so, then we have a memory ordering violation. 6663826Ssaidi@eecs.umich.edu int load_idx = store_inst->lqIdx; 6673826Ssaidi@eecs.umich.edu 6683826Ssaidi@eecs.umich.edu Fault store_fault = store_inst->initiateAcc(); 6693826Ssaidi@eecs.umich.edu 6703826Ssaidi@eecs.umich.edu if (store_inst->isTranslationDelayed() && 6713826Ssaidi@eecs.umich.edu store_fault == NoFault) 6723826Ssaidi@eecs.umich.edu return store_fault; 6733826Ssaidi@eecs.umich.edu 6743826Ssaidi@eecs.umich.edu if (!store_inst->readPredicate()) 6753826Ssaidi@eecs.umich.edu store_inst->forwardOldRegs(); 6763826Ssaidi@eecs.umich.edu 6773826Ssaidi@eecs.umich.edu if (storeQueue[store_idx].size == 0) { 6783826Ssaidi@eecs.umich.edu DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 6793826Ssaidi@eecs.umich.edu store_inst->pcState(), store_inst->seqNum); 6803826Ssaidi@eecs.umich.edu 6813833Ssaidi@eecs.umich.edu return store_fault; 6823804Ssaidi@eecs.umich.edu } else if (!store_inst->readPredicate()) { 6833804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 6843804Ssaidi@eecs.umich.edu store_inst->seqNum); 6853804Ssaidi@eecs.umich.edu return store_fault; 6863804Ssaidi@eecs.umich.edu } 6873836Ssaidi@eecs.umich.edu 6883804Ssaidi@eecs.umich.edu assert(store_fault == NoFault); 6893804Ssaidi@eecs.umich.edu 6903804Ssaidi@eecs.umich.edu if (store_inst->isStoreConditional()) { 6913836Ssaidi@eecs.umich.edu // Store conditionals need to set themselves as able to 6923804Ssaidi@eecs.umich.edu // writeback if we haven't had a fault by here. 6933804Ssaidi@eecs.umich.edu storeQueue[store_idx].canWB = true; 6943804Ssaidi@eecs.umich.edu 6953804Ssaidi@eecs.umich.edu ++storesToWB; 6963811Ssaidi@eecs.umich.edu } 6973804Ssaidi@eecs.umich.edu 6983804Ssaidi@eecs.umich.edu return checkViolations(load_idx, store_inst); 6993804Ssaidi@eecs.umich.edu 7003804Ssaidi@eecs.umich.edu} 7013804Ssaidi@eecs.umich.edu 7023804Ssaidi@eecs.umich.edutemplate <class Impl> 7033804Ssaidi@eecs.umich.eduvoid 7043804Ssaidi@eecs.umich.eduLSQUnit<Impl>::commitLoad() 7053804Ssaidi@eecs.umich.edu{ 7063804Ssaidi@eecs.umich.edu assert(loadQueue[loadHead]); 7073804Ssaidi@eecs.umich.edu 7083804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 7093804Ssaidi@eecs.umich.edu loadQueue[loadHead]->pcState()); 7103804Ssaidi@eecs.umich.edu 7113804Ssaidi@eecs.umich.edu loadQueue[loadHead] = NULL; 7123804Ssaidi@eecs.umich.edu 7133804Ssaidi@eecs.umich.edu incrLdIdx(loadHead); 7143804Ssaidi@eecs.umich.edu 7153804Ssaidi@eecs.umich.edu --loads; 7163804Ssaidi@eecs.umich.edu} 7173804Ssaidi@eecs.umich.edu 7183804Ssaidi@eecs.umich.edutemplate <class Impl> 7193804Ssaidi@eecs.umich.eduvoid 7203804Ssaidi@eecs.umich.eduLSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 7213804Ssaidi@eecs.umich.edu{ 7223804Ssaidi@eecs.umich.edu assert(loads == 0 || loadQueue[loadHead]); 7233804Ssaidi@eecs.umich.edu 7243836Ssaidi@eecs.umich.edu while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) { 7253836Ssaidi@eecs.umich.edu commitLoad(); 7263881Ssaidi@eecs.umich.edu } 7273881Ssaidi@eecs.umich.edu} 7283881Ssaidi@eecs.umich.edu 7293881Ssaidi@eecs.umich.edutemplate <class Impl> 7303881Ssaidi@eecs.umich.eduvoid 7313836Ssaidi@eecs.umich.eduLSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 7323836Ssaidi@eecs.umich.edu{ 7333836Ssaidi@eecs.umich.edu assert(stores == 0 || storeQueue[storeHead].inst); 7343836Ssaidi@eecs.umich.edu 7353836Ssaidi@eecs.umich.edu int store_idx = storeHead; 7363836Ssaidi@eecs.umich.edu 7373836Ssaidi@eecs.umich.edu while (store_idx != storeTail) { 7383836Ssaidi@eecs.umich.edu assert(storeQueue[store_idx].inst); 7393881Ssaidi@eecs.umich.edu // Mark any stores that are now committed and have not yet 7403826Ssaidi@eecs.umich.edu // been marked as able to write back. 7413836Ssaidi@eecs.umich.edu if (!storeQueue[store_idx].canWB) { 7423836Ssaidi@eecs.umich.edu if (storeQueue[store_idx].inst->seqNum > youngest_inst) { 7433804Ssaidi@eecs.umich.edu break; 7443806Ssaidi@eecs.umich.edu } 7453804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Marking store as able to write back, PC " 7463806Ssaidi@eecs.umich.edu "%s [sn:%lli]\n", 7473806Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->pcState(), 7483806Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->seqNum); 7493806Ssaidi@eecs.umich.edu 7503806Ssaidi@eecs.umich.edu storeQueue[store_idx].canWB = true; 7513824Ssaidi@eecs.umich.edu 7523824Ssaidi@eecs.umich.edu ++storesToWB; 7533824Ssaidi@eecs.umich.edu } 7543824Ssaidi@eecs.umich.edu 7553824Ssaidi@eecs.umich.edu incrStIdx(store_idx); 7563824Ssaidi@eecs.umich.edu } 7573824Ssaidi@eecs.umich.edu} 7583881Ssaidi@eecs.umich.edu 7593824Ssaidi@eecs.umich.edutemplate <class Impl> 7603824Ssaidi@eecs.umich.eduvoid 7613824Ssaidi@eecs.umich.eduLSQUnit<Impl>::writebackPendingStore() 7623824Ssaidi@eecs.umich.edu{ 7633824Ssaidi@eecs.umich.edu if (hasPendingPkt) { 7643825Ssaidi@eecs.umich.edu assert(pendingPkt != NULL); 7653825Ssaidi@eecs.umich.edu 7663825Ssaidi@eecs.umich.edu // If the cache is blocked, this will store the packet for retry. 7673825Ssaidi@eecs.umich.edu if (sendStore(pendingPkt)) { 7683825Ssaidi@eecs.umich.edu storePostSend(pendingPkt); 7693825Ssaidi@eecs.umich.edu } 7703825Ssaidi@eecs.umich.edu pendingPkt = NULL; 7713825Ssaidi@eecs.umich.edu hasPendingPkt = false; 7723825Ssaidi@eecs.umich.edu } 7733825Ssaidi@eecs.umich.edu} 7743825Ssaidi@eecs.umich.edu 7753825Ssaidi@eecs.umich.edutemplate <class Impl> 7763825Ssaidi@eecs.umich.eduvoid 7773824Ssaidi@eecs.umich.eduLSQUnit<Impl>::writebackStores() 7783804Ssaidi@eecs.umich.edu{ 7793811Ssaidi@eecs.umich.edu // First writeback the second packet from any split store that didn't 7803806Ssaidi@eecs.umich.edu // complete last cycle because there weren't enough cache ports available. 7813806Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc) { 7823806Ssaidi@eecs.umich.edu writebackPendingStore(); 7833804Ssaidi@eecs.umich.edu } 7843804Ssaidi@eecs.umich.edu 7853806Ssaidi@eecs.umich.edu while (storesToWB > 0 && 7863806Ssaidi@eecs.umich.edu storeWBIdx != storeTail && 7873806Ssaidi@eecs.umich.edu storeQueue[storeWBIdx].inst && 7883823Ssaidi@eecs.umich.edu storeQueue[storeWBIdx].canWB && 7893823Ssaidi@eecs.umich.edu ((!needsTSO) || (!storeInFlight)) && 7903833Ssaidi@eecs.umich.edu usedPorts < cachePorts) { 7913833Ssaidi@eecs.umich.edu 7923823Ssaidi@eecs.umich.edu if (isStoreBlocked || lsq->cacheBlocked()) { 7933823Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 7943823Ssaidi@eecs.umich.edu " is blocked!\n"); 7953823Ssaidi@eecs.umich.edu break; 7963823Ssaidi@eecs.umich.edu } 7973823Ssaidi@eecs.umich.edu 7983823Ssaidi@eecs.umich.edu // Store didn't write any data so no need to write it back to 7993823Ssaidi@eecs.umich.edu // memory. 8003823Ssaidi@eecs.umich.edu if (storeQueue[storeWBIdx].size == 0) { 8013823Ssaidi@eecs.umich.edu completeStore(storeWBIdx); 8023823Ssaidi@eecs.umich.edu 8033823Ssaidi@eecs.umich.edu incrStIdx(storeWBIdx); 8043823Ssaidi@eecs.umich.edu 8053823Ssaidi@eecs.umich.edu continue; 8063823Ssaidi@eecs.umich.edu } 8073823Ssaidi@eecs.umich.edu 8083823Ssaidi@eecs.umich.edu ++usedPorts; 8093823Ssaidi@eecs.umich.edu 8103823Ssaidi@eecs.umich.edu if (storeQueue[storeWBIdx].inst->isDataPrefetch()) { 8113823Ssaidi@eecs.umich.edu incrStIdx(storeWBIdx); 8123823Ssaidi@eecs.umich.edu 8133824Ssaidi@eecs.umich.edu continue; 8143824Ssaidi@eecs.umich.edu } 8153824Ssaidi@eecs.umich.edu 8163824Ssaidi@eecs.umich.edu assert(storeQueue[storeWBIdx].req); 8173823Ssaidi@eecs.umich.edu assert(!storeQueue[storeWBIdx].committed); 8183823Ssaidi@eecs.umich.edu 8193823Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) { 8203823Ssaidi@eecs.umich.edu assert(storeQueue[storeWBIdx].sreqLow); 8213823Ssaidi@eecs.umich.edu assert(storeQueue[storeWBIdx].sreqHigh); 8223823Ssaidi@eecs.umich.edu } 8233823Ssaidi@eecs.umich.edu 8243823Ssaidi@eecs.umich.edu DynInstPtr inst = storeQueue[storeWBIdx].inst; 8253823Ssaidi@eecs.umich.edu 8263823Ssaidi@eecs.umich.edu Request *req = storeQueue[storeWBIdx].req; 8273823Ssaidi@eecs.umich.edu RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow; 8283823Ssaidi@eecs.umich.edu RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh; 8293823Ssaidi@eecs.umich.edu 8303823Ssaidi@eecs.umich.edu storeQueue[storeWBIdx].committed = true; 8313823Ssaidi@eecs.umich.edu 8323823Ssaidi@eecs.umich.edu assert(!inst->memData); 8333823Ssaidi@eecs.umich.edu inst->memData = new uint8_t[req->getSize()]; 8343823Ssaidi@eecs.umich.edu 8353823Ssaidi@eecs.umich.edu if (storeQueue[storeWBIdx].isAllZeros) 8363823Ssaidi@eecs.umich.edu memset(inst->memData, 0, req->getSize()); 8373823Ssaidi@eecs.umich.edu else 8383823Ssaidi@eecs.umich.edu memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize()); 8393823Ssaidi@eecs.umich.edu 8403823Ssaidi@eecs.umich.edu MemCmd command = 8413823Ssaidi@eecs.umich.edu req->isSwap() ? MemCmd::SwapReq : 8423823Ssaidi@eecs.umich.edu (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq); 8433823Ssaidi@eecs.umich.edu PacketPtr data_pkt; 8443823Ssaidi@eecs.umich.edu PacketPtr snd_data_pkt = NULL; 8453823Ssaidi@eecs.umich.edu 8463823Ssaidi@eecs.umich.edu LSQSenderState *state = new LSQSenderState; 8473823Ssaidi@eecs.umich.edu state->isLoad = false; 8483823Ssaidi@eecs.umich.edu state->idx = storeWBIdx; 8493823Ssaidi@eecs.umich.edu state->inst = inst; 8503823Ssaidi@eecs.umich.edu 8513823Ssaidi@eecs.umich.edu if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) { 8523823Ssaidi@eecs.umich.edu 8533823Ssaidi@eecs.umich.edu // Build a single data packet if the store isn't split. 8543823Ssaidi@eecs.umich.edu data_pkt = new Packet(req, command); 8553823Ssaidi@eecs.umich.edu data_pkt->dataStatic(inst->memData); 8563823Ssaidi@eecs.umich.edu data_pkt->senderState = state; 8573823Ssaidi@eecs.umich.edu } else { 8583823Ssaidi@eecs.umich.edu // Create two packets if the store is split in two. 8593823Ssaidi@eecs.umich.edu data_pkt = new Packet(sreqLow, command); 8603823Ssaidi@eecs.umich.edu snd_data_pkt = new Packet(sreqHigh, command); 8613823Ssaidi@eecs.umich.edu 8623823Ssaidi@eecs.umich.edu data_pkt->dataStatic(inst->memData); 8633823Ssaidi@eecs.umich.edu snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize()); 8643823Ssaidi@eecs.umich.edu 8653826Ssaidi@eecs.umich.edu data_pkt->senderState = state; 8663826Ssaidi@eecs.umich.edu snd_data_pkt->senderState = state; 8673826Ssaidi@eecs.umich.edu 8683826Ssaidi@eecs.umich.edu state->isSplit = true; 8693823Ssaidi@eecs.umich.edu state->outstanding = 2; 8703823Ssaidi@eecs.umich.edu 8713823Ssaidi@eecs.umich.edu // Can delete the main request now. 8723823Ssaidi@eecs.umich.edu delete req; 8733826Ssaidi@eecs.umich.edu req = sreqLow; 8743826Ssaidi@eecs.umich.edu } 8753833Ssaidi@eecs.umich.edu 8763833Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 8773833Ssaidi@eecs.umich.edu "to Addr:%#x, data:%#x [sn:%lli]\n", 8783833Ssaidi@eecs.umich.edu storeWBIdx, inst->pcState(), 8793906Ssaidi@eecs.umich.edu req->getPaddr(), (int)*(inst->memData), 8803906Ssaidi@eecs.umich.edu inst->seqNum); 8813906Ssaidi@eecs.umich.edu 8823826Ssaidi@eecs.umich.edu // @todo: Remove this SC hack once the memory system handles it. 8833826Ssaidi@eecs.umich.edu if (inst->isStoreConditional()) { 8843826Ssaidi@eecs.umich.edu assert(!storeQueue[storeWBIdx].isSplit); 8853826Ssaidi@eecs.umich.edu // Disable recording the result temporarily. Writing to 8863826Ssaidi@eecs.umich.edu // misc regs normally updates the result, but this is not 8873826Ssaidi@eecs.umich.edu // the desired behavior when handling store conditionals. 8883826Ssaidi@eecs.umich.edu inst->recordResult(false); 8893823Ssaidi@eecs.umich.edu bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask); 8903823Ssaidi@eecs.umich.edu inst->recordResult(true); 8913833Ssaidi@eecs.umich.edu 8923833Ssaidi@eecs.umich.edu if (!success) { 8933833Ssaidi@eecs.umich.edu // Instantly complete this store. 8943833Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 8953906Ssaidi@eecs.umich.edu "Instantly completing it.\n", 8963906Ssaidi@eecs.umich.edu inst->seqNum); 8973906Ssaidi@eecs.umich.edu WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this); 8983906Ssaidi@eecs.umich.edu cpu->schedule(wb, curTick() + 1); 8993906Ssaidi@eecs.umich.edu if (cpu->checker) { 9003906Ssaidi@eecs.umich.edu // Make sure to set the LLSC data for verification 9013826Ssaidi@eecs.umich.edu // if checker is loaded 9023826Ssaidi@eecs.umich.edu inst->reqToVerify->setExtraData(0); 9033826Ssaidi@eecs.umich.edu inst->completeAcc(data_pkt); 9043823Ssaidi@eecs.umich.edu } 9053823Ssaidi@eecs.umich.edu completeStore(storeWBIdx); 9063823Ssaidi@eecs.umich.edu incrStIdx(storeWBIdx); 9073823Ssaidi@eecs.umich.edu continue; 9083823Ssaidi@eecs.umich.edu } 9093823Ssaidi@eecs.umich.edu } else { 9103823Ssaidi@eecs.umich.edu // Non-store conditionals do not need a writeback. 9113833Ssaidi@eecs.umich.edu state->noWB = true; 9123833Ssaidi@eecs.umich.edu } 9133833Ssaidi@eecs.umich.edu 9143833Ssaidi@eecs.umich.edu bool split = 9153833Ssaidi@eecs.umich.edu TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit; 9163833Ssaidi@eecs.umich.edu 9173833Ssaidi@eecs.umich.edu ThreadContext *thread = cpu->tcBase(lsqID); 9183833Ssaidi@eecs.umich.edu 9193833Ssaidi@eecs.umich.edu if (req->isMmappedIpr()) { 9203833Ssaidi@eecs.umich.edu assert(!inst->isStoreConditional()); 9213833Ssaidi@eecs.umich.edu TheISA::handleIprWrite(thread, data_pkt); 9223833Ssaidi@eecs.umich.edu delete data_pkt; 9233833Ssaidi@eecs.umich.edu if (split) { 9243833Ssaidi@eecs.umich.edu assert(snd_data_pkt->req->isMmappedIpr()); 9253833Ssaidi@eecs.umich.edu TheISA::handleIprWrite(thread, snd_data_pkt); 9263833Ssaidi@eecs.umich.edu delete snd_data_pkt; 9273833Ssaidi@eecs.umich.edu delete sreqLow; 9283833Ssaidi@eecs.umich.edu delete sreqHigh; 9293833Ssaidi@eecs.umich.edu } 9303833Ssaidi@eecs.umich.edu delete state; 9313833Ssaidi@eecs.umich.edu delete req; 9323833Ssaidi@eecs.umich.edu completeStore(storeWBIdx); 9333833Ssaidi@eecs.umich.edu incrStIdx(storeWBIdx); 9343833Ssaidi@eecs.umich.edu } else if (!sendStore(data_pkt)) { 9353833Ssaidi@eecs.umich.edu DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will" 9363833Ssaidi@eecs.umich.edu "retry later\n", 9373833Ssaidi@eecs.umich.edu inst->seqNum); 9383833Ssaidi@eecs.umich.edu 9393833Ssaidi@eecs.umich.edu // Need to store the second packet, if split. 9403833Ssaidi@eecs.umich.edu if (split) { 9413899Ssaidi@eecs.umich.edu state->pktToSend = true; 9423899Ssaidi@eecs.umich.edu state->pendingPacket = snd_data_pkt; 9433899Ssaidi@eecs.umich.edu } 9443899Ssaidi@eecs.umich.edu } else { 9453899Ssaidi@eecs.umich.edu 9463899Ssaidi@eecs.umich.edu // If split, try to send the second packet too 9473899Ssaidi@eecs.umich.edu if (split) { 9483899Ssaidi@eecs.umich.edu assert(snd_data_pkt); 9493899Ssaidi@eecs.umich.edu 9503899Ssaidi@eecs.umich.edu // Ensure there are enough ports to use. 9513899Ssaidi@eecs.umich.edu if (usedPorts < cachePorts) { 9523899Ssaidi@eecs.umich.edu ++usedPorts; 9533899Ssaidi@eecs.umich.edu if (sendStore(snd_data_pkt)) { 9543899Ssaidi@eecs.umich.edu storePostSend(snd_data_pkt); 9553899Ssaidi@eecs.umich.edu } else { 9563899Ssaidi@eecs.umich.edu DPRINTF(IEW, "D-Cache became blocked when writing" 9573899Ssaidi@eecs.umich.edu " [sn:%lli] second packet, will retry later\n", 9583899Ssaidi@eecs.umich.edu inst->seqNum); 9593899Ssaidi@eecs.umich.edu } 9603899Ssaidi@eecs.umich.edu } else { 9613899Ssaidi@eecs.umich.edu 9623899Ssaidi@eecs.umich.edu // Store the packet for when there's free ports. 9633899Ssaidi@eecs.umich.edu assert(pendingPkt == NULL); 9643899Ssaidi@eecs.umich.edu pendingPkt = snd_data_pkt; 9653899Ssaidi@eecs.umich.edu hasPendingPkt = true; 9663899Ssaidi@eecs.umich.edu } 9673899Ssaidi@eecs.umich.edu } else { 9683899Ssaidi@eecs.umich.edu 9693899Ssaidi@eecs.umich.edu // Not a split store. 9703899Ssaidi@eecs.umich.edu storePostSend(data_pkt); 9713833Ssaidi@eecs.umich.edu } 9723823Ssaidi@eecs.umich.edu } 9733823Ssaidi@eecs.umich.edu } 9743823Ssaidi@eecs.umich.edu 9753823Ssaidi@eecs.umich.edu // Not sure this should set it to 0. 9763823Ssaidi@eecs.umich.edu usedPorts = 0; 9773823Ssaidi@eecs.umich.edu 9783823Ssaidi@eecs.umich.edu assert(stores >= 0 && storesToWB >= 0); 9793806Ssaidi@eecs.umich.edu} 9803806Ssaidi@eecs.umich.edu 9813806Ssaidi@eecs.umich.edu/*template <class Impl> 9823806Ssaidi@eecs.umich.eduvoid 9833806Ssaidi@eecs.umich.eduLSQUnit<Impl>::removeMSHR(InstSeqNum seqNum) 9843823Ssaidi@eecs.umich.edu{ 9853823Ssaidi@eecs.umich.edu list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(), 9863823Ssaidi@eecs.umich.edu mshrSeqNums.end(), 9873823Ssaidi@eecs.umich.edu seqNum); 9883826Ssaidi@eecs.umich.edu 9893826Ssaidi@eecs.umich.edu if (mshr_it != mshrSeqNums.end()) { 9903826Ssaidi@eecs.umich.edu mshrSeqNums.erase(mshr_it); 9913826Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size()); 9923826Ssaidi@eecs.umich.edu } 9933826Ssaidi@eecs.umich.edu}*/ 9943863Ssaidi@eecs.umich.edu 9953863Ssaidi@eecs.umich.edutemplate <class Impl> 9963863Ssaidi@eecs.umich.eduvoid 9973826Ssaidi@eecs.umich.eduLSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 9983826Ssaidi@eecs.umich.edu{ 9993825Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 10003823Ssaidi@eecs.umich.edu "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 10013823Ssaidi@eecs.umich.edu 10023823Ssaidi@eecs.umich.edu int load_idx = loadTail; 10033823Ssaidi@eecs.umich.edu decrLdIdx(load_idx); 10043823Ssaidi@eecs.umich.edu 10053823Ssaidi@eecs.umich.edu while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) { 10063823Ssaidi@eecs.umich.edu DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 10073823Ssaidi@eecs.umich.edu "[sn:%lli]\n", 10083823Ssaidi@eecs.umich.edu loadQueue[load_idx]->pcState(), 10093823Ssaidi@eecs.umich.edu loadQueue[load_idx]->seqNum); 10103823Ssaidi@eecs.umich.edu 10113823Ssaidi@eecs.umich.edu if (isStalled() && load_idx == stallingLoadIdx) { 10123823Ssaidi@eecs.umich.edu stalled = false; 10133823Ssaidi@eecs.umich.edu stallingStoreIsn = 0; 10143823Ssaidi@eecs.umich.edu stallingLoadIdx = 0; 10153823Ssaidi@eecs.umich.edu } 10163823Ssaidi@eecs.umich.edu 10173823Ssaidi@eecs.umich.edu // Clear the smart pointer to make sure it is decremented. 10183823Ssaidi@eecs.umich.edu loadQueue[load_idx]->setSquashed(); 10193824Ssaidi@eecs.umich.edu loadQueue[load_idx] = NULL; 10203825Ssaidi@eecs.umich.edu --loads; 10213824Ssaidi@eecs.umich.edu 10223824Ssaidi@eecs.umich.edu // Inefficient! 10233824Ssaidi@eecs.umich.edu loadTail = load_idx; 10243823Ssaidi@eecs.umich.edu 10253823Ssaidi@eecs.umich.edu decrLdIdx(load_idx); 10263823Ssaidi@eecs.umich.edu ++lsqSquashedLoads; 10273823Ssaidi@eecs.umich.edu } 10283823Ssaidi@eecs.umich.edu 10293823Ssaidi@eecs.umich.edu if (isLoadBlocked) { 10303823Ssaidi@eecs.umich.edu if (squashed_num < blockedLoadSeqNum) { 10313823Ssaidi@eecs.umich.edu isLoadBlocked = false; 10323823Ssaidi@eecs.umich.edu loadBlockedHandled = false; 10333823Ssaidi@eecs.umich.edu blockedLoadSeqNum = 0; 10343823Ssaidi@eecs.umich.edu } 10353823Ssaidi@eecs.umich.edu } 10363823Ssaidi@eecs.umich.edu 10373823Ssaidi@eecs.umich.edu if (memDepViolator && squashed_num < memDepViolator->seqNum) { 10383823Ssaidi@eecs.umich.edu memDepViolator = NULL; 10393823Ssaidi@eecs.umich.edu } 10403823Ssaidi@eecs.umich.edu 10413823Ssaidi@eecs.umich.edu int store_idx = storeTail; 10423823Ssaidi@eecs.umich.edu decrStIdx(store_idx); 10433823Ssaidi@eecs.umich.edu 10443823Ssaidi@eecs.umich.edu while (stores != 0 && 10453823Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->seqNum > squashed_num) { 10463823Ssaidi@eecs.umich.edu // Instructions marked as can WB are already committed. 10473823Ssaidi@eecs.umich.edu if (storeQueue[store_idx].canWB) { 10483823Ssaidi@eecs.umich.edu break; 10493823Ssaidi@eecs.umich.edu } 10503823Ssaidi@eecs.umich.edu 10513823Ssaidi@eecs.umich.edu DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 10523823Ssaidi@eecs.umich.edu "idx:%i [sn:%lli]\n", 10533823Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->pcState(), 10543823Ssaidi@eecs.umich.edu store_idx, storeQueue[store_idx].inst->seqNum); 10553823Ssaidi@eecs.umich.edu 10563823Ssaidi@eecs.umich.edu // I don't think this can happen. It should have been cleared 10573823Ssaidi@eecs.umich.edu // by the stalling load. 10583823Ssaidi@eecs.umich.edu if (isStalled() && 10593823Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 10603823Ssaidi@eecs.umich.edu panic("Is stalled should have been cleared by stalling load!\n"); 10613823Ssaidi@eecs.umich.edu stalled = false; 10623823Ssaidi@eecs.umich.edu stallingStoreIsn = 0; 10633823Ssaidi@eecs.umich.edu } 10643823Ssaidi@eecs.umich.edu 10653823Ssaidi@eecs.umich.edu // Clear the smart pointer to make sure it is decremented. 10663823Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->setSquashed(); 10673823Ssaidi@eecs.umich.edu storeQueue[store_idx].inst = NULL; 10683823Ssaidi@eecs.umich.edu storeQueue[store_idx].canWB = 0; 10693823Ssaidi@eecs.umich.edu 10703823Ssaidi@eecs.umich.edu // Must delete request now that it wasn't handed off to 10713823Ssaidi@eecs.umich.edu // memory. This is quite ugly. @todo: Figure out the proper 10723825Ssaidi@eecs.umich.edu // place to really handle request deletes. 10733825Ssaidi@eecs.umich.edu delete storeQueue[store_idx].req; 10743825Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) { 10753825Ssaidi@eecs.umich.edu delete storeQueue[store_idx].sreqLow; 10763823Ssaidi@eecs.umich.edu delete storeQueue[store_idx].sreqHigh; 10773823Ssaidi@eecs.umich.edu 10783823Ssaidi@eecs.umich.edu storeQueue[store_idx].sreqLow = NULL; 10793823Ssaidi@eecs.umich.edu storeQueue[store_idx].sreqHigh = NULL; 10803826Ssaidi@eecs.umich.edu } 10813826Ssaidi@eecs.umich.edu 10823906Ssaidi@eecs.umich.edu storeQueue[store_idx].req = NULL; 10833906Ssaidi@eecs.umich.edu --stores; 10843906Ssaidi@eecs.umich.edu 10853826Ssaidi@eecs.umich.edu // Inefficient! 10863826Ssaidi@eecs.umich.edu storeTail = store_idx; 10873826Ssaidi@eecs.umich.edu 10883826Ssaidi@eecs.umich.edu decrStIdx(store_idx); 10893826Ssaidi@eecs.umich.edu ++lsqSquashedStores; 10903826Ssaidi@eecs.umich.edu } 10913826Ssaidi@eecs.umich.edu} 10923826Ssaidi@eecs.umich.edu 10933826Ssaidi@eecs.umich.edutemplate <class Impl> 10943826Ssaidi@eecs.umich.eduvoid 10953826Ssaidi@eecs.umich.eduLSQUnit<Impl>::storePostSend(PacketPtr pkt) 10963826Ssaidi@eecs.umich.edu{ 10973826Ssaidi@eecs.umich.edu if (isStalled() && 10983826Ssaidi@eecs.umich.edu storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) { 10993826Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 11003826Ssaidi@eecs.umich.edu "load idx:%i\n", 11013826Ssaidi@eecs.umich.edu stallingStoreIsn, stallingLoadIdx); 11023826Ssaidi@eecs.umich.edu stalled = false; 11033826Ssaidi@eecs.umich.edu stallingStoreIsn = 0; 11043826Ssaidi@eecs.umich.edu iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 11053826Ssaidi@eecs.umich.edu } 11063826Ssaidi@eecs.umich.edu 11073826Ssaidi@eecs.umich.edu if (!storeQueue[storeWBIdx].inst->isStoreConditional()) { 11083826Ssaidi@eecs.umich.edu // The store is basically completed at this time. This 11093826Ssaidi@eecs.umich.edu // only works so long as the checker doesn't try to 11103826Ssaidi@eecs.umich.edu // verify the value in memory for stores. 11113826Ssaidi@eecs.umich.edu storeQueue[storeWBIdx].inst->setCompleted(); 11123826Ssaidi@eecs.umich.edu 11133826Ssaidi@eecs.umich.edu if (cpu->checker) { 11143826Ssaidi@eecs.umich.edu cpu->checker->verify(storeQueue[storeWBIdx].inst); 11153826Ssaidi@eecs.umich.edu } 11163826Ssaidi@eecs.umich.edu } 11173826Ssaidi@eecs.umich.edu 11183826Ssaidi@eecs.umich.edu if (needsTSO) { 11193863Ssaidi@eecs.umich.edu storeInFlight = true; 11203863Ssaidi@eecs.umich.edu } 11213863Ssaidi@eecs.umich.edu 11223863Ssaidi@eecs.umich.edu incrStIdx(storeWBIdx); 11233863Ssaidi@eecs.umich.edu} 11243863Ssaidi@eecs.umich.edu 11253863Ssaidi@eecs.umich.edutemplate <class Impl> 11263863Ssaidi@eecs.umich.eduvoid 11273863Ssaidi@eecs.umich.eduLSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt) 11283863Ssaidi@eecs.umich.edu{ 11293863Ssaidi@eecs.umich.edu iewStage->wakeCPU(); 11303863Ssaidi@eecs.umich.edu 11313863Ssaidi@eecs.umich.edu // Squashed instructions do not need to complete their access. 11323863Ssaidi@eecs.umich.edu if (inst->isSquashed()) { 11333863Ssaidi@eecs.umich.edu iewStage->decrWb(inst->seqNum); 11343863Ssaidi@eecs.umich.edu assert(!inst->isStore()); 11353863Ssaidi@eecs.umich.edu ++lsqIgnoredResponses; 11363863Ssaidi@eecs.umich.edu return; 11373863Ssaidi@eecs.umich.edu } 11383863Ssaidi@eecs.umich.edu 11393863Ssaidi@eecs.umich.edu if (!inst->isExecuted()) { 11403863Ssaidi@eecs.umich.edu inst->setExecuted(); 11413863Ssaidi@eecs.umich.edu 11423863Ssaidi@eecs.umich.edu // Complete access to copy data to proper place. 11433863Ssaidi@eecs.umich.edu inst->completeAcc(pkt); 11443863Ssaidi@eecs.umich.edu } 11453863Ssaidi@eecs.umich.edu 11463863Ssaidi@eecs.umich.edu // Need to insert instruction into queue to commit 11473863Ssaidi@eecs.umich.edu iewStage->instToCommit(inst); 11483863Ssaidi@eecs.umich.edu 11493863Ssaidi@eecs.umich.edu iewStage->activityThisCycle(); 11503863Ssaidi@eecs.umich.edu 11513863Ssaidi@eecs.umich.edu // see if this load changed the PC 11523863Ssaidi@eecs.umich.edu iewStage->checkMisprediction(inst); 11533863Ssaidi@eecs.umich.edu} 11543823Ssaidi@eecs.umich.edu 11553823Ssaidi@eecs.umich.edutemplate <class Impl> 11563906Ssaidi@eecs.umich.eduvoid 11573906Ssaidi@eecs.umich.eduLSQUnit<Impl>::completeStore(int store_idx) 11583906Ssaidi@eecs.umich.edu{ 11593826Ssaidi@eecs.umich.edu assert(storeQueue[store_idx].inst); 11603826Ssaidi@eecs.umich.edu storeQueue[store_idx].completed = true; 11613826Ssaidi@eecs.umich.edu --storesToWB; 11623823Ssaidi@eecs.umich.edu // A bit conservative because a store completion may not free up entries, 11633823Ssaidi@eecs.umich.edu // but hopefully avoids two store completions in one cycle from making 11643823Ssaidi@eecs.umich.edu // the CPU tick twice. 11653823Ssaidi@eecs.umich.edu cpu->wakeCPU(); 11663823Ssaidi@eecs.umich.edu cpu->activityThisCycle(); 11673823Ssaidi@eecs.umich.edu 11683823Ssaidi@eecs.umich.edu if (store_idx == storeHead) { 11693863Ssaidi@eecs.umich.edu do { 11703863Ssaidi@eecs.umich.edu incrStIdx(storeHead); 11713863Ssaidi@eecs.umich.edu 11723863Ssaidi@eecs.umich.edu --stores; 11733863Ssaidi@eecs.umich.edu } while (storeQueue[storeHead].completed && 11743863Ssaidi@eecs.umich.edu storeHead != storeTail); 11753863Ssaidi@eecs.umich.edu 11763863Ssaidi@eecs.umich.edu iewStage->updateLSQNextCycle = true; 11773863Ssaidi@eecs.umich.edu } 11783863Ssaidi@eecs.umich.edu 11793863Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 11803863Ssaidi@eecs.umich.edu "idx:%i\n", 11813863Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->seqNum, store_idx, storeHead); 11823863Ssaidi@eecs.umich.edu 11833863Ssaidi@eecs.umich.edu#if TRACING_ON 11843863Ssaidi@eecs.umich.edu if (DTRACE(O3PipeView)) { 11853863Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->storeTick = 11863863Ssaidi@eecs.umich.edu curTick() - storeQueue[store_idx].inst->fetchTick; 11873863Ssaidi@eecs.umich.edu } 11883863Ssaidi@eecs.umich.edu#endif 11893863Ssaidi@eecs.umich.edu 11903863Ssaidi@eecs.umich.edu if (isStalled() && 11913863Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 11923863Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 11933863Ssaidi@eecs.umich.edu "load idx:%i\n", 11943863Ssaidi@eecs.umich.edu stallingStoreIsn, stallingLoadIdx); 11953863Ssaidi@eecs.umich.edu stalled = false; 11963863Ssaidi@eecs.umich.edu stallingStoreIsn = 0; 11973863Ssaidi@eecs.umich.edu iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 11983863Ssaidi@eecs.umich.edu } 11993863Ssaidi@eecs.umich.edu 12003863Ssaidi@eecs.umich.edu storeQueue[store_idx].inst->setCompleted(); 12013863Ssaidi@eecs.umich.edu 12023863Ssaidi@eecs.umich.edu if (needsTSO) { 12033823Ssaidi@eecs.umich.edu storeInFlight = false; 12043823Ssaidi@eecs.umich.edu } 12053823Ssaidi@eecs.umich.edu 12063823Ssaidi@eecs.umich.edu // Tell the checker we've completed this instruction. Some stores 12073823Ssaidi@eecs.umich.edu // may get reported twice to the checker, but the checker can 12083823Ssaidi@eecs.umich.edu // handle that case. 12093823Ssaidi@eecs.umich.edu if (cpu->checker) { 12103806Ssaidi@eecs.umich.edu cpu->checker->verify(storeQueue[store_idx].inst); 12113806Ssaidi@eecs.umich.edu } 12123804Ssaidi@eecs.umich.edu} 12133804Ssaidi@eecs.umich.edu 12143804Ssaidi@eecs.umich.edutemplate <class Impl> 12153804Ssaidi@eecs.umich.edubool 12163804Ssaidi@eecs.umich.eduLSQUnit<Impl>::sendStore(PacketPtr data_pkt) 12173804Ssaidi@eecs.umich.edu{ 12183804Ssaidi@eecs.umich.edu if (!dcachePort->sendTimingReq(data_pkt)) { 12193804Ssaidi@eecs.umich.edu // Need to handle becoming blocked on a store. 12203804Ssaidi@eecs.umich.edu isStoreBlocked = true; 12213804Ssaidi@eecs.umich.edu ++lsqCacheBlocked; 12223804Ssaidi@eecs.umich.edu assert(retryPkt == NULL); 12233804Ssaidi@eecs.umich.edu retryPkt = data_pkt; 12243804Ssaidi@eecs.umich.edu lsq->setRetryTid(lsqID); 12253804Ssaidi@eecs.umich.edu return false; 12263804Ssaidi@eecs.umich.edu } 12273804Ssaidi@eecs.umich.edu return true; 12283804Ssaidi@eecs.umich.edu} 12293804Ssaidi@eecs.umich.edu 12303804Ssaidi@eecs.umich.edutemplate <class Impl> 12313804Ssaidi@eecs.umich.eduvoid 12323804Ssaidi@eecs.umich.eduLSQUnit<Impl>::recvRetry() 12333804Ssaidi@eecs.umich.edu{ 12343804Ssaidi@eecs.umich.edu if (isStoreBlocked) { 12353804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Receiving retry: store blocked\n"); 12363804Ssaidi@eecs.umich.edu assert(retryPkt != NULL); 12373804Ssaidi@eecs.umich.edu 12383804Ssaidi@eecs.umich.edu LSQSenderState *state = 12393804Ssaidi@eecs.umich.edu dynamic_cast<LSQSenderState *>(retryPkt->senderState); 12403804Ssaidi@eecs.umich.edu 12413804Ssaidi@eecs.umich.edu if (dcachePort->sendTimingReq(retryPkt)) { 12423804Ssaidi@eecs.umich.edu // Don't finish the store unless this is the last packet. 12433804Ssaidi@eecs.umich.edu if (!TheISA::HasUnalignedMemAcc || !state->pktToSend || 12443804Ssaidi@eecs.umich.edu state->pendingPacket == retryPkt) { 12453804Ssaidi@eecs.umich.edu state->pktToSend = false; 12463804Ssaidi@eecs.umich.edu storePostSend(retryPkt); 12473804Ssaidi@eecs.umich.edu } 12483804Ssaidi@eecs.umich.edu retryPkt = NULL; 12493804Ssaidi@eecs.umich.edu isStoreBlocked = false; 12503804Ssaidi@eecs.umich.edu lsq->setRetryTid(InvalidThreadID); 12513804Ssaidi@eecs.umich.edu 12523804Ssaidi@eecs.umich.edu // Send any outstanding packet. 12533804Ssaidi@eecs.umich.edu if (TheISA::HasUnalignedMemAcc && state->pktToSend) { 12543804Ssaidi@eecs.umich.edu assert(state->pendingPacket); 12553804Ssaidi@eecs.umich.edu if (sendStore(state->pendingPacket)) { 12563804Ssaidi@eecs.umich.edu storePostSend(state->pendingPacket); 12573804Ssaidi@eecs.umich.edu } 12583804Ssaidi@eecs.umich.edu } 12593804Ssaidi@eecs.umich.edu } else { 12603804Ssaidi@eecs.umich.edu // Still blocked! 12613804Ssaidi@eecs.umich.edu ++lsqCacheBlocked; 12623804Ssaidi@eecs.umich.edu lsq->setRetryTid(lsqID); 12633804Ssaidi@eecs.umich.edu } 12643804Ssaidi@eecs.umich.edu } else if (isLoadBlocked) { 12653804Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, " 12663804Ssaidi@eecs.umich.edu "no need to resend packet.\n"); 1267 } else { 1268 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n"); 1269 } 1270} 1271 1272template <class Impl> 1273inline void 1274LSQUnit<Impl>::incrStIdx(int &store_idx) const 1275{ 1276 if (++store_idx >= SQEntries) 1277 store_idx = 0; 1278} 1279 1280template <class Impl> 1281inline void 1282LSQUnit<Impl>::decrStIdx(int &store_idx) const 1283{ 1284 if (--store_idx < 0) 1285 store_idx += SQEntries; 1286} 1287 1288template <class Impl> 1289inline void 1290LSQUnit<Impl>::incrLdIdx(int &load_idx) const 1291{ 1292 if (++load_idx >= LQEntries) 1293 load_idx = 0; 1294} 1295 1296template <class Impl> 1297inline void 1298LSQUnit<Impl>::decrLdIdx(int &load_idx) const 1299{ 1300 if (--load_idx < 0) 1301 load_idx += LQEntries; 1302} 1303 1304template <class Impl> 1305void 1306LSQUnit<Impl>::dumpInsts() const 1307{ 1308 cprintf("Load store queue: Dumping instructions.\n"); 1309 cprintf("Load queue size: %i\n", loads); 1310 cprintf("Load queue: "); 1311 1312 int load_idx = loadHead; 1313 1314 while (load_idx != loadTail && loadQueue[load_idx]) { 1315 const DynInstPtr &inst(loadQueue[load_idx]); 1316 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1317 1318 incrLdIdx(load_idx); 1319 } 1320 cprintf("\n"); 1321 1322 cprintf("Store queue size: %i\n", stores); 1323 cprintf("Store queue: "); 1324 1325 int store_idx = storeHead; 1326 1327 while (store_idx != storeTail && storeQueue[store_idx].inst) { 1328 const DynInstPtr &inst(storeQueue[store_idx].inst); 1329 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1330 1331 incrStIdx(store_idx); 1332 } 1333 1334 cprintf("\n"); 1335} 1336 1337#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 1338