lsq_unit_impl.hh revision 10030:b531e328342d
19793Sakash.bagdia@arm.com 29518SAndreas.Sandberg@ARM.com/* 311320Ssteve.reinhardt@amd.com * Copyright (c) 2010-2013 ARM Limited 49518SAndreas.Sandberg@ARM.com * All rights reserved 59518SAndreas.Sandberg@ARM.com * 69518SAndreas.Sandberg@ARM.com * The license below extends only to copyright in the software and shall 79518SAndreas.Sandberg@ARM.com * not be construed as granting a license to any other intellectual 89518SAndreas.Sandberg@ARM.com * property including but not limited to intellectual property relating 99518SAndreas.Sandberg@ARM.com * to a hardware implementation of the functionality of the software 109518SAndreas.Sandberg@ARM.com * licensed hereunder. You may use the software subject to the license 119518SAndreas.Sandberg@ARM.com * terms below provided that you ensure that this notice is replicated 129518SAndreas.Sandberg@ARM.com * unmodified and in its entirety in all distributions of the software, 135347Ssaidi@eecs.umich.edu * modified or unmodified, in source code or in binary form. 147534Ssteve.reinhardt@amd.com * 153395Shsul@eecs.umich.edu * Copyright (c) 2004-2005 The Regents of The University of Michigan 163395Shsul@eecs.umich.edu * All rights reserved. 173395Shsul@eecs.umich.edu * 183395Shsul@eecs.umich.edu * Redistribution and use in source and binary forms, with or without 193395Shsul@eecs.umich.edu * modification, are permitted provided that the following conditions are 203395Shsul@eecs.umich.edu * met: redistributions of source code must retain the above copyright 213395Shsul@eecs.umich.edu * notice, this list of conditions and the following disclaimer; 223395Shsul@eecs.umich.edu * redistributions in binary form must reproduce the above copyright 233395Shsul@eecs.umich.edu * notice, this list of conditions and the following disclaimer in the 243395Shsul@eecs.umich.edu * documentation and/or other materials provided with the distribution; 253395Shsul@eecs.umich.edu * neither the name of the copyright holders nor the names of its 263395Shsul@eecs.umich.edu * contributors may be used to endorse or promote products derived from 273395Shsul@eecs.umich.edu * this software without specific prior written permission. 283395Shsul@eecs.umich.edu * 293395Shsul@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 303395Shsul@eecs.umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 313395Shsul@eecs.umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 323395Shsul@eecs.umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 333395Shsul@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 343395Shsul@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 353395Shsul@eecs.umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 363395Shsul@eecs.umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 373395Shsul@eecs.umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 383395Shsul@eecs.umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 393395Shsul@eecs.umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 403395Shsul@eecs.umich.edu * 413395Shsul@eecs.umich.edu * Authors: Kevin Lim 429457Svilanova@ac.upc.edu * Korey Sewell 433395Shsul@eecs.umich.edu */ 443509Shsul@eecs.umich.edu 456654Snate@binkert.org#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__ 4611688Sandreas.hansson@arm.com#define __CPU_O3_LSQ_UNIT_IMPL_HH__ 4711688Sandreas.hansson@arm.com 489520SAndreas.Sandberg@ARM.com#include "arch/generic/debugfaults.hh" 493395Shsul@eecs.umich.edu#include "arch/locked_mem.hh" 506654Snate@binkert.org#include "base/str.hh" 513395Shsul@eecs.umich.edu#include "config/the_isa.hh" 526654Snate@binkert.org#include "cpu/checker/cpu.hh" 536654Snate@binkert.org#include "cpu/o3/lsq.hh" 546654Snate@binkert.org#include "cpu/o3/lsq_unit.hh" 553395Shsul@eecs.umich.edu#include "debug/Activity.hh" 569139Snilay@cs.wisc.edu#include "debug/IEW.hh" 579520SAndreas.Sandberg@ARM.com#include "debug/LSQUnit.hh" 589520SAndreas.Sandberg@ARM.com#include "debug/O3PipeView.hh" 599520SAndreas.Sandberg@ARM.com#include "mem/packet.hh" 609139Snilay@cs.wisc.edu#include "mem/request.hh" 613481Shsul@eecs.umich.edu 629139Snilay@cs.wisc.edutemplate<class Impl> 633481Shsul@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt, 649139Snilay@cs.wisc.edu LSQUnit *lsq_ptr) 659139Snilay@cs.wisc.edu : Event(Default_Pri, AutoDelete), 669139Snilay@cs.wisc.edu inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr) 679139Snilay@cs.wisc.edu{ 689139Snilay@cs.wisc.edu} 699139Snilay@cs.wisc.edu 709139Snilay@cs.wisc.edutemplate<class Impl> 719139Snilay@cs.wisc.eduvoid 723481Shsul@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::process() 739518SAndreas.Sandberg@ARM.com{ 749518SAndreas.Sandberg@ARM.com assert(!lsqPtr->cpu->switchedOut()); 759518SAndreas.Sandberg@ARM.com 763481Shsul@eecs.umich.edu lsqPtr->writeback(inst, pkt); 779139Snilay@cs.wisc.edu 789139Snilay@cs.wisc.edu if (pkt->senderState) 793481Shsul@eecs.umich.edu delete pkt->senderState; 809139Snilay@cs.wisc.edu 819139Snilay@cs.wisc.edu delete pkt->req; 829139Snilay@cs.wisc.edu delete pkt; 839139Snilay@cs.wisc.edu} 849139Snilay@cs.wisc.edu 853481Shsul@eecs.umich.edutemplate<class Impl> 863481Shsul@eecs.umich.educonst char * 873481Shsul@eecs.umich.eduLSQUnit<Impl>::WritebackEvent::description() const 889665Sandreas.hansson@arm.com{ 899665Sandreas.hansson@arm.com return "Store writeback"; 909665Sandreas.hansson@arm.com} 919665Sandreas.hansson@arm.com 929665Sandreas.hansson@arm.comtemplate<class Impl> 938919Snilay@cs.wisc.eduvoid 948919Snilay@cs.wisc.eduLSQUnit<Impl>::completeDataAccess(PacketPtr pkt) 958919Snilay@cs.wisc.edu{ 9610159Sgedare@rtems.org LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState); 9710159Sgedare@rtems.org DynInstPtr inst = state->inst; 988919Snilay@cs.wisc.edu DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum); 998919Snilay@cs.wisc.edu DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum); 1008919Snilay@cs.wisc.edu 1018919Snilay@cs.wisc.edu //iewStage->ldstQueue.removeMSHR(inst->threadNumber,inst->seqNum); 1028919Snilay@cs.wisc.edu 1038919Snilay@cs.wisc.edu // If this is a split access, wait until all packets are received. 1048919Snilay@cs.wisc.edu if (TheISA::HasUnalignedMemAcc && !state->complete()) { 1058919Snilay@cs.wisc.edu delete pkt->req; 1068919Snilay@cs.wisc.edu delete pkt; 1078919Snilay@cs.wisc.edu return; 1088919Snilay@cs.wisc.edu } 1098919Snilay@cs.wisc.edu 1103481Shsul@eecs.umich.edu assert(!cpu->switchedOut()); 1119816Sjthestness@gmail.com if (inst->isSquashed()) { 1129140Snilay@cs.wisc.edu iewStage->decrWb(inst->seqNum); 1139140Snilay@cs.wisc.edu } else { 1149140Snilay@cs.wisc.edu if (!state->noWB) { 1159140Snilay@cs.wisc.edu if (!TheISA::HasUnalignedMemAcc || !state->isSplit || 1169140Snilay@cs.wisc.edu !state->isLoad) { 1179140Snilay@cs.wisc.edu writeback(inst, pkt); 1189140Snilay@cs.wisc.edu } else { 1199140Snilay@cs.wisc.edu writeback(inst, state->mainPkt); 1209140Snilay@cs.wisc.edu } 1219140Snilay@cs.wisc.edu } 1229140Snilay@cs.wisc.edu 1239140Snilay@cs.wisc.edu if (inst->isStore()) { 1249140Snilay@cs.wisc.edu completeStore(state->idx); 1259140Snilay@cs.wisc.edu } 1269140Snilay@cs.wisc.edu } 1279140Snilay@cs.wisc.edu 1289140Snilay@cs.wisc.edu if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) { 1299140Snilay@cs.wisc.edu delete state->mainPkt->req; 1309140Snilay@cs.wisc.edu delete state->mainPkt; 1319867Sjthestness@gmail.com } 1329140Snilay@cs.wisc.edu 1339140Snilay@cs.wisc.edu pkt->req->setAccessLatency(); 1349140Snilay@cs.wisc.edu cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt)); 1359140Snilay@cs.wisc.edu 1369140Snilay@cs.wisc.edu delete state; 1379140Snilay@cs.wisc.edu delete pkt->req; 1389140Snilay@cs.wisc.edu delete pkt; 1399140Snilay@cs.wisc.edu} 1409140Snilay@cs.wisc.edu 1419140Snilay@cs.wisc.edutemplate <class Impl> 1429140Snilay@cs.wisc.eduLSQUnit<Impl>::LSQUnit() 14310608Sdam.sunwoo@arm.com : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false), 14410608Sdam.sunwoo@arm.com isStoreBlocked(false), isLoadBlocked(false), 14510608Sdam.sunwoo@arm.com loadBlockedHandled(false), storeInFlight(false), hasPendingPkt(false) 14610608Sdam.sunwoo@arm.com{ 14710608Sdam.sunwoo@arm.com} 14810608Sdam.sunwoo@arm.com 14910608Sdam.sunwoo@arm.comtemplate<class Impl> 15010608Sdam.sunwoo@arm.comvoid 15110608Sdam.sunwoo@arm.comLSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 15210608Sdam.sunwoo@arm.com LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 15310608Sdam.sunwoo@arm.com unsigned id) 15410608Sdam.sunwoo@arm.com{ 15510608Sdam.sunwoo@arm.com cpu = cpu_ptr; 15610608Sdam.sunwoo@arm.com iewStage = iew_ptr; 15710608Sdam.sunwoo@arm.com 15810608Sdam.sunwoo@arm.com DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id); 15910608Sdam.sunwoo@arm.com 16010608Sdam.sunwoo@arm.com lsq = lsq_ptr; 16110608Sdam.sunwoo@arm.com 16210608Sdam.sunwoo@arm.com lsqID = id; 16310608Sdam.sunwoo@arm.com 16410608Sdam.sunwoo@arm.com // Add 1 for the sentinel entry (they are circular queues). 16510608Sdam.sunwoo@arm.com LQEntries = maxLQEntries + 1; 16610608Sdam.sunwoo@arm.com SQEntries = maxSQEntries + 1; 16710608Sdam.sunwoo@arm.com 16810608Sdam.sunwoo@arm.com //Due to uint8_t index in LSQSenderState 16910608Sdam.sunwoo@arm.com assert(LQEntries <= 256); 17010608Sdam.sunwoo@arm.com assert(SQEntries <= 256); 17110608Sdam.sunwoo@arm.com 17210608Sdam.sunwoo@arm.com loadQueue.resize(LQEntries); 17310608Sdam.sunwoo@arm.com storeQueue.resize(SQEntries); 17410608Sdam.sunwoo@arm.com 17510608Sdam.sunwoo@arm.com depCheckShift = params->LSQDepCheckShift; 17610608Sdam.sunwoo@arm.com checkLoads = params->LSQCheckLoads; 17710608Sdam.sunwoo@arm.com cachePorts = params->cachePorts; 17810608Sdam.sunwoo@arm.com needsTSO = params->needsTSO; 17910608Sdam.sunwoo@arm.com 1809140Snilay@cs.wisc.edu resetState(); 1819140Snilay@cs.wisc.edu} 18210608Sdam.sunwoo@arm.com 1839140Snilay@cs.wisc.edu 1849140Snilay@cs.wisc.edutemplate<class Impl> 1859140Snilay@cs.wisc.eduvoid 1869140Snilay@cs.wisc.eduLSQUnit<Impl>::resetState() 1879140Snilay@cs.wisc.edu{ 1889140Snilay@cs.wisc.edu loads = stores = storesToWB = 0; 1899140Snilay@cs.wisc.edu 1909140Snilay@cs.wisc.edu loadHead = loadTail = 0; 1919140Snilay@cs.wisc.edu 1929140Snilay@cs.wisc.edu storeHead = storeWBIdx = storeTail = 0; 1939140Snilay@cs.wisc.edu 1949140Snilay@cs.wisc.edu usedPorts = 0; 1959816Sjthestness@gmail.com 1969140Snilay@cs.wisc.edu retryPkt = NULL; 1979140Snilay@cs.wisc.edu memDepViolator = NULL; 1989816Sjthestness@gmail.com 1999140Snilay@cs.wisc.edu blockedLoadSeqNum = 0; 2009215Sandreas.hansson@arm.com 2019140Snilay@cs.wisc.edu stalled = false; 2029140Snilay@cs.wisc.edu isLoadBlocked = false; 2039140Snilay@cs.wisc.edu loadBlockedHandled = false; 2049140Snilay@cs.wisc.edu 2059140Snilay@cs.wisc.edu cacheBlockMask = ~(cpu->cacheLineSize() - 1); 2069140Snilay@cs.wisc.edu} 2079140Snilay@cs.wisc.edu 2089140Snilay@cs.wisc.edutemplate<class Impl> 2099140Snilay@cs.wisc.edustd::string 2109140Snilay@cs.wisc.eduLSQUnit<Impl>::name() const 2119140Snilay@cs.wisc.edu{ 2129140Snilay@cs.wisc.edu if (Impl::MaxThreads == 1) { 2139140Snilay@cs.wisc.edu return iewStage->name() + ".lsq"; 2149140Snilay@cs.wisc.edu } else { 2159140Snilay@cs.wisc.edu return iewStage->name() + ".lsq.thread" + to_string(lsqID); 2169140Snilay@cs.wisc.edu } 2179140Snilay@cs.wisc.edu} 2189140Snilay@cs.wisc.edu 2199140Snilay@cs.wisc.edutemplate<class Impl> 2209140Snilay@cs.wisc.eduvoid 2219140Snilay@cs.wisc.eduLSQUnit<Impl>::regStats() 2229140Snilay@cs.wisc.edu{ 2239140Snilay@cs.wisc.edu lsqForwLoads 2249140Snilay@cs.wisc.edu .name(name() + ".forwLoads") 2259140Snilay@cs.wisc.edu .desc("Number of loads that had data forwarded from stores"); 2269140Snilay@cs.wisc.edu 2279156Sandreas.hansson@arm.com invAddrLoads 2289140Snilay@cs.wisc.edu .name(name() + ".invAddrLoads") 2299634Sjthestness@gmail.com .desc("Number of loads ignored due to an invalid address"); 2309140Snilay@cs.wisc.edu 2319140Snilay@cs.wisc.edu lsqSquashedLoads 2329140Snilay@cs.wisc.edu .name(name() + ".squashedLoads") 2339140Snilay@cs.wisc.edu .desc("Number of loads squashed"); 2349140Snilay@cs.wisc.edu 2359140Snilay@cs.wisc.edu lsqIgnoredResponses 2369140Snilay@cs.wisc.edu .name(name() + ".ignoredResponses") 2379140Snilay@cs.wisc.edu .desc("Number of memory responses ignored because the instruction is squashed"); 2389140Snilay@cs.wisc.edu 2399140Snilay@cs.wisc.edu lsqMemOrderViolation 2409140Snilay@cs.wisc.edu .name(name() + ".memOrderViolation") 2419140Snilay@cs.wisc.edu .desc("Number of memory ordering violations"); 2429140Snilay@cs.wisc.edu 2439140Snilay@cs.wisc.edu lsqSquashedStores 2449140Snilay@cs.wisc.edu .name(name() + ".squashedStores") 2459140Snilay@cs.wisc.edu .desc("Number of stores squashed"); 2469140Snilay@cs.wisc.edu 2479140Snilay@cs.wisc.edu invAddrSwpfs 2489140Snilay@cs.wisc.edu .name(name() + ".invAddrSwpfs") 2499140Snilay@cs.wisc.edu .desc("Number of software prefetches ignored due to an invalid address"); 2509140Snilay@cs.wisc.edu 2519140Snilay@cs.wisc.edu lsqBlockedLoads 2529140Snilay@cs.wisc.edu .name(name() + ".blockedLoads") 2539140Snilay@cs.wisc.edu .desc("Number of blocked loads due to partial load-store forwarding"); 2549140Snilay@cs.wisc.edu 2559140Snilay@cs.wisc.edu lsqRescheduledLoads 2569140Snilay@cs.wisc.edu .name(name() + ".rescheduledLoads") 2579140Snilay@cs.wisc.edu .desc("Number of loads that were rescheduled"); 2589606Snilay@cs.wisc.edu 2599140Snilay@cs.wisc.edu lsqCacheBlocked 2609140Snilay@cs.wisc.edu .name(name() + ".cacheBlocked") 2619634Sjthestness@gmail.com .desc("Number of times an access to memory failed due to the cache being blocked"); 2629140Snilay@cs.wisc.edu} 2639140Snilay@cs.wisc.edu 2649140Snilay@cs.wisc.edutemplate<class Impl> 2659140Snilay@cs.wisc.eduvoid 2669140Snilay@cs.wisc.eduLSQUnit<Impl>::setDcachePort(MasterPort *dcache_port) 2679140Snilay@cs.wisc.edu{ 2689140Snilay@cs.wisc.edu dcachePort = dcache_port; 2699140Snilay@cs.wisc.edu} 2709140Snilay@cs.wisc.edu 2719140Snilay@cs.wisc.edutemplate<class Impl> 2729140Snilay@cs.wisc.eduvoid 2739140Snilay@cs.wisc.eduLSQUnit<Impl>::clearLQ() 2749140Snilay@cs.wisc.edu{ 2759140Snilay@cs.wisc.edu loadQueue.clear(); 2769140Snilay@cs.wisc.edu} 2779460Ssaidi@eecs.umich.edu 2789140Snilay@cs.wisc.edutemplate<class Impl> 27910608Sdam.sunwoo@arm.comvoid 28010608Sdam.sunwoo@arm.comLSQUnit<Impl>::clearSQ() 28110608Sdam.sunwoo@arm.com{ 28210608Sdam.sunwoo@arm.com storeQueue.clear(); 28310608Sdam.sunwoo@arm.com} 28410608Sdam.sunwoo@arm.com 28510608Sdam.sunwoo@arm.comtemplate<class Impl> 28610608Sdam.sunwoo@arm.comvoid 28710608Sdam.sunwoo@arm.comLSQUnit<Impl>::drainSanityCheck() const 28810608Sdam.sunwoo@arm.com{ 28910608Sdam.sunwoo@arm.com for (int i = 0; i < loadQueue.size(); ++i) 29010608Sdam.sunwoo@arm.com assert(!loadQueue[i]); 29110608Sdam.sunwoo@arm.com 29210608Sdam.sunwoo@arm.com assert(storesToWB == 0); 29310608Sdam.sunwoo@arm.com assert(!retryPkt); 29410608Sdam.sunwoo@arm.com} 29510608Sdam.sunwoo@arm.com 29610608Sdam.sunwoo@arm.comtemplate<class Impl> 29710608Sdam.sunwoo@arm.comvoid 29810608Sdam.sunwoo@arm.comLSQUnit<Impl>::takeOverFrom() 29910608Sdam.sunwoo@arm.com{ 30010608Sdam.sunwoo@arm.com resetState(); 30110608Sdam.sunwoo@arm.com} 30210608Sdam.sunwoo@arm.com 30310608Sdam.sunwoo@arm.comtemplate<class Impl> 30410608Sdam.sunwoo@arm.comvoid 30510608Sdam.sunwoo@arm.comLSQUnit<Impl>::resizeLQ(unsigned size) 30610608Sdam.sunwoo@arm.com{ 30710608Sdam.sunwoo@arm.com unsigned size_plus_sentinel = size + 1; 30810608Sdam.sunwoo@arm.com assert(size_plus_sentinel >= LQEntries); 30910608Sdam.sunwoo@arm.com 31010608Sdam.sunwoo@arm.com if (size_plus_sentinel > LQEntries) { 31110608Sdam.sunwoo@arm.com while (size_plus_sentinel > loadQueue.size()) { 31210608Sdam.sunwoo@arm.com DynInstPtr dummy; 31310608Sdam.sunwoo@arm.com loadQueue.push_back(dummy); 31410608Sdam.sunwoo@arm.com LQEntries++; 31510608Sdam.sunwoo@arm.com } 31610608Sdam.sunwoo@arm.com } else { 31710608Sdam.sunwoo@arm.com LQEntries = size_plus_sentinel; 31810608Sdam.sunwoo@arm.com } 31910608Sdam.sunwoo@arm.com 32010608Sdam.sunwoo@arm.com assert(LQEntries <= 256); 32110608Sdam.sunwoo@arm.com} 32210608Sdam.sunwoo@arm.com 32310608Sdam.sunwoo@arm.comtemplate<class Impl> 32410608Sdam.sunwoo@arm.comvoid 32510608Sdam.sunwoo@arm.comLSQUnit<Impl>::resizeSQ(unsigned size) 32610608Sdam.sunwoo@arm.com{ 32710608Sdam.sunwoo@arm.com unsigned size_plus_sentinel = size + 1; 32810608Sdam.sunwoo@arm.com if (size_plus_sentinel > SQEntries) { 32910608Sdam.sunwoo@arm.com while (size_plus_sentinel > storeQueue.size()) { 33010608Sdam.sunwoo@arm.com SQEntry dummy; 33110608Sdam.sunwoo@arm.com storeQueue.push_back(dummy); 33210608Sdam.sunwoo@arm.com SQEntries++; 33310608Sdam.sunwoo@arm.com } 33410608Sdam.sunwoo@arm.com } else { 33510608Sdam.sunwoo@arm.com SQEntries = size_plus_sentinel; 33610608Sdam.sunwoo@arm.com } 33710608Sdam.sunwoo@arm.com 33810608Sdam.sunwoo@arm.com assert(SQEntries <= 256); 33910608Sdam.sunwoo@arm.com} 34010608Sdam.sunwoo@arm.com 34110608Sdam.sunwoo@arm.comtemplate <class Impl> 34210608Sdam.sunwoo@arm.comvoid 34310608Sdam.sunwoo@arm.comLSQUnit<Impl>::insert(DynInstPtr &inst) 34410608Sdam.sunwoo@arm.com{ 34510608Sdam.sunwoo@arm.com assert(inst->isMemRef()); 34610608Sdam.sunwoo@arm.com 34710608Sdam.sunwoo@arm.com assert(inst->isLoad() || inst->isStore()); 34810608Sdam.sunwoo@arm.com 34910608Sdam.sunwoo@arm.com if (inst->isLoad()) { 35010608Sdam.sunwoo@arm.com insertLoad(inst); 35110608Sdam.sunwoo@arm.com } else { 35210608Sdam.sunwoo@arm.com insertStore(inst); 35310608Sdam.sunwoo@arm.com } 35410608Sdam.sunwoo@arm.com 35510608Sdam.sunwoo@arm.com inst->setInLSQ(); 35610608Sdam.sunwoo@arm.com} 35710608Sdam.sunwoo@arm.com 35810608Sdam.sunwoo@arm.comtemplate <class Impl> 35910608Sdam.sunwoo@arm.comvoid 36010608Sdam.sunwoo@arm.comLSQUnit<Impl>::insertLoad(DynInstPtr &load_inst) 36110608Sdam.sunwoo@arm.com{ 36210608Sdam.sunwoo@arm.com assert((loadTail + 1) % LQEntries != loadHead); 36310608Sdam.sunwoo@arm.com assert(loads < LQEntries); 36410608Sdam.sunwoo@arm.com 36510608Sdam.sunwoo@arm.com DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n", 36610608Sdam.sunwoo@arm.com load_inst->pcState(), loadTail, load_inst->seqNum); 36710608Sdam.sunwoo@arm.com 36810608Sdam.sunwoo@arm.com load_inst->lqIdx = loadTail; 36910608Sdam.sunwoo@arm.com 37010608Sdam.sunwoo@arm.com if (stores == 0) { 37110608Sdam.sunwoo@arm.com load_inst->sqIdx = -1; 37210608Sdam.sunwoo@arm.com } else { 37310608Sdam.sunwoo@arm.com load_inst->sqIdx = storeTail; 37410608Sdam.sunwoo@arm.com } 37510608Sdam.sunwoo@arm.com 37610608Sdam.sunwoo@arm.com loadQueue[loadTail] = load_inst; 37710608Sdam.sunwoo@arm.com 37810608Sdam.sunwoo@arm.com incrLdIdx(loadTail); 37910608Sdam.sunwoo@arm.com 38010608Sdam.sunwoo@arm.com ++loads; 38110608Sdam.sunwoo@arm.com} 38210608Sdam.sunwoo@arm.com 38310608Sdam.sunwoo@arm.comtemplate <class Impl> 38410608Sdam.sunwoo@arm.comvoid 38510608Sdam.sunwoo@arm.comLSQUnit<Impl>::insertStore(DynInstPtr &store_inst) 38610608Sdam.sunwoo@arm.com{ 38710608Sdam.sunwoo@arm.com // Make sure it is not full before inserting an instruction. 38810608Sdam.sunwoo@arm.com assert((storeTail + 1) % SQEntries != storeHead); 38910608Sdam.sunwoo@arm.com assert(stores < SQEntries); 39010608Sdam.sunwoo@arm.com 39110608Sdam.sunwoo@arm.com DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n", 39210608Sdam.sunwoo@arm.com store_inst->pcState(), storeTail, store_inst->seqNum); 39310608Sdam.sunwoo@arm.com 39410608Sdam.sunwoo@arm.com store_inst->sqIdx = storeTail; 39510608Sdam.sunwoo@arm.com store_inst->lqIdx = loadTail; 39610608Sdam.sunwoo@arm.com 39710608Sdam.sunwoo@arm.com storeQueue[storeTail] = SQEntry(store_inst); 39810608Sdam.sunwoo@arm.com 39910608Sdam.sunwoo@arm.com incrStIdx(storeTail); 40010608Sdam.sunwoo@arm.com 40110608Sdam.sunwoo@arm.com ++stores; 40210608Sdam.sunwoo@arm.com} 40310608Sdam.sunwoo@arm.com 4049151Satgutier@umich.edutemplate <class Impl> 4059151Satgutier@umich.edutypename Impl::DynInstPtr 4069151Satgutier@umich.eduLSQUnit<Impl>::getMemDepViolator() 4079151Satgutier@umich.edu{ 4089151Satgutier@umich.edu DynInstPtr temp = memDepViolator; 4099151Satgutier@umich.edu 4109151Satgutier@umich.edu memDepViolator = NULL; 4119460Ssaidi@eecs.umich.edu 4129151Satgutier@umich.edu return temp; 4139521SAndreas.Sandberg@ARM.com} 4149151Satgutier@umich.edu 4159151Satgutier@umich.edutemplate <class Impl> 4169151Satgutier@umich.eduunsigned 4179151Satgutier@umich.eduLSQUnit<Impl>::numFreeEntries() 4189151Satgutier@umich.edu{ 4199151Satgutier@umich.edu unsigned free_lq_entries = LQEntries - loads; 4209151Satgutier@umich.edu unsigned free_sq_entries = SQEntries - stores; 4219151Satgutier@umich.edu 4229460Ssaidi@eecs.umich.edu // Both the LQ and SQ entries have an extra dummy entry to differentiate 4239151Satgutier@umich.edu // empty/full conditions. Subtract 1 from the free entries. 4243481Shsul@eecs.umich.edu if (free_lq_entries < free_sq_entries) { 4253395Shsul@eecs.umich.edu return free_lq_entries - 1; 4263395Shsul@eecs.umich.edu } else { 4275211Ssaidi@eecs.umich.edu return free_sq_entries - 1; 4285211Ssaidi@eecs.umich.edu } 4293395Shsul@eecs.umich.edu} 4303395Shsul@eecs.umich.edu 4313395Shsul@eecs.umich.edutemplate <class Impl> 4325370Ssaidi@eecs.umich.eduvoid 4336654Snate@binkert.orgLSQUnit<Impl>::checkSnoop(PacketPtr pkt) 4345370Ssaidi@eecs.umich.edu{ 4355371Shsul@eecs.umich.edu int load_idx = loadHead; 4366654Snate@binkert.org DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr()); 4375370Ssaidi@eecs.umich.edu 4389151Satgutier@umich.edu // Unlock the cpu-local monitor when the CPU sees a snoop to a locked 4399151Satgutier@umich.edu // address. The CPU can speculatively execute a LL operation after a pending 4409151Satgutier@umich.edu // SC operation in the pipeline and that can make the cache monitor the CPU 4419151Satgutier@umich.edu // is connected to valid while it really shouldn't be. 4429151Satgutier@umich.edu for (int x = 0; x < cpu->numContexts(); x++) { 4439151Satgutier@umich.edu ThreadContext *tc = cpu->getContext(x); 4443395Shsul@eecs.umich.edu bool no_squash = cpu->thread[x]->noSquashFromTC; 4453481Shsul@eecs.umich.edu cpu->thread[x]->noSquashFromTC = true; 4463481Shsul@eecs.umich.edu TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask); 4478318Sksewell@umich.edu cpu->thread[x]->noSquashFromTC = no_squash; 4486144Sksewell@umich.edu } 4498311Sksewell@umich.edu 4506144Sksewell@umich.edu Addr invalidate_addr = pkt->getAddr() & cacheBlockMask; 4516641Sksewell@umich.edu 4526641Sksewell@umich.edu DynInstPtr ld_inst = loadQueue[load_idx]; 4536641Sksewell@umich.edu if (ld_inst) { 4546641Sksewell@umich.edu Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 4553481Shsul@eecs.umich.edu // Check that this snoop didn't just invalidate our lock flag 4569433SAndreas.Sandberg@ARM.com if (ld_inst->effAddrValid() && load_addr == invalidate_addr && 4573481Shsul@eecs.umich.edu ld_inst->memReqFlags & Request::LLSC) 4583481Shsul@eecs.umich.edu TheISA::handleLockedSnoopHit(ld_inst.get()); 4593481Shsul@eecs.umich.edu } 4605361Srstrong@cs.ucsd.edu 4615369Ssaidi@eecs.umich.edu // If this is the only load in the LSQ we don't care 46211251Sradhika.jagtap@ARM.com if (load_idx == loadTail) 4638803Sgblack@eecs.umich.edu return; 4649793Sakash.bagdia@arm.com 46511251Sradhika.jagtap@ARM.com incrLdIdx(load_idx); 46611251Sradhika.jagtap@ARM.com 46712374Saustinharris@utexas.edu while (load_idx != loadTail) { 4685369Ssaidi@eecs.umich.edu DynInstPtr ld_inst = loadQueue[load_idx]; 4698311Sksewell@umich.edu 4708311Sksewell@umich.edu if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 4718887Sgeoffrey.blake@arm.com incrLdIdx(load_idx); 4728887Sgeoffrey.blake@arm.com continue; 4738887Sgeoffrey.blake@arm.com } 4743481Shsul@eecs.umich.edu 47511251Sradhika.jagtap@ARM.com Addr load_addr = ld_inst->physEffAddr & cacheBlockMask; 47611251Sradhika.jagtap@ARM.com DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n", 47711251Sradhika.jagtap@ARM.com ld_inst->seqNum, load_addr, invalidate_addr); 47811251Sradhika.jagtap@ARM.com 47911251Sradhika.jagtap@ARM.com if (load_addr == invalidate_addr) { 4805311Ssaidi@eecs.umich.edu if (ld_inst->possibleLoadViolation()) { 4813481Shsul@eecs.umich.edu DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n", 4823395Shsul@eecs.umich.edu pkt->getAddr(), ld_inst->seqNum); 4839151Satgutier@umich.edu 4849518SAndreas.Sandberg@ARM.com // Mark the load for re-execution 4859518SAndreas.Sandberg@ARM.com ld_inst->fault = new ReExec; 4869518SAndreas.Sandberg@ARM.com } else { 4879518SAndreas.Sandberg@ARM.com DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n", 4889518SAndreas.Sandberg@ARM.com pkt->getAddr(), ld_inst->seqNum); 4899518SAndreas.Sandberg@ARM.com 4909518SAndreas.Sandberg@ARM.com // Make sure that we don't lose a snoop hitting a LOCKED 4919518SAndreas.Sandberg@ARM.com // address since the LOCK* flags don't get updated until 4929151Satgutier@umich.edu // commit. 4939518SAndreas.Sandberg@ARM.com if (ld_inst->memReqFlags & Request::LLSC) 4949518SAndreas.Sandberg@ARM.com TheISA::handleLockedSnoopHit(ld_inst.get()); 4959151Satgutier@umich.edu 4969151Satgutier@umich.edu // If a older load checks this and it's true 4979151Satgutier@umich.edu // then we might have missed the snoop 4989151Satgutier@umich.edu // in which case we need to invalidate to be sure 4999793Sakash.bagdia@arm.com ld_inst->hitExternalSnoop(true); 50012374Saustinharris@utexas.edu } 5019151Satgutier@umich.edu } 5029151Satgutier@umich.edu incrLdIdx(load_idx); 5039151Satgutier@umich.edu } 5049151Satgutier@umich.edu return; 5059151Satgutier@umich.edu} 5069151Satgutier@umich.edu 5079151Satgutier@umich.edutemplate <class Impl> 5089151Satgutier@umich.eduFault 5099151Satgutier@umich.eduLSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst) 5109151Satgutier@umich.edu{ 5119151Satgutier@umich.edu Addr inst_eff_addr1 = inst->effAddr >> depCheckShift; 5129151Satgutier@umich.edu Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift; 5139151Satgutier@umich.edu 5149151Satgutier@umich.edu /** @todo in theory you only need to check an instruction that has executed 5159151Satgutier@umich.edu * however, there isn't a good way in the pipeline at the moment to check 5169151Satgutier@umich.edu * all instructions that will execute before the store writes back. Thus, 5173395Shsul@eecs.umich.edu * like the implementation that came before it, we're overly conservative. 5189433SAndreas.Sandberg@ARM.com */ 5193395Shsul@eecs.umich.edu while (load_idx != loadTail) { 5209433SAndreas.Sandberg@ARM.com DynInstPtr ld_inst = loadQueue[load_idx]; 5213395Shsul@eecs.umich.edu if (!ld_inst->effAddrValid() || ld_inst->uncacheable()) { 5223478Shsul@eecs.umich.edu incrLdIdx(load_idx); 5233395Shsul@eecs.umich.edu continue; 5243395Shsul@eecs.umich.edu } 5253478Shsul@eecs.umich.edu 5268803Sgblack@eecs.umich.edu Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift; 5278803Sgblack@eecs.umich.edu Addr ld_eff_addr2 = 5289793Sakash.bagdia@arm.com (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift; 5299793Sakash.bagdia@arm.com 53012374Saustinharris@utexas.edu if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) { 53112374Saustinharris@utexas.edu if (inst->isLoad()) { 5323480Shsul@eecs.umich.edu // If this load is to the same block as an external snoop 5335361Srstrong@cs.ucsd.edu // invalidate that we've observed then the load needs to be 5345369Ssaidi@eecs.umich.edu // squashed as it could have newer data 5355361Srstrong@cs.ucsd.edu if (ld_inst->hitExternalSnoop()) { 5365361Srstrong@cs.ucsd.edu if (!memDepViolator || 5375361Srstrong@cs.ucsd.edu ld_inst->seqNum < memDepViolator->seqNum) { 5385369Ssaidi@eecs.umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] " 5395361Srstrong@cs.ucsd.edu "and [sn:%lli] at address %#x\n", 5405361Srstrong@cs.ucsd.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5415378Ssaidi@eecs.umich.edu memDepViolator = ld_inst; 5426654Snate@binkert.org 5435361Srstrong@cs.ucsd.edu ++lsqMemOrderViolation; 5445361Srstrong@cs.ucsd.edu 5455361Srstrong@cs.ucsd.edu return new GenericISA::M5PanicFault( 5465361Srstrong@cs.ucsd.edu "Detected fault with inst [sn:%lli] and " 5475361Srstrong@cs.ucsd.edu "[sn:%lli] at address %#x\n", 5485361Srstrong@cs.ucsd.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5495361Srstrong@cs.ucsd.edu } 5505361Srstrong@cs.ucsd.edu } 5515361Srstrong@cs.ucsd.edu 5525361Srstrong@cs.ucsd.edu // Otherwise, mark the load has a possible load violation 5535361Srstrong@cs.ucsd.edu // and if we see a snoop before it's commited, we need to squash 5548311Sksewell@umich.edu ld_inst->possibleLoadViolation(true); 5558311Sksewell@umich.edu DPRINTF(LSQUnit, "Found possible load violaiton at addr: %#x" 5565353Svilas.sridharan@gmail.com " between instructions [sn:%lli] and [sn:%lli]\n", 5578887Sgeoffrey.blake@arm.com inst_eff_addr1, inst->seqNum, ld_inst->seqNum); 5588887Sgeoffrey.blake@arm.com } else { 5598887Sgeoffrey.blake@arm.com // A load/store incorrectly passed this store. 5608887Sgeoffrey.blake@arm.com // Check if we already have a violator, or if it's newer 5618887Sgeoffrey.blake@arm.com // squash and refetch. 5628211Satgutier@umich.edu if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum) 5638211Satgutier@umich.edu break; 5648211Satgutier@umich.edu 5658211Satgutier@umich.edu DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and " 5663395Shsul@eecs.umich.edu "[sn:%lli] at address %#x\n", 5675361Srstrong@cs.ucsd.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5685369Ssaidi@eecs.umich.edu memDepViolator = ld_inst; 5695361Srstrong@cs.ucsd.edu 5705361Srstrong@cs.ucsd.edu ++lsqMemOrderViolation; 5715361Srstrong@cs.ucsd.edu 5725361Srstrong@cs.ucsd.edu return new GenericISA::M5PanicFault("Detected fault with " 5735361Srstrong@cs.ucsd.edu "inst [sn:%lli] and [sn:%lli] at address %#x\n", 5745378Ssaidi@eecs.umich.edu inst->seqNum, ld_inst->seqNum, ld_eff_addr1); 5756654Snate@binkert.org } 5765369Ssaidi@eecs.umich.edu } 5775361Srstrong@cs.ucsd.edu 5785361Srstrong@cs.ucsd.edu incrLdIdx(load_idx); 5795361Srstrong@cs.ucsd.edu } 5805361Srstrong@cs.ucsd.edu return NoFault; 5815361Srstrong@cs.ucsd.edu} 5825361Srstrong@cs.ucsd.edu 5835361Srstrong@cs.ucsd.edu 5845361Srstrong@cs.ucsd.edu 5855361Srstrong@cs.ucsd.edu 5865361Srstrong@cs.ucsd.edutemplate <class Impl> 58710608Sdam.sunwoo@arm.comFault 58810608Sdam.sunwoo@arm.comLSQUnit<Impl>::executeLoad(DynInstPtr &inst) 58910608Sdam.sunwoo@arm.com{ 5907531Ssteve.reinhardt@amd.com using namespace TheISA; 5919816Sjthestness@gmail.com // Execute a specific load. 5929816Sjthestness@gmail.com Fault load_fault = NoFault; 5937531Ssteve.reinhardt@amd.com 5943395Shsul@eecs.umich.edu DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n", 59510757SCurtis.Dunham@arm.com inst->pcState(), inst->seqNum); 59610757SCurtis.Dunham@arm.com 59710757SCurtis.Dunham@arm.com assert(!inst->isSquashed()); 59810757SCurtis.Dunham@arm.com 59910757SCurtis.Dunham@arm.com load_fault = inst->initiateAcc(); 60010757SCurtis.Dunham@arm.com 60110757SCurtis.Dunham@arm.com if (inst->isTranslationDelayed() && 6029816Sjthestness@gmail.com load_fault == NoFault) 6039816Sjthestness@gmail.com return load_fault; 6049816Sjthestness@gmail.com 6059816Sjthestness@gmail.com // If the instruction faulted or predicated false, then we need to send it 6069816Sjthestness@gmail.com // along to commit without the instruction completing. 6079816Sjthestness@gmail.com if (load_fault != NoFault || inst->readPredicate() == false) { 6089816Sjthestness@gmail.com // Send this instruction to commit, also make sure iew stage 6099816Sjthestness@gmail.com // realizes there is activity. 6109816Sjthestness@gmail.com // Mark it as executed unless it is an uncached load that 6119816Sjthestness@gmail.com // needs to hit the head of commit. 6129816Sjthestness@gmail.com if (inst->readPredicate() == false) 6139816Sjthestness@gmail.com inst->forwardOldRegs(); 6149816Sjthestness@gmail.com DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n", 6159816Sjthestness@gmail.com inst->seqNum, 6169816Sjthestness@gmail.com (load_fault != NoFault ? "fault" : "predication")); 6179816Sjthestness@gmail.com if (!(inst->hasRequest() && inst->uncacheable()) || 6189816Sjthestness@gmail.com inst->isAtCommit()) { 6199867Sjthestness@gmail.com inst->setExecuted(); 6209867Sjthestness@gmail.com } 6219867Sjthestness@gmail.com iewStage->instToCommit(inst); 6229867Sjthestness@gmail.com iewStage->activityThisCycle(); 6239867Sjthestness@gmail.com } else if (!loadBlocked()) { 6249816Sjthestness@gmail.com assert(inst->effAddrValid()); 6259816Sjthestness@gmail.com int load_idx = inst->lqIdx; 6269816Sjthestness@gmail.com incrLdIdx(load_idx); 6279816Sjthestness@gmail.com 6289816Sjthestness@gmail.com if (checkLoads) 6299816Sjthestness@gmail.com return checkViolations(load_idx, inst); 6309816Sjthestness@gmail.com } 6319816Sjthestness@gmail.com 6329816Sjthestness@gmail.com return load_fault; 6339816Sjthestness@gmail.com} 6349816Sjthestness@gmail.com 6359816Sjthestness@gmail.comtemplate <class Impl> 6369816Sjthestness@gmail.comFault 6373481Shsul@eecs.umich.eduLSQUnit<Impl>::executeStore(DynInstPtr &store_inst) 6385361Srstrong@cs.ucsd.edu{ 6395361Srstrong@cs.ucsd.edu using namespace TheISA; 6405361Srstrong@cs.ucsd.edu // Make sure that a store exists. 6415361Srstrong@cs.ucsd.edu assert(stores != 0); 6425361Srstrong@cs.ucsd.edu 6435361Srstrong@cs.ucsd.edu int store_idx = store_inst->sqIdx; 6445361Srstrong@cs.ucsd.edu 6455361Srstrong@cs.ucsd.edu DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n", 6465361Srstrong@cs.ucsd.edu store_inst->pcState(), store_inst->seqNum); 6475361Srstrong@cs.ucsd.edu 6485361Srstrong@cs.ucsd.edu assert(!store_inst->isSquashed()); 6497766Sgblack@eecs.umich.edu 6503395Shsul@eecs.umich.edu // Check the recently completed loads to see if any match this store's 6519521SAndreas.Sandberg@ARM.com // address. If so, then we have a memory ordering violation. 6523395Shsul@eecs.umich.edu int load_idx = store_inst->lqIdx; 6533481Shsul@eecs.umich.edu 6545361Srstrong@cs.ucsd.edu Fault store_fault = store_inst->initiateAcc(); 6555361Srstrong@cs.ucsd.edu 6565361Srstrong@cs.ucsd.edu if (store_inst->isTranslationDelayed() && 6575361Srstrong@cs.ucsd.edu store_fault == NoFault) 6585361Srstrong@cs.ucsd.edu return store_fault; 6595361Srstrong@cs.ucsd.edu 6605361Srstrong@cs.ucsd.edu if (store_inst->readPredicate() == false) 6619151Satgutier@umich.edu store_inst->forwardOldRegs(); 6627766Sgblack@eecs.umich.edu 6635361Srstrong@cs.ucsd.edu if (storeQueue[store_idx].size == 0) { 6645361Srstrong@cs.ucsd.edu DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n", 6659521SAndreas.Sandberg@ARM.com store_inst->pcState(), store_inst->seqNum); 6663395Shsul@eecs.umich.edu 6677489Ssteve.reinhardt@amd.com return store_fault; 6687489Ssteve.reinhardt@amd.com } else if (store_inst->readPredicate() == false) { 6697489Ssteve.reinhardt@amd.com DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n", 6707489Ssteve.reinhardt@amd.com store_inst->seqNum); 67110608Sdam.sunwoo@arm.com return store_fault; 67210608Sdam.sunwoo@arm.com } 67310608Sdam.sunwoo@arm.com 6747489Ssteve.reinhardt@amd.com assert(store_fault == NoFault); 6757489Ssteve.reinhardt@amd.com 6767489Ssteve.reinhardt@amd.com if (store_inst->isStoreConditional()) { 6777489Ssteve.reinhardt@amd.com // Store conditionals need to set themselves as able to 6787489Ssteve.reinhardt@amd.com // writeback if we haven't had a fault by here. 6795369Ssaidi@eecs.umich.edu storeQueue[store_idx].canWB = true; 6809140Snilay@cs.wisc.edu 6819140Snilay@cs.wisc.edu ++storesToWB; 6829140Snilay@cs.wisc.edu } 6839140Snilay@cs.wisc.edu 6849606Snilay@cs.wisc.edu return checkViolations(load_idx, store_inst); 68510608Sdam.sunwoo@arm.com 68610608Sdam.sunwoo@arm.com} 68710608Sdam.sunwoo@arm.com 68810608Sdam.sunwoo@arm.comtemplate <class Impl> 68910608Sdam.sunwoo@arm.comvoid 69010608Sdam.sunwoo@arm.comLSQUnit<Impl>::commitLoad() 69110608Sdam.sunwoo@arm.com{ 69210608Sdam.sunwoo@arm.com assert(loadQueue[loadHead]); 69310608Sdam.sunwoo@arm.com 6949140Snilay@cs.wisc.edu DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n", 6959151Satgutier@umich.edu loadQueue[loadHead]->pcState()); 6969151Satgutier@umich.edu 6979151Satgutier@umich.edu loadQueue[loadHead] = NULL; 6989151Satgutier@umich.edu 6999140Snilay@cs.wisc.edu incrLdIdx(loadHead); 7009140Snilay@cs.wisc.edu 7019151Satgutier@umich.edu --loads; 7029460Ssaidi@eecs.umich.edu} 7039151Satgutier@umich.edu 7049151Satgutier@umich.edutemplate <class Impl> 7059460Ssaidi@eecs.umich.eduvoid 7063395Shsul@eecs.umich.eduLSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst) 7079460Ssaidi@eecs.umich.edu{ 7086776SBrad.Beckmann@amd.com assert(loads == 0 || loadQueue[loadHead]); 7097525Ssteve.reinhardt@amd.com 7109457Svilanova@ac.upc.edu while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) { 7119494Sandreas@sandberg.pp.se commitLoad(); 7129494Sandreas@sandberg.pp.se } 713} 714 715template <class Impl> 716void 717LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst) 718{ 719 assert(stores == 0 || storeQueue[storeHead].inst); 720 721 int store_idx = storeHead; 722 723 while (store_idx != storeTail) { 724 assert(storeQueue[store_idx].inst); 725 // Mark any stores that are now committed and have not yet 726 // been marked as able to write back. 727 if (!storeQueue[store_idx].canWB) { 728 if (storeQueue[store_idx].inst->seqNum > youngest_inst) { 729 break; 730 } 731 DPRINTF(LSQUnit, "Marking store as able to write back, PC " 732 "%s [sn:%lli]\n", 733 storeQueue[store_idx].inst->pcState(), 734 storeQueue[store_idx].inst->seqNum); 735 736 storeQueue[store_idx].canWB = true; 737 738 ++storesToWB; 739 } 740 741 incrStIdx(store_idx); 742 } 743} 744 745template <class Impl> 746void 747LSQUnit<Impl>::writebackPendingStore() 748{ 749 if (hasPendingPkt) { 750 assert(pendingPkt != NULL); 751 752 // If the cache is blocked, this will store the packet for retry. 753 if (sendStore(pendingPkt)) { 754 storePostSend(pendingPkt); 755 } 756 pendingPkt = NULL; 757 hasPendingPkt = false; 758 } 759} 760 761template <class Impl> 762void 763LSQUnit<Impl>::writebackStores() 764{ 765 // First writeback the second packet from any split store that didn't 766 // complete last cycle because there weren't enough cache ports available. 767 if (TheISA::HasUnalignedMemAcc) { 768 writebackPendingStore(); 769 } 770 771 while (storesToWB > 0 && 772 storeWBIdx != storeTail && 773 storeQueue[storeWBIdx].inst && 774 storeQueue[storeWBIdx].canWB && 775 ((!needsTSO) || (!storeInFlight)) && 776 usedPorts < cachePorts) { 777 778 if (isStoreBlocked || lsq->cacheBlocked()) { 779 DPRINTF(LSQUnit, "Unable to write back any more stores, cache" 780 " is blocked!\n"); 781 break; 782 } 783 784 // Store didn't write any data so no need to write it back to 785 // memory. 786 if (storeQueue[storeWBIdx].size == 0) { 787 completeStore(storeWBIdx); 788 789 incrStIdx(storeWBIdx); 790 791 continue; 792 } 793 794 ++usedPorts; 795 796 if (storeQueue[storeWBIdx].inst->isDataPrefetch()) { 797 incrStIdx(storeWBIdx); 798 799 continue; 800 } 801 802 assert(storeQueue[storeWBIdx].req); 803 assert(!storeQueue[storeWBIdx].committed); 804 805 if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) { 806 assert(storeQueue[storeWBIdx].sreqLow); 807 assert(storeQueue[storeWBIdx].sreqHigh); 808 } 809 810 DynInstPtr inst = storeQueue[storeWBIdx].inst; 811 812 Request *req = storeQueue[storeWBIdx].req; 813 RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow; 814 RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh; 815 816 storeQueue[storeWBIdx].committed = true; 817 818 assert(!inst->memData); 819 inst->memData = new uint8_t[64]; 820 821 memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize()); 822 823 MemCmd command = 824 req->isSwap() ? MemCmd::SwapReq : 825 (req->isLLSC() ? MemCmd::StoreCondReq : MemCmd::WriteReq); 826 PacketPtr data_pkt; 827 PacketPtr snd_data_pkt = NULL; 828 829 LSQSenderState *state = new LSQSenderState; 830 state->isLoad = false; 831 state->idx = storeWBIdx; 832 state->inst = inst; 833 834 if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) { 835 836 // Build a single data packet if the store isn't split. 837 data_pkt = new Packet(req, command); 838 data_pkt->dataStatic(inst->memData); 839 data_pkt->senderState = state; 840 } else { 841 // Create two packets if the store is split in two. 842 data_pkt = new Packet(sreqLow, command); 843 snd_data_pkt = new Packet(sreqHigh, command); 844 845 data_pkt->dataStatic(inst->memData); 846 snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize()); 847 848 data_pkt->senderState = state; 849 snd_data_pkt->senderState = state; 850 851 state->isSplit = true; 852 state->outstanding = 2; 853 854 // Can delete the main request now. 855 delete req; 856 req = sreqLow; 857 } 858 859 DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s " 860 "to Addr:%#x, data:%#x [sn:%lli]\n", 861 storeWBIdx, inst->pcState(), 862 req->getPaddr(), (int)*(inst->memData), 863 inst->seqNum); 864 865 // @todo: Remove this SC hack once the memory system handles it. 866 if (inst->isStoreConditional()) { 867 assert(!storeQueue[storeWBIdx].isSplit); 868 // Disable recording the result temporarily. Writing to 869 // misc regs normally updates the result, but this is not 870 // the desired behavior when handling store conditionals. 871 inst->recordResult(false); 872 bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask); 873 inst->recordResult(true); 874 875 if (!success) { 876 // Instantly complete this store. 877 DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed. " 878 "Instantly completing it.\n", 879 inst->seqNum); 880 WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this); 881 cpu->schedule(wb, curTick() + 1); 882 if (cpu->checker) { 883 // Make sure to set the LLSC data for verification 884 // if checker is loaded 885 inst->reqToVerify->setExtraData(0); 886 inst->completeAcc(data_pkt); 887 } 888 completeStore(storeWBIdx); 889 incrStIdx(storeWBIdx); 890 continue; 891 } 892 } else { 893 // Non-store conditionals do not need a writeback. 894 state->noWB = true; 895 } 896 897 bool split = 898 TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit; 899 900 ThreadContext *thread = cpu->tcBase(lsqID); 901 902 if (req->isMmappedIpr()) { 903 assert(!inst->isStoreConditional()); 904 TheISA::handleIprWrite(thread, data_pkt); 905 delete data_pkt; 906 if (split) { 907 assert(snd_data_pkt->req->isMmappedIpr()); 908 TheISA::handleIprWrite(thread, snd_data_pkt); 909 delete snd_data_pkt; 910 delete sreqLow; 911 delete sreqHigh; 912 } 913 delete state; 914 delete req; 915 completeStore(storeWBIdx); 916 incrStIdx(storeWBIdx); 917 } else if (!sendStore(data_pkt)) { 918 DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will" 919 "retry later\n", 920 inst->seqNum); 921 922 // Need to store the second packet, if split. 923 if (split) { 924 state->pktToSend = true; 925 state->pendingPacket = snd_data_pkt; 926 } 927 } else { 928 929 // If split, try to send the second packet too 930 if (split) { 931 assert(snd_data_pkt); 932 933 // Ensure there are enough ports to use. 934 if (usedPorts < cachePorts) { 935 ++usedPorts; 936 if (sendStore(snd_data_pkt)) { 937 storePostSend(snd_data_pkt); 938 } else { 939 DPRINTF(IEW, "D-Cache became blocked when writing" 940 " [sn:%lli] second packet, will retry later\n", 941 inst->seqNum); 942 } 943 } else { 944 945 // Store the packet for when there's free ports. 946 assert(pendingPkt == NULL); 947 pendingPkt = snd_data_pkt; 948 hasPendingPkt = true; 949 } 950 } else { 951 952 // Not a split store. 953 storePostSend(data_pkt); 954 } 955 } 956 } 957 958 // Not sure this should set it to 0. 959 usedPorts = 0; 960 961 assert(stores >= 0 && storesToWB >= 0); 962} 963 964/*template <class Impl> 965void 966LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum) 967{ 968 list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(), 969 mshrSeqNums.end(), 970 seqNum); 971 972 if (mshr_it != mshrSeqNums.end()) { 973 mshrSeqNums.erase(mshr_it); 974 DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size()); 975 } 976}*/ 977 978template <class Impl> 979void 980LSQUnit<Impl>::squash(const InstSeqNum &squashed_num) 981{ 982 DPRINTF(LSQUnit, "Squashing until [sn:%lli]!" 983 "(Loads:%i Stores:%i)\n", squashed_num, loads, stores); 984 985 int load_idx = loadTail; 986 decrLdIdx(load_idx); 987 988 while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) { 989 DPRINTF(LSQUnit,"Load Instruction PC %s squashed, " 990 "[sn:%lli]\n", 991 loadQueue[load_idx]->pcState(), 992 loadQueue[load_idx]->seqNum); 993 994 if (isStalled() && load_idx == stallingLoadIdx) { 995 stalled = false; 996 stallingStoreIsn = 0; 997 stallingLoadIdx = 0; 998 } 999 1000 // Clear the smart pointer to make sure it is decremented. 1001 loadQueue[load_idx]->setSquashed(); 1002 loadQueue[load_idx] = NULL; 1003 --loads; 1004 1005 // Inefficient! 1006 loadTail = load_idx; 1007 1008 decrLdIdx(load_idx); 1009 ++lsqSquashedLoads; 1010 } 1011 1012 if (isLoadBlocked) { 1013 if (squashed_num < blockedLoadSeqNum) { 1014 isLoadBlocked = false; 1015 loadBlockedHandled = false; 1016 blockedLoadSeqNum = 0; 1017 } 1018 } 1019 1020 if (memDepViolator && squashed_num < memDepViolator->seqNum) { 1021 memDepViolator = NULL; 1022 } 1023 1024 int store_idx = storeTail; 1025 decrStIdx(store_idx); 1026 1027 while (stores != 0 && 1028 storeQueue[store_idx].inst->seqNum > squashed_num) { 1029 // Instructions marked as can WB are already committed. 1030 if (storeQueue[store_idx].canWB) { 1031 break; 1032 } 1033 1034 DPRINTF(LSQUnit,"Store Instruction PC %s squashed, " 1035 "idx:%i [sn:%lli]\n", 1036 storeQueue[store_idx].inst->pcState(), 1037 store_idx, storeQueue[store_idx].inst->seqNum); 1038 1039 // I don't think this can happen. It should have been cleared 1040 // by the stalling load. 1041 if (isStalled() && 1042 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 1043 panic("Is stalled should have been cleared by stalling load!\n"); 1044 stalled = false; 1045 stallingStoreIsn = 0; 1046 } 1047 1048 // Clear the smart pointer to make sure it is decremented. 1049 storeQueue[store_idx].inst->setSquashed(); 1050 storeQueue[store_idx].inst = NULL; 1051 storeQueue[store_idx].canWB = 0; 1052 1053 // Must delete request now that it wasn't handed off to 1054 // memory. This is quite ugly. @todo: Figure out the proper 1055 // place to really handle request deletes. 1056 delete storeQueue[store_idx].req; 1057 if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) { 1058 delete storeQueue[store_idx].sreqLow; 1059 delete storeQueue[store_idx].sreqHigh; 1060 1061 storeQueue[store_idx].sreqLow = NULL; 1062 storeQueue[store_idx].sreqHigh = NULL; 1063 } 1064 1065 storeQueue[store_idx].req = NULL; 1066 --stores; 1067 1068 // Inefficient! 1069 storeTail = store_idx; 1070 1071 decrStIdx(store_idx); 1072 ++lsqSquashedStores; 1073 } 1074} 1075 1076template <class Impl> 1077void 1078LSQUnit<Impl>::storePostSend(PacketPtr pkt) 1079{ 1080 if (isStalled() && 1081 storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) { 1082 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1083 "load idx:%i\n", 1084 stallingStoreIsn, stallingLoadIdx); 1085 stalled = false; 1086 stallingStoreIsn = 0; 1087 iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 1088 } 1089 1090 if (!storeQueue[storeWBIdx].inst->isStoreConditional()) { 1091 // The store is basically completed at this time. This 1092 // only works so long as the checker doesn't try to 1093 // verify the value in memory for stores. 1094 storeQueue[storeWBIdx].inst->setCompleted(); 1095 1096 if (cpu->checker) { 1097 cpu->checker->verify(storeQueue[storeWBIdx].inst); 1098 } 1099 } 1100 1101 if (needsTSO) { 1102 storeInFlight = true; 1103 } 1104 1105 incrStIdx(storeWBIdx); 1106} 1107 1108template <class Impl> 1109void 1110LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt) 1111{ 1112 iewStage->wakeCPU(); 1113 1114 // Squashed instructions do not need to complete their access. 1115 if (inst->isSquashed()) { 1116 iewStage->decrWb(inst->seqNum); 1117 assert(!inst->isStore()); 1118 ++lsqIgnoredResponses; 1119 return; 1120 } 1121 1122 if (!inst->isExecuted()) { 1123 inst->setExecuted(); 1124 1125 // Complete access to copy data to proper place. 1126 inst->completeAcc(pkt); 1127 } 1128 1129 // Need to insert instruction into queue to commit 1130 iewStage->instToCommit(inst); 1131 1132 iewStage->activityThisCycle(); 1133 1134 // see if this load changed the PC 1135 iewStage->checkMisprediction(inst); 1136} 1137 1138template <class Impl> 1139void 1140LSQUnit<Impl>::completeStore(int store_idx) 1141{ 1142 assert(storeQueue[store_idx].inst); 1143 storeQueue[store_idx].completed = true; 1144 --storesToWB; 1145 // A bit conservative because a store completion may not free up entries, 1146 // but hopefully avoids two store completions in one cycle from making 1147 // the CPU tick twice. 1148 cpu->wakeCPU(); 1149 cpu->activityThisCycle(); 1150 1151 if (store_idx == storeHead) { 1152 do { 1153 incrStIdx(storeHead); 1154 1155 --stores; 1156 } while (storeQueue[storeHead].completed && 1157 storeHead != storeTail); 1158 1159 iewStage->updateLSQNextCycle = true; 1160 } 1161 1162 DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head " 1163 "idx:%i\n", 1164 storeQueue[store_idx].inst->seqNum, store_idx, storeHead); 1165 1166#if TRACING_ON 1167 if (DTRACE(O3PipeView)) { 1168 storeQueue[store_idx].inst->storeTick = 1169 curTick() - storeQueue[store_idx].inst->fetchTick; 1170 } 1171#endif 1172 1173 if (isStalled() && 1174 storeQueue[store_idx].inst->seqNum == stallingStoreIsn) { 1175 DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] " 1176 "load idx:%i\n", 1177 stallingStoreIsn, stallingLoadIdx); 1178 stalled = false; 1179 stallingStoreIsn = 0; 1180 iewStage->replayMemInst(loadQueue[stallingLoadIdx]); 1181 } 1182 1183 storeQueue[store_idx].inst->setCompleted(); 1184 1185 if (needsTSO) { 1186 storeInFlight = false; 1187 } 1188 1189 // Tell the checker we've completed this instruction. Some stores 1190 // may get reported twice to the checker, but the checker can 1191 // handle that case. 1192 if (cpu->checker) { 1193 cpu->checker->verify(storeQueue[store_idx].inst); 1194 } 1195} 1196 1197template <class Impl> 1198bool 1199LSQUnit<Impl>::sendStore(PacketPtr data_pkt) 1200{ 1201 if (!dcachePort->sendTimingReq(data_pkt)) { 1202 // Need to handle becoming blocked on a store. 1203 isStoreBlocked = true; 1204 ++lsqCacheBlocked; 1205 assert(retryPkt == NULL); 1206 retryPkt = data_pkt; 1207 lsq->setRetryTid(lsqID); 1208 return false; 1209 } 1210 return true; 1211} 1212 1213template <class Impl> 1214void 1215LSQUnit<Impl>::recvRetry() 1216{ 1217 if (isStoreBlocked) { 1218 DPRINTF(LSQUnit, "Receiving retry: store blocked\n"); 1219 assert(retryPkt != NULL); 1220 1221 LSQSenderState *state = 1222 dynamic_cast<LSQSenderState *>(retryPkt->senderState); 1223 1224 if (dcachePort->sendTimingReq(retryPkt)) { 1225 // Don't finish the store unless this is the last packet. 1226 if (!TheISA::HasUnalignedMemAcc || !state->pktToSend || 1227 state->pendingPacket == retryPkt) { 1228 state->pktToSend = false; 1229 storePostSend(retryPkt); 1230 } 1231 retryPkt = NULL; 1232 isStoreBlocked = false; 1233 lsq->setRetryTid(InvalidThreadID); 1234 1235 // Send any outstanding packet. 1236 if (TheISA::HasUnalignedMemAcc && state->pktToSend) { 1237 assert(state->pendingPacket); 1238 if (sendStore(state->pendingPacket)) { 1239 storePostSend(state->pendingPacket); 1240 } 1241 } 1242 } else { 1243 // Still blocked! 1244 ++lsqCacheBlocked; 1245 lsq->setRetryTid(lsqID); 1246 } 1247 } else if (isLoadBlocked) { 1248 DPRINTF(LSQUnit, "Loads squash themselves and all younger insts, " 1249 "no need to resend packet.\n"); 1250 } else { 1251 DPRINTF(LSQUnit, "Retry received but LSQ is no longer blocked.\n"); 1252 } 1253} 1254 1255template <class Impl> 1256inline void 1257LSQUnit<Impl>::incrStIdx(int &store_idx) const 1258{ 1259 if (++store_idx >= SQEntries) 1260 store_idx = 0; 1261} 1262 1263template <class Impl> 1264inline void 1265LSQUnit<Impl>::decrStIdx(int &store_idx) const 1266{ 1267 if (--store_idx < 0) 1268 store_idx += SQEntries; 1269} 1270 1271template <class Impl> 1272inline void 1273LSQUnit<Impl>::incrLdIdx(int &load_idx) const 1274{ 1275 if (++load_idx >= LQEntries) 1276 load_idx = 0; 1277} 1278 1279template <class Impl> 1280inline void 1281LSQUnit<Impl>::decrLdIdx(int &load_idx) const 1282{ 1283 if (--load_idx < 0) 1284 load_idx += LQEntries; 1285} 1286 1287template <class Impl> 1288void 1289LSQUnit<Impl>::dumpInsts() const 1290{ 1291 cprintf("Load store queue: Dumping instructions.\n"); 1292 cprintf("Load queue size: %i\n", loads); 1293 cprintf("Load queue: "); 1294 1295 int load_idx = loadHead; 1296 1297 while (load_idx != loadTail && loadQueue[load_idx]) { 1298 const DynInstPtr &inst(loadQueue[load_idx]); 1299 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1300 1301 incrLdIdx(load_idx); 1302 } 1303 cprintf("\n"); 1304 1305 cprintf("Store queue size: %i\n", stores); 1306 cprintf("Store queue: "); 1307 1308 int store_idx = storeHead; 1309 1310 while (store_idx != storeTail && storeQueue[store_idx].inst) { 1311 const DynInstPtr &inst(storeQueue[store_idx].inst); 1312 cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum); 1313 1314 incrStIdx(store_idx); 1315 } 1316 1317 cprintf("\n"); 1318} 1319 1320#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__ 1321