lsq_unit.hh revision 7511
12292SN/A/* 22329SN/A * Copyright (c) 2004-2006 The Regents of The University of Michigan 32292SN/A * All rights reserved. 42292SN/A * 52292SN/A * Redistribution and use in source and binary forms, with or without 62292SN/A * modification, are permitted provided that the following conditions are 72292SN/A * met: redistributions of source code must retain the above copyright 82292SN/A * notice, this list of conditions and the following disclaimer; 92292SN/A * redistributions in binary form must reproduce the above copyright 102292SN/A * notice, this list of conditions and the following disclaimer in the 112292SN/A * documentation and/or other materials provided with the distribution; 122292SN/A * neither the name of the copyright holders nor the names of its 132292SN/A * contributors may be used to endorse or promote products derived from 142292SN/A * this software without specific prior written permission. 152292SN/A * 162292SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 172292SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 182292SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 192292SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 202292SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 212292SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 222292SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 232292SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 242292SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 252292SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 262292SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 272689Sktlim@umich.edu * 282689Sktlim@umich.edu * Authors: Kevin Lim 292689Sktlim@umich.edu * Korey Sewell 302292SN/A */ 312292SN/A 322292SN/A#ifndef __CPU_O3_LSQ_UNIT_HH__ 332292SN/A#define __CPU_O3_LSQ_UNIT_HH__ 342292SN/A 352329SN/A#include <algorithm> 364395Ssaidi@eecs.umich.edu#include <cstring> 372292SN/A#include <map> 382292SN/A#include <queue> 392292SN/A 402329SN/A#include "arch/faults.hh" 413326Sktlim@umich.edu#include "arch/locked_mem.hh" 422292SN/A#include "config/full_system.hh" 436658Snate@binkert.org#include "config/the_isa.hh" 445386Sstever@gmail.com#include "base/fast_alloc.hh" 452292SN/A#include "base/hashmap.hh" 462292SN/A#include "cpu/inst_seq.hh" 473348Sbinkertn@umich.edu#include "mem/packet.hh" 482669Sktlim@umich.edu#include "mem/port.hh" 492292SN/A 505529Snate@binkert.orgclass DerivO3CPUParams; 515529Snate@binkert.org 522292SN/A/** 532329SN/A * Class that implements the actual LQ and SQ for each specific 542329SN/A * thread. Both are circular queues; load entries are freed upon 552329SN/A * committing, while store entries are freed once they writeback. The 562329SN/A * LSQUnit tracks if there are memory ordering violations, and also 572329SN/A * detects partial load to store forwarding cases (a store only has 582329SN/A * part of a load's data) that requires the load to wait until the 592329SN/A * store writes back. In the former case it holds onto the instruction 602329SN/A * until the dependence unit looks at it, and in the latter it stalls 612329SN/A * the LSQ until the store writes back. At that point the load is 622329SN/A * replayed. 632292SN/A */ 642292SN/Atemplate <class Impl> 652292SN/Aclass LSQUnit { 662292SN/A protected: 672292SN/A typedef TheISA::IntReg IntReg; 682292SN/A public: 692733Sktlim@umich.edu typedef typename Impl::O3CPU O3CPU; 702292SN/A typedef typename Impl::DynInstPtr DynInstPtr; 712292SN/A typedef typename Impl::CPUPol::IEW IEW; 722907Sktlim@umich.edu typedef typename Impl::CPUPol::LSQ LSQ; 732292SN/A typedef typename Impl::CPUPol::IssueStruct IssueStruct; 742292SN/A 752292SN/A public: 762292SN/A /** Constructs an LSQ unit. init() must be called prior to use. */ 772292SN/A LSQUnit(); 782292SN/A 792292SN/A /** Initializes the LSQ unit with the specified number of entries. */ 805529Snate@binkert.org void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 815529Snate@binkert.org LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 825529Snate@binkert.org unsigned id); 832292SN/A 842292SN/A /** Returns the name of the LSQ unit. */ 852292SN/A std::string name() const; 862292SN/A 872727Sktlim@umich.edu /** Registers statistics. */ 882727Sktlim@umich.edu void regStats(); 892727Sktlim@umich.edu 902907Sktlim@umich.edu /** Sets the pointer to the dcache port. */ 914329Sktlim@umich.edu void setDcachePort(Port *dcache_port); 922907Sktlim@umich.edu 932348SN/A /** Switches out LSQ unit. */ 942307SN/A void switchOut(); 952307SN/A 962348SN/A /** Takes over from another CPU's thread. */ 972307SN/A void takeOverFrom(); 982307SN/A 992348SN/A /** Returns if the LSQ is switched out. */ 1002307SN/A bool isSwitchedOut() { return switchedOut; } 1012307SN/A 1022292SN/A /** Ticks the LSQ unit, which in this case only resets the number of 1032292SN/A * used cache ports. 1042292SN/A * @todo: Move the number of used ports up to the LSQ level so it can 1052292SN/A * be shared by all LSQ units. 1062292SN/A */ 1072292SN/A void tick() { usedPorts = 0; } 1082292SN/A 1092292SN/A /** Inserts an instruction. */ 1102292SN/A void insert(DynInstPtr &inst); 1112292SN/A /** Inserts a load instruction. */ 1122292SN/A void insertLoad(DynInstPtr &load_inst); 1132292SN/A /** Inserts a store instruction. */ 1142292SN/A void insertStore(DynInstPtr &store_inst); 1152292SN/A 1162292SN/A /** Executes a load instruction. */ 1172292SN/A Fault executeLoad(DynInstPtr &inst); 1182292SN/A 1192329SN/A Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 1202292SN/A /** Executes a store instruction. */ 1212292SN/A Fault executeStore(DynInstPtr &inst); 1222292SN/A 1232292SN/A /** Commits the head load. */ 1242292SN/A void commitLoad(); 1252292SN/A /** Commits loads older than a specific sequence number. */ 1262292SN/A void commitLoads(InstSeqNum &youngest_inst); 1272292SN/A 1282292SN/A /** Commits stores older than a specific sequence number. */ 1292292SN/A void commitStores(InstSeqNum &youngest_inst); 1302292SN/A 1312292SN/A /** Writes back stores. */ 1322292SN/A void writebackStores(); 1332292SN/A 1342790Sktlim@umich.edu /** Completes the data access that has been returned from the 1352790Sktlim@umich.edu * memory system. */ 1362669Sktlim@umich.edu void completeDataAccess(PacketPtr pkt); 1372669Sktlim@umich.edu 1382292SN/A /** Clears all the entries in the LQ. */ 1392292SN/A void clearLQ(); 1402292SN/A 1412292SN/A /** Clears all the entries in the SQ. */ 1422292SN/A void clearSQ(); 1432292SN/A 1442292SN/A /** Resizes the LQ to a given size. */ 1452292SN/A void resizeLQ(unsigned size); 1462292SN/A 1472292SN/A /** Resizes the SQ to a given size. */ 1482292SN/A void resizeSQ(unsigned size); 1492292SN/A 1502292SN/A /** Squashes all instructions younger than a specific sequence number. */ 1512292SN/A void squash(const InstSeqNum &squashed_num); 1522292SN/A 1532292SN/A /** Returns if there is a memory ordering violation. Value is reset upon 1542292SN/A * call to getMemDepViolator(). 1552292SN/A */ 1562292SN/A bool violation() { return memDepViolator; } 1572292SN/A 1582292SN/A /** Returns the memory ordering violator. */ 1592292SN/A DynInstPtr getMemDepViolator(); 1602292SN/A 1612329SN/A /** Returns if a load became blocked due to the memory system. */ 1622292SN/A bool loadBlocked() 1632292SN/A { return isLoadBlocked; } 1642292SN/A 1652348SN/A /** Clears the signal that a load became blocked. */ 1662292SN/A void clearLoadBlocked() 1672292SN/A { isLoadBlocked = false; } 1682292SN/A 1692348SN/A /** Returns if the blocked load was handled. */ 1702292SN/A bool isLoadBlockedHandled() 1712292SN/A { return loadBlockedHandled; } 1722292SN/A 1732348SN/A /** Records the blocked load as being handled. */ 1742292SN/A void setLoadBlockedHandled() 1752292SN/A { loadBlockedHandled = true; } 1762292SN/A 1772292SN/A /** Returns the number of free entries (min of free LQ and SQ entries). */ 1782292SN/A unsigned numFreeEntries(); 1792292SN/A 1802292SN/A /** Returns the number of loads ready to execute. */ 1812292SN/A int numLoadsReady(); 1822292SN/A 1832292SN/A /** Returns the number of loads in the LQ. */ 1842292SN/A int numLoads() { return loads; } 1852292SN/A 1862292SN/A /** Returns the number of stores in the SQ. */ 1872292SN/A int numStores() { return stores; } 1882292SN/A 1892292SN/A /** Returns if either the LQ or SQ is full. */ 1902292SN/A bool isFull() { return lqFull() || sqFull(); } 1912292SN/A 1922292SN/A /** Returns if the LQ is full. */ 1932292SN/A bool lqFull() { return loads >= (LQEntries - 1); } 1942292SN/A 1952292SN/A /** Returns if the SQ is full. */ 1962292SN/A bool sqFull() { return stores >= (SQEntries - 1); } 1972292SN/A 1982292SN/A /** Returns the number of instructions in the LSQ. */ 1992292SN/A unsigned getCount() { return loads + stores; } 2002292SN/A 2012292SN/A /** Returns if there are any stores to writeback. */ 2022292SN/A bool hasStoresToWB() { return storesToWB; } 2032292SN/A 2042292SN/A /** Returns the number of stores to writeback. */ 2052292SN/A int numStoresToWB() { return storesToWB; } 2062292SN/A 2072292SN/A /** Returns if the LSQ unit will writeback on this cycle. */ 2082292SN/A bool willWB() { return storeQueue[storeWBIdx].canWB && 2092678Sktlim@umich.edu !storeQueue[storeWBIdx].completed && 2102678Sktlim@umich.edu !isStoreBlocked; } 2112292SN/A 2122907Sktlim@umich.edu /** Handles doing the retry. */ 2132907Sktlim@umich.edu void recvRetry(); 2142907Sktlim@umich.edu 2152292SN/A private: 2162698Sktlim@umich.edu /** Writes back the instruction, sending it to IEW. */ 2172678Sktlim@umich.edu void writeback(DynInstPtr &inst, PacketPtr pkt); 2182678Sktlim@umich.edu 2196974Stjones1@inf.ed.ac.uk /** Writes back a store that couldn't be completed the previous cycle. */ 2206974Stjones1@inf.ed.ac.uk void writebackPendingStore(); 2216974Stjones1@inf.ed.ac.uk 2222698Sktlim@umich.edu /** Handles completing the send of a store to memory. */ 2233349Sbinkertn@umich.edu void storePostSend(PacketPtr pkt); 2242693Sktlim@umich.edu 2252292SN/A /** Completes the store at the specified index. */ 2262292SN/A void completeStore(int store_idx); 2272292SN/A 2286974Stjones1@inf.ed.ac.uk /** Attempts to send a store to the cache. */ 2296974Stjones1@inf.ed.ac.uk bool sendStore(PacketPtr data_pkt); 2306974Stjones1@inf.ed.ac.uk 2312292SN/A /** Increments the given store index (circular queue). */ 2322292SN/A inline void incrStIdx(int &store_idx); 2332292SN/A /** Decrements the given store index (circular queue). */ 2342292SN/A inline void decrStIdx(int &store_idx); 2352292SN/A /** Increments the given load index (circular queue). */ 2362292SN/A inline void incrLdIdx(int &load_idx); 2372292SN/A /** Decrements the given load index (circular queue). */ 2382292SN/A inline void decrLdIdx(int &load_idx); 2392292SN/A 2402329SN/A public: 2412329SN/A /** Debugging function to dump instructions in the LSQ. */ 2422329SN/A void dumpInsts(); 2432329SN/A 2442292SN/A private: 2452292SN/A /** Pointer to the CPU. */ 2462733Sktlim@umich.edu O3CPU *cpu; 2472292SN/A 2482292SN/A /** Pointer to the IEW stage. */ 2492292SN/A IEW *iewStage; 2502292SN/A 2512907Sktlim@umich.edu /** Pointer to the LSQ. */ 2522907Sktlim@umich.edu LSQ *lsq; 2532669Sktlim@umich.edu 2542907Sktlim@umich.edu /** Pointer to the dcache port. Used only for sending. */ 2552907Sktlim@umich.edu Port *dcachePort; 2562292SN/A 2572698Sktlim@umich.edu /** Derived class to hold any sender state the LSQ needs. */ 2585386Sstever@gmail.com class LSQSenderState : public Packet::SenderState, public FastAlloc 2592678Sktlim@umich.edu { 2602678Sktlim@umich.edu public: 2612698Sktlim@umich.edu /** Default constructor. */ 2622678Sktlim@umich.edu LSQSenderState() 2636974Stjones1@inf.ed.ac.uk : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 2646974Stjones1@inf.ed.ac.uk mainPkt(NULL), pendingPacket(NULL) 2652678Sktlim@umich.edu { } 2662678Sktlim@umich.edu 2672698Sktlim@umich.edu /** Instruction who initiated the access to memory. */ 2682678Sktlim@umich.edu DynInstPtr inst; 2692698Sktlim@umich.edu /** Whether or not it is a load. */ 2702678Sktlim@umich.edu bool isLoad; 2712698Sktlim@umich.edu /** The LQ/SQ index of the instruction. */ 2722678Sktlim@umich.edu int idx; 2732698Sktlim@umich.edu /** Whether or not the instruction will need to writeback. */ 2742678Sktlim@umich.edu bool noWB; 2756974Stjones1@inf.ed.ac.uk /** Whether or not this access is split in two. */ 2766974Stjones1@inf.ed.ac.uk bool isSplit; 2776974Stjones1@inf.ed.ac.uk /** Whether or not there is a packet that needs sending. */ 2786974Stjones1@inf.ed.ac.uk bool pktToSend; 2796974Stjones1@inf.ed.ac.uk /** Number of outstanding packets to complete. */ 2806974Stjones1@inf.ed.ac.uk int outstanding; 2816974Stjones1@inf.ed.ac.uk /** The main packet from a split load, used during writeback. */ 2826974Stjones1@inf.ed.ac.uk PacketPtr mainPkt; 2836974Stjones1@inf.ed.ac.uk /** A second packet from a split store that needs sending. */ 2846974Stjones1@inf.ed.ac.uk PacketPtr pendingPacket; 2856974Stjones1@inf.ed.ac.uk 2866974Stjones1@inf.ed.ac.uk /** Completes a packet and returns whether the access is finished. */ 2876974Stjones1@inf.ed.ac.uk inline bool complete() { return --outstanding == 0; } 2882678Sktlim@umich.edu }; 2892678Sktlim@umich.edu 2902698Sktlim@umich.edu /** Writeback event, specifically for when stores forward data to loads. */ 2912678Sktlim@umich.edu class WritebackEvent : public Event { 2922678Sktlim@umich.edu public: 2932678Sktlim@umich.edu /** Constructs a writeback event. */ 2942678Sktlim@umich.edu WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 2952678Sktlim@umich.edu 2962678Sktlim@umich.edu /** Processes the writeback event. */ 2972678Sktlim@umich.edu void process(); 2982678Sktlim@umich.edu 2992678Sktlim@umich.edu /** Returns the description of this event. */ 3005336Shines@cs.fsu.edu const char *description() const; 3012678Sktlim@umich.edu 3022678Sktlim@umich.edu private: 3032698Sktlim@umich.edu /** Instruction whose results are being written back. */ 3042678Sktlim@umich.edu DynInstPtr inst; 3052678Sktlim@umich.edu 3062698Sktlim@umich.edu /** The packet that would have been sent to memory. */ 3072678Sktlim@umich.edu PacketPtr pkt; 3082678Sktlim@umich.edu 3092678Sktlim@umich.edu /** The pointer to the LSQ unit that issued the store. */ 3102678Sktlim@umich.edu LSQUnit<Impl> *lsqPtr; 3112678Sktlim@umich.edu }; 3122678Sktlim@umich.edu 3132292SN/A public: 3142292SN/A struct SQEntry { 3152292SN/A /** Constructs an empty store queue entry. */ 3162292SN/A SQEntry() 3174326Sgblack@eecs.umich.edu : inst(NULL), req(NULL), size(0), 3182292SN/A canWB(0), committed(0), completed(0) 3194326Sgblack@eecs.umich.edu { 3204395Ssaidi@eecs.umich.edu std::memset(data, 0, sizeof(data)); 3214326Sgblack@eecs.umich.edu } 3222292SN/A 3232292SN/A /** Constructs a store queue entry for a given instruction. */ 3242292SN/A SQEntry(DynInstPtr &_inst) 3256974Stjones1@inf.ed.ac.uk : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 3266974Stjones1@inf.ed.ac.uk isSplit(0), canWB(0), committed(0), completed(0) 3274326Sgblack@eecs.umich.edu { 3284395Ssaidi@eecs.umich.edu std::memset(data, 0, sizeof(data)); 3294326Sgblack@eecs.umich.edu } 3302292SN/A 3312292SN/A /** The store instruction. */ 3322292SN/A DynInstPtr inst; 3332669Sktlim@umich.edu /** The request for the store. */ 3342669Sktlim@umich.edu RequestPtr req; 3356974Stjones1@inf.ed.ac.uk /** The split requests for the store. */ 3366974Stjones1@inf.ed.ac.uk RequestPtr sreqLow; 3376974Stjones1@inf.ed.ac.uk RequestPtr sreqHigh; 3382292SN/A /** The size of the store. */ 3392292SN/A int size; 3402292SN/A /** The store data. */ 3414326Sgblack@eecs.umich.edu char data[sizeof(IntReg)]; 3426974Stjones1@inf.ed.ac.uk /** Whether or not the store is split into two requests. */ 3436974Stjones1@inf.ed.ac.uk bool isSplit; 3442292SN/A /** Whether or not the store can writeback. */ 3452292SN/A bool canWB; 3462292SN/A /** Whether or not the store is committed. */ 3472292SN/A bool committed; 3482292SN/A /** Whether or not the store is completed. */ 3492292SN/A bool completed; 3502292SN/A }; 3512329SN/A 3522292SN/A private: 3532292SN/A /** The LSQUnit thread id. */ 3546221Snate@binkert.org ThreadID lsqID; 3552292SN/A 3562292SN/A /** The store queue. */ 3572292SN/A std::vector<SQEntry> storeQueue; 3582292SN/A 3592292SN/A /** The load queue. */ 3602292SN/A std::vector<DynInstPtr> loadQueue; 3612292SN/A 3622329SN/A /** The number of LQ entries, plus a sentinel entry (circular queue). 3632329SN/A * @todo: Consider having var that records the true number of LQ entries. 3642329SN/A */ 3652292SN/A unsigned LQEntries; 3662329SN/A /** The number of SQ entries, plus a sentinel entry (circular queue). 3672329SN/A * @todo: Consider having var that records the true number of SQ entries. 3682329SN/A */ 3692292SN/A unsigned SQEntries; 3702292SN/A 3712292SN/A /** The number of load instructions in the LQ. */ 3722292SN/A int loads; 3732329SN/A /** The number of store instructions in the SQ. */ 3742292SN/A int stores; 3752292SN/A /** The number of store instructions in the SQ waiting to writeback. */ 3762292SN/A int storesToWB; 3772292SN/A 3782292SN/A /** The index of the head instruction in the LQ. */ 3792292SN/A int loadHead; 3802292SN/A /** The index of the tail instruction in the LQ. */ 3812292SN/A int loadTail; 3822292SN/A 3832292SN/A /** The index of the head instruction in the SQ. */ 3842292SN/A int storeHead; 3852329SN/A /** The index of the first instruction that may be ready to be 3862329SN/A * written back, and has not yet been written back. 3872292SN/A */ 3882292SN/A int storeWBIdx; 3892292SN/A /** The index of the tail instruction in the SQ. */ 3902292SN/A int storeTail; 3912292SN/A 3922292SN/A /// @todo Consider moving to a more advanced model with write vs read ports 3932292SN/A /** The number of cache ports available each cycle. */ 3942292SN/A int cachePorts; 3952292SN/A 3962292SN/A /** The number of used cache ports in this cycle. */ 3972292SN/A int usedPorts; 3982292SN/A 3992348SN/A /** Is the LSQ switched out. */ 4002307SN/A bool switchedOut; 4012307SN/A 4022292SN/A //list<InstSeqNum> mshrSeqNums; 4032292SN/A 4042292SN/A /** Wire to read information from the issue stage time queue. */ 4052292SN/A typename TimeBuffer<IssueStruct>::wire fromIssue; 4062292SN/A 4072292SN/A /** Whether or not the LSQ is stalled. */ 4082292SN/A bool stalled; 4092292SN/A /** The store that causes the stall due to partial store to load 4102292SN/A * forwarding. 4112292SN/A */ 4122292SN/A InstSeqNum stallingStoreIsn; 4132292SN/A /** The index of the above store. */ 4142292SN/A int stallingLoadIdx; 4152292SN/A 4162698Sktlim@umich.edu /** The packet that needs to be retried. */ 4172698Sktlim@umich.edu PacketPtr retryPkt; 4182693Sktlim@umich.edu 4192698Sktlim@umich.edu /** Whehter or not a store is blocked due to the memory system. */ 4202678Sktlim@umich.edu bool isStoreBlocked; 4212678Sktlim@umich.edu 4222329SN/A /** Whether or not a load is blocked due to the memory system. */ 4232292SN/A bool isLoadBlocked; 4242292SN/A 4252348SN/A /** Has the blocked load been handled. */ 4262292SN/A bool loadBlockedHandled; 4272292SN/A 4282348SN/A /** The sequence number of the blocked load. */ 4292292SN/A InstSeqNum blockedLoadSeqNum; 4302292SN/A 4312292SN/A /** The oldest load that caused a memory ordering violation. */ 4322292SN/A DynInstPtr memDepViolator; 4332292SN/A 4346974Stjones1@inf.ed.ac.uk /** Whether or not there is a packet that couldn't be sent because of 4356974Stjones1@inf.ed.ac.uk * a lack of cache ports. */ 4366974Stjones1@inf.ed.ac.uk bool hasPendingPkt; 4376974Stjones1@inf.ed.ac.uk 4386974Stjones1@inf.ed.ac.uk /** The packet that is pending free cache ports. */ 4396974Stjones1@inf.ed.ac.uk PacketPtr pendingPkt; 4406974Stjones1@inf.ed.ac.uk 4412292SN/A // Will also need how many read/write ports the Dcache has. Or keep track 4422292SN/A // of that in stage that is one level up, and only call executeLoad/Store 4432292SN/A // the appropriate number of times. 4442727Sktlim@umich.edu /** Total number of loads forwaded from LSQ stores. */ 4455999Snate@binkert.org Stats::Scalar lsqForwLoads; 4462307SN/A 4473126Sktlim@umich.edu /** Total number of loads ignored due to invalid addresses. */ 4485999Snate@binkert.org Stats::Scalar invAddrLoads; 4493126Sktlim@umich.edu 4503126Sktlim@umich.edu /** Total number of squashed loads. */ 4515999Snate@binkert.org Stats::Scalar lsqSquashedLoads; 4523126Sktlim@umich.edu 4533126Sktlim@umich.edu /** Total number of responses from the memory system that are 4543126Sktlim@umich.edu * ignored due to the instruction already being squashed. */ 4555999Snate@binkert.org Stats::Scalar lsqIgnoredResponses; 4563126Sktlim@umich.edu 4573126Sktlim@umich.edu /** Tota number of memory ordering violations. */ 4585999Snate@binkert.org Stats::Scalar lsqMemOrderViolation; 4593126Sktlim@umich.edu 4602727Sktlim@umich.edu /** Total number of squashed stores. */ 4615999Snate@binkert.org Stats::Scalar lsqSquashedStores; 4622727Sktlim@umich.edu 4632727Sktlim@umich.edu /** Total number of software prefetches ignored due to invalid addresses. */ 4645999Snate@binkert.org Stats::Scalar invAddrSwpfs; 4652727Sktlim@umich.edu 4662727Sktlim@umich.edu /** Ready loads blocked due to partial store-forwarding. */ 4675999Snate@binkert.org Stats::Scalar lsqBlockedLoads; 4682727Sktlim@umich.edu 4692727Sktlim@umich.edu /** Number of loads that were rescheduled. */ 4705999Snate@binkert.org Stats::Scalar lsqRescheduledLoads; 4712727Sktlim@umich.edu 4722727Sktlim@umich.edu /** Number of times the LSQ is blocked due to the cache. */ 4735999Snate@binkert.org Stats::Scalar lsqCacheBlocked; 4742727Sktlim@umich.edu 4752292SN/A public: 4762292SN/A /** Executes the load at the given index. */ 4772292SN/A template <class T> 4786974Stjones1@inf.ed.ac.uk Fault read(Request *req, Request *sreqLow, Request *sreqHigh, T &data, 4796974Stjones1@inf.ed.ac.uk int load_idx); 4802292SN/A 4812292SN/A /** Executes the store at the given index. */ 4822292SN/A template <class T> 4836974Stjones1@inf.ed.ac.uk Fault write(Request *req, Request *sreqLow, Request *sreqHigh, T &data, 4846974Stjones1@inf.ed.ac.uk int store_idx); 4852292SN/A 4862292SN/A /** Returns the index of the head load instruction. */ 4872292SN/A int getLoadHead() { return loadHead; } 4882292SN/A /** Returns the sequence number of the head load instruction. */ 4892292SN/A InstSeqNum getLoadHeadSeqNum() 4902292SN/A { 4912292SN/A if (loadQueue[loadHead]) { 4922292SN/A return loadQueue[loadHead]->seqNum; 4932292SN/A } else { 4942292SN/A return 0; 4952292SN/A } 4962292SN/A 4972292SN/A } 4982292SN/A 4992292SN/A /** Returns the index of the head store instruction. */ 5002292SN/A int getStoreHead() { return storeHead; } 5012292SN/A /** Returns the sequence number of the head store instruction. */ 5022292SN/A InstSeqNum getStoreHeadSeqNum() 5032292SN/A { 5042292SN/A if (storeQueue[storeHead].inst) { 5052292SN/A return storeQueue[storeHead].inst->seqNum; 5062292SN/A } else { 5072292SN/A return 0; 5082292SN/A } 5092292SN/A 5102292SN/A } 5112292SN/A 5122292SN/A /** Returns whether or not the LSQ unit is stalled. */ 5132292SN/A bool isStalled() { return stalled; } 5142292SN/A}; 5152292SN/A 5162292SN/Atemplate <class Impl> 5172292SN/Atemplate <class T> 5182292SN/AFault 5196974Stjones1@inf.ed.ac.ukLSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 5206974Stjones1@inf.ed.ac.uk T &data, int load_idx) 5212292SN/A{ 5222669Sktlim@umich.edu DynInstPtr load_inst = loadQueue[load_idx]; 5232292SN/A 5242669Sktlim@umich.edu assert(load_inst); 5252669Sktlim@umich.edu 5262669Sktlim@umich.edu assert(!load_inst->isExecuted()); 5272292SN/A 5282292SN/A // Make sure this isn't an uncacheable access 5292292SN/A // A bit of a hackish way to get uncached accesses to work only if they're 5302292SN/A // at the head of the LSQ and are ready to commit (at the head of the ROB 5312292SN/A // too). 5323172Sstever@eecs.umich.edu if (req->isUncacheable() && 5332731Sktlim@umich.edu (load_idx != loadHead || !load_inst->isAtCommit())) { 5342669Sktlim@umich.edu iewStage->rescheduleMemInst(load_inst); 5352727Sktlim@umich.edu ++lsqRescheduledLoads; 5364032Sktlim@umich.edu 5374032Sktlim@umich.edu // Must delete request now that it wasn't handed off to 5384032Sktlim@umich.edu // memory. This is quite ugly. @todo: Figure out the proper 5394032Sktlim@umich.edu // place to really handle request deletes. 5404032Sktlim@umich.edu delete req; 5416974Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow) { 5426974Stjones1@inf.ed.ac.uk delete sreqLow; 5436974Stjones1@inf.ed.ac.uk delete sreqHigh; 5446974Stjones1@inf.ed.ac.uk } 5452292SN/A return TheISA::genMachineCheckFault(); 5462292SN/A } 5472292SN/A 5482292SN/A // Check the SQ for any previous stores that might lead to forwarding 5492669Sktlim@umich.edu int store_idx = load_inst->sqIdx; 5502292SN/A 5512292SN/A int store_size = 0; 5522292SN/A 5532292SN/A DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 5546974Stjones1@inf.ed.ac.uk "storeHead: %i addr: %#x%s\n", 5556974Stjones1@inf.ed.ac.uk load_idx, store_idx, storeHead, req->getPaddr(), 5566974Stjones1@inf.ed.ac.uk sreqLow ? " split" : ""); 5572292SN/A 5586102Sgblack@eecs.umich.edu if (req->isLLSC()) { 5596974Stjones1@inf.ed.ac.uk assert(!sreqLow); 5603326Sktlim@umich.edu // Disable recording the result temporarily. Writing to misc 5613326Sktlim@umich.edu // regs normally updates the result, but this is not the 5623326Sktlim@umich.edu // desired behavior when handling store conditionals. 5633326Sktlim@umich.edu load_inst->recordResult = false; 5643326Sktlim@umich.edu TheISA::handleLockedRead(load_inst.get(), req); 5653326Sktlim@umich.edu load_inst->recordResult = true; 5662292SN/A } 5672292SN/A 5682292SN/A while (store_idx != -1) { 5692292SN/A // End once we've reached the top of the LSQ 5702292SN/A if (store_idx == storeWBIdx) { 5712292SN/A break; 5722292SN/A } 5732292SN/A 5742292SN/A // Move the index to one younger 5752292SN/A if (--store_idx < 0) 5762292SN/A store_idx += SQEntries; 5772292SN/A 5782292SN/A assert(storeQueue[store_idx].inst); 5792292SN/A 5802292SN/A store_size = storeQueue[store_idx].size; 5812292SN/A 5822292SN/A if (store_size == 0) 5832292SN/A continue; 5844032Sktlim@umich.edu else if (storeQueue[store_idx].inst->uncacheable()) 5854032Sktlim@umich.edu continue; 5864032Sktlim@umich.edu 5874032Sktlim@umich.edu assert(storeQueue[store_idx].inst->effAddrValid); 5882292SN/A 5892292SN/A // Check if the store data is within the lower and upper bounds of 5902292SN/A // addresses that the request needs. 5912292SN/A bool store_has_lower_limit = 5922669Sktlim@umich.edu req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 5932292SN/A bool store_has_upper_limit = 5942669Sktlim@umich.edu (req->getVaddr() + req->getSize()) <= 5952669Sktlim@umich.edu (storeQueue[store_idx].inst->effAddr + store_size); 5962292SN/A bool lower_load_has_store_part = 5972669Sktlim@umich.edu req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 5982292SN/A store_size); 5992292SN/A bool upper_load_has_store_part = 6002669Sktlim@umich.edu (req->getVaddr() + req->getSize()) > 6012669Sktlim@umich.edu storeQueue[store_idx].inst->effAddr; 6022292SN/A 6032292SN/A // If the store's data has all of the data needed, we can forward. 6044032Sktlim@umich.edu if ((store_has_lower_limit && store_has_upper_limit)) { 6052329SN/A // Get shift amount for offset into the store's data. 6062669Sktlim@umich.edu int shift_amt = req->getVaddr() & (store_size - 1); 6072292SN/A 6084326Sgblack@eecs.umich.edu memcpy(&data, storeQueue[store_idx].data + shift_amt, sizeof(T)); 6093803Sgblack@eecs.umich.edu 6102669Sktlim@umich.edu assert(!load_inst->memData); 6112669Sktlim@umich.edu load_inst->memData = new uint8_t[64]; 6122292SN/A 6134326Sgblack@eecs.umich.edu memcpy(load_inst->memData, 6144326Sgblack@eecs.umich.edu storeQueue[store_idx].data + shift_amt, req->getSize()); 6152292SN/A 6162292SN/A DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 6172292SN/A "addr %#x, data %#x\n", 6182693Sktlim@umich.edu store_idx, req->getVaddr(), data); 6192678Sktlim@umich.edu 6204022Sstever@eecs.umich.edu PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 6214022Sstever@eecs.umich.edu Packet::Broadcast); 6222678Sktlim@umich.edu data_pkt->dataStatic(load_inst->memData); 6232678Sktlim@umich.edu 6242678Sktlim@umich.edu WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 6252292SN/A 6262292SN/A // We'll say this has a 1 cycle load-store forwarding latency 6272292SN/A // for now. 6282292SN/A // @todo: Need to make this a parameter. 6295606Snate@binkert.org cpu->schedule(wb, curTick); 6302678Sktlim@umich.edu 6316974Stjones1@inf.ed.ac.uk // Don't need to do anything special for split loads. 6326974Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow) { 6336974Stjones1@inf.ed.ac.uk delete sreqLow; 6346974Stjones1@inf.ed.ac.uk delete sreqHigh; 6356974Stjones1@inf.ed.ac.uk } 6366974Stjones1@inf.ed.ac.uk 6372727Sktlim@umich.edu ++lsqForwLoads; 6382292SN/A return NoFault; 6392292SN/A } else if ((store_has_lower_limit && lower_load_has_store_part) || 6402292SN/A (store_has_upper_limit && upper_load_has_store_part) || 6412292SN/A (lower_load_has_store_part && upper_load_has_store_part)) { 6422292SN/A // This is the partial store-load forwarding case where a store 6432292SN/A // has only part of the load's data. 6442292SN/A 6452292SN/A // If it's already been written back, then don't worry about 6462292SN/A // stalling on it. 6472292SN/A if (storeQueue[store_idx].completed) { 6484032Sktlim@umich.edu panic("Should not check one of these"); 6492292SN/A continue; 6502292SN/A } 6512292SN/A 6522292SN/A // Must stall load and force it to retry, so long as it's the oldest 6532292SN/A // load that needs to do so. 6542292SN/A if (!stalled || 6552292SN/A (stalled && 6562669Sktlim@umich.edu load_inst->seqNum < 6572292SN/A loadQueue[stallingLoadIdx]->seqNum)) { 6582292SN/A stalled = true; 6592292SN/A stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 6602292SN/A stallingLoadIdx = load_idx; 6612292SN/A } 6622292SN/A 6632292SN/A // Tell IQ/mem dep unit that this instruction will need to be 6642292SN/A // rescheduled eventually 6652669Sktlim@umich.edu iewStage->rescheduleMemInst(load_inst); 6662927Sktlim@umich.edu iewStage->decrWb(load_inst->seqNum); 6674032Sktlim@umich.edu load_inst->clearIssued(); 6682727Sktlim@umich.edu ++lsqRescheduledLoads; 6692292SN/A 6702292SN/A // Do not generate a writeback event as this instruction is not 6712292SN/A // complete. 6722292SN/A DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 6732292SN/A "Store idx %i to load addr %#x\n", 6742669Sktlim@umich.edu store_idx, req->getVaddr()); 6752292SN/A 6764032Sktlim@umich.edu // Must delete request now that it wasn't handed off to 6774032Sktlim@umich.edu // memory. This is quite ugly. @todo: Figure out the 6784032Sktlim@umich.edu // proper place to really handle request deletes. 6794032Sktlim@umich.edu delete req; 6806974Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow) { 6816974Stjones1@inf.ed.ac.uk delete sreqLow; 6826974Stjones1@inf.ed.ac.uk delete sreqHigh; 6836974Stjones1@inf.ed.ac.uk } 6844032Sktlim@umich.edu 6852292SN/A return NoFault; 6862292SN/A } 6872292SN/A } 6882292SN/A 6892292SN/A // If there's no forwarding case, then go access memory 6902907Sktlim@umich.edu DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %#x\n", 6912669Sktlim@umich.edu load_inst->seqNum, load_inst->readPC()); 6922292SN/A 6932669Sktlim@umich.edu assert(!load_inst->memData); 6942669Sktlim@umich.edu load_inst->memData = new uint8_t[64]; 6952292SN/A 6962292SN/A ++usedPorts; 6972292SN/A 6982907Sktlim@umich.edu // if we the cache is not blocked, do cache access 6996974Stjones1@inf.ed.ac.uk bool completedFirst = false; 7002907Sktlim@umich.edu if (!lsq->cacheBlocked()) { 7016974Stjones1@inf.ed.ac.uk MemCmd command = 7026974Stjones1@inf.ed.ac.uk req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 7036974Stjones1@inf.ed.ac.uk PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 7046974Stjones1@inf.ed.ac.uk PacketPtr fst_data_pkt = NULL; 7056974Stjones1@inf.ed.ac.uk PacketPtr snd_data_pkt = NULL; 7066974Stjones1@inf.ed.ac.uk 7073228Sktlim@umich.edu data_pkt->dataStatic(load_inst->memData); 7083228Sktlim@umich.edu 7093228Sktlim@umich.edu LSQSenderState *state = new LSQSenderState; 7103228Sktlim@umich.edu state->isLoad = true; 7113228Sktlim@umich.edu state->idx = load_idx; 7123228Sktlim@umich.edu state->inst = load_inst; 7133228Sktlim@umich.edu data_pkt->senderState = state; 7143228Sktlim@umich.edu 7156974Stjones1@inf.ed.ac.uk if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 7166974Stjones1@inf.ed.ac.uk 7176974Stjones1@inf.ed.ac.uk // Point the first packet at the main data packet. 7186974Stjones1@inf.ed.ac.uk fst_data_pkt = data_pkt; 7196974Stjones1@inf.ed.ac.uk } else { 7206974Stjones1@inf.ed.ac.uk 7216974Stjones1@inf.ed.ac.uk // Create the split packets. 7226974Stjones1@inf.ed.ac.uk fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 7236974Stjones1@inf.ed.ac.uk snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 7246974Stjones1@inf.ed.ac.uk 7256974Stjones1@inf.ed.ac.uk fst_data_pkt->dataStatic(load_inst->memData); 7266974Stjones1@inf.ed.ac.uk snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 7276974Stjones1@inf.ed.ac.uk 7286974Stjones1@inf.ed.ac.uk fst_data_pkt->senderState = state; 7296974Stjones1@inf.ed.ac.uk snd_data_pkt->senderState = state; 7306974Stjones1@inf.ed.ac.uk 7316974Stjones1@inf.ed.ac.uk state->isSplit = true; 7326974Stjones1@inf.ed.ac.uk state->outstanding = 2; 7336974Stjones1@inf.ed.ac.uk state->mainPkt = data_pkt; 7346974Stjones1@inf.ed.ac.uk } 7356974Stjones1@inf.ed.ac.uk 7366974Stjones1@inf.ed.ac.uk if (!dcachePort->sendTiming(fst_data_pkt)) { 7373228Sktlim@umich.edu // Delete state and data packet because a load retry 7383228Sktlim@umich.edu // initiates a pipeline restart; it does not retry. 7393228Sktlim@umich.edu delete state; 7404032Sktlim@umich.edu delete data_pkt->req; 7413228Sktlim@umich.edu delete data_pkt; 7426974Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow) { 7436974Stjones1@inf.ed.ac.uk delete fst_data_pkt->req; 7446974Stjones1@inf.ed.ac.uk delete fst_data_pkt; 7456974Stjones1@inf.ed.ac.uk delete snd_data_pkt->req; 7466974Stjones1@inf.ed.ac.uk delete snd_data_pkt; 7477511Stjones1@inf.ed.ac.uk sreqLow = NULL; 7487511Stjones1@inf.ed.ac.uk sreqHigh = NULL; 7496974Stjones1@inf.ed.ac.uk } 7503228Sktlim@umich.edu 7514032Sktlim@umich.edu req = NULL; 7524032Sktlim@umich.edu 7532907Sktlim@umich.edu // If the access didn't succeed, tell the LSQ by setting 7542907Sktlim@umich.edu // the retry thread id. 7552907Sktlim@umich.edu lsq->setRetryTid(lsqID); 7566974Stjones1@inf.ed.ac.uk } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 7576974Stjones1@inf.ed.ac.uk completedFirst = true; 7586974Stjones1@inf.ed.ac.uk 7596974Stjones1@inf.ed.ac.uk // The first packet was sent without problems, so send this one 7606974Stjones1@inf.ed.ac.uk // too. If there is a problem with this packet then the whole 7616974Stjones1@inf.ed.ac.uk // load will be squashed, so indicate this to the state object. 7626974Stjones1@inf.ed.ac.uk // The first packet will return in completeDataAccess and be 7636974Stjones1@inf.ed.ac.uk // handled there. 7646974Stjones1@inf.ed.ac.uk ++usedPorts; 7656974Stjones1@inf.ed.ac.uk if (!dcachePort->sendTiming(snd_data_pkt)) { 7666974Stjones1@inf.ed.ac.uk 7676974Stjones1@inf.ed.ac.uk // The main packet will be deleted in completeDataAccess. 7686974Stjones1@inf.ed.ac.uk delete snd_data_pkt->req; 7696974Stjones1@inf.ed.ac.uk delete snd_data_pkt; 7706974Stjones1@inf.ed.ac.uk 7716974Stjones1@inf.ed.ac.uk state->complete(); 7726974Stjones1@inf.ed.ac.uk 7736974Stjones1@inf.ed.ac.uk req = NULL; 7747511Stjones1@inf.ed.ac.uk sreqHigh = NULL; 7756974Stjones1@inf.ed.ac.uk 7766974Stjones1@inf.ed.ac.uk lsq->setRetryTid(lsqID); 7776974Stjones1@inf.ed.ac.uk } 7782907Sktlim@umich.edu } 7792907Sktlim@umich.edu } 7802907Sktlim@umich.edu 7812907Sktlim@umich.edu // If the cache was blocked, or has become blocked due to the access, 7822907Sktlim@umich.edu // handle it. 7832907Sktlim@umich.edu if (lsq->cacheBlocked()) { 7844032Sktlim@umich.edu if (req) 7854032Sktlim@umich.edu delete req; 7866974Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 7876974Stjones1@inf.ed.ac.uk delete sreqLow; 7886974Stjones1@inf.ed.ac.uk delete sreqHigh; 7896974Stjones1@inf.ed.ac.uk } 7904032Sktlim@umich.edu 7912727Sktlim@umich.edu ++lsqCacheBlocked; 7923014Srdreslin@umich.edu 7933014Srdreslin@umich.edu iewStage->decrWb(load_inst->seqNum); 7942669Sktlim@umich.edu // There's an older load that's already going to squash. 7952669Sktlim@umich.edu if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 7962669Sktlim@umich.edu return NoFault; 7972292SN/A 7982669Sktlim@umich.edu // Record that the load was blocked due to memory. This 7992669Sktlim@umich.edu // load will squash all instructions after it, be 8002669Sktlim@umich.edu // refetched, and re-executed. 8012669Sktlim@umich.edu isLoadBlocked = true; 8022669Sktlim@umich.edu loadBlockedHandled = false; 8032669Sktlim@umich.edu blockedLoadSeqNum = load_inst->seqNum; 8042669Sktlim@umich.edu // No fault occurred, even though the interface is blocked. 8052669Sktlim@umich.edu return NoFault; 8062292SN/A } 8072292SN/A 8082669Sktlim@umich.edu return NoFault; 8092292SN/A} 8102292SN/A 8112292SN/Atemplate <class Impl> 8122292SN/Atemplate <class T> 8132292SN/AFault 8146974Stjones1@inf.ed.ac.ukLSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 8156974Stjones1@inf.ed.ac.uk T &data, int store_idx) 8162292SN/A{ 8172292SN/A assert(storeQueue[store_idx].inst); 8182292SN/A 8192292SN/A DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 8202292SN/A " | storeHead:%i [sn:%i]\n", 8212669Sktlim@umich.edu store_idx, req->getPaddr(), data, storeHead, 8222292SN/A storeQueue[store_idx].inst->seqNum); 8232329SN/A 8242292SN/A storeQueue[store_idx].req = req; 8256974Stjones1@inf.ed.ac.uk storeQueue[store_idx].sreqLow = sreqLow; 8266974Stjones1@inf.ed.ac.uk storeQueue[store_idx].sreqHigh = sreqHigh; 8272292SN/A storeQueue[store_idx].size = sizeof(T); 8287509Stjones1@inf.ed.ac.uk 8297509Stjones1@inf.ed.ac.uk // Split stores can only occur in ISAs with unaligned memory accesses. If 8307509Stjones1@inf.ed.ac.uk // a store request has been split, sreqLow and sreqHigh will be non-null. 8317509Stjones1@inf.ed.ac.uk if (TheISA::HasUnalignedMemAcc && sreqLow) { 8327509Stjones1@inf.ed.ac.uk storeQueue[store_idx].isSplit = true; 8337509Stjones1@inf.ed.ac.uk } 8344326Sgblack@eecs.umich.edu assert(sizeof(T) <= sizeof(storeQueue[store_idx].data)); 8354326Sgblack@eecs.umich.edu 8364326Sgblack@eecs.umich.edu T gData = htog(data); 8374326Sgblack@eecs.umich.edu memcpy(storeQueue[store_idx].data, &gData, sizeof(T)); 8382329SN/A 8392292SN/A // This function only writes the data to the store queue, so no fault 8402292SN/A // can happen here. 8412292SN/A return NoFault; 8422292SN/A} 8432292SN/A 8442292SN/A#endif // __CPU_O3_LSQ_UNIT_HH__ 845