lsq_unit.hh revision 10824
17118Sgblack@eecs.umich.edu/* 27118Sgblack@eecs.umich.edu * Copyright (c) 2012-2014 ARM Limited 37118Sgblack@eecs.umich.edu * All rights reserved 47118Sgblack@eecs.umich.edu * 57118Sgblack@eecs.umich.edu * The license below extends only to copyright in the software and shall 67118Sgblack@eecs.umich.edu * not be construed as granting a license to any other intellectual 77118Sgblack@eecs.umich.edu * property including but not limited to intellectual property relating 87118Sgblack@eecs.umich.edu * to a hardware implementation of the functionality of the software 97118Sgblack@eecs.umich.edu * licensed hereunder. You may use the software subject to the license 107118Sgblack@eecs.umich.edu * terms below provided that you ensure that this notice is replicated 117118Sgblack@eecs.umich.edu * unmodified and in its entirety in all distributions of the software, 127118Sgblack@eecs.umich.edu * modified or unmodified, in source code or in binary form. 137118Sgblack@eecs.umich.edu * 147118Sgblack@eecs.umich.edu * Copyright (c) 2004-2006 The Regents of The University of Michigan 156253Sgblack@eecs.umich.edu * Copyright (c) 2013 Advanced Micro Devices, Inc. 166253Sgblack@eecs.umich.edu * All rights reserved. 176253Sgblack@eecs.umich.edu * 186253Sgblack@eecs.umich.edu * Redistribution and use in source and binary forms, with or without 196253Sgblack@eecs.umich.edu * modification, are permitted provided that the following conditions are 206253Sgblack@eecs.umich.edu * met: redistributions of source code must retain the above copyright 216253Sgblack@eecs.umich.edu * notice, this list of conditions and the following disclaimer; 226253Sgblack@eecs.umich.edu * redistributions in binary form must reproduce the above copyright 236253Sgblack@eecs.umich.edu * notice, this list of conditions and the following disclaimer in the 246253Sgblack@eecs.umich.edu * documentation and/or other materials provided with the distribution; 256253Sgblack@eecs.umich.edu * neither the name of the copyright holders nor the names of its 266253Sgblack@eecs.umich.edu * contributors may be used to endorse or promote products derived from 276253Sgblack@eecs.umich.edu * this software without specific prior written permission. 286253Sgblack@eecs.umich.edu * 296253Sgblack@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 306253Sgblack@eecs.umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 316253Sgblack@eecs.umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 326253Sgblack@eecs.umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 336253Sgblack@eecs.umich.edu * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 346253Sgblack@eecs.umich.edu * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 356253Sgblack@eecs.umich.edu * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 366253Sgblack@eecs.umich.edu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 376253Sgblack@eecs.umich.edu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 386253Sgblack@eecs.umich.edu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 396253Sgblack@eecs.umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 406253Sgblack@eecs.umich.edu * 416253Sgblack@eecs.umich.edu * Authors: Kevin Lim 426253Sgblack@eecs.umich.edu * Korey Sewell 436253Sgblack@eecs.umich.edu */ 446253Sgblack@eecs.umich.edu 456253Sgblack@eecs.umich.edu#ifndef __CPU_O3_LSQ_UNIT_HH__ 466253Sgblack@eecs.umich.edu#define __CPU_O3_LSQ_UNIT_HH__ 476253Sgblack@eecs.umich.edu 486253Sgblack@eecs.umich.edu#include <algorithm> 497118Sgblack@eecs.umich.edu#include <cstring> 507205Sgblack@eecs.umich.edu#include <map> 517205Sgblack@eecs.umich.edu#include <queue> 527205Sgblack@eecs.umich.edu 537205Sgblack@eecs.umich.edu#include "arch/generic/debugfaults.hh" 547205Sgblack@eecs.umich.edu#include "arch/isa_traits.hh" 557205Sgblack@eecs.umich.edu#include "arch/locked_mem.hh" 567205Sgblack@eecs.umich.edu#include "arch/mmapped_ipr.hh" 577205Sgblack@eecs.umich.edu#include "base/hashmap.hh" 587205Sgblack@eecs.umich.edu#include "config/the_isa.hh" 597205Sgblack@eecs.umich.edu#include "cpu/inst_seq.hh" 607205Sgblack@eecs.umich.edu#include "cpu/timebuf.hh" 617205Sgblack@eecs.umich.edu#include "debug/LSQUnit.hh" 627205Sgblack@eecs.umich.edu#include "mem/packet.hh" 637205Sgblack@eecs.umich.edu#include "mem/port.hh" 647205Sgblack@eecs.umich.edu 657205Sgblack@eecs.umich.edustruct DerivO3CPUParams; 667291Sgblack@eecs.umich.edu 677291Sgblack@eecs.umich.edu/** 687291Sgblack@eecs.umich.edu * Class that implements the actual LQ and SQ for each specific 697291Sgblack@eecs.umich.edu * thread. Both are circular queues; load entries are freed upon 707291Sgblack@eecs.umich.edu * committing, while store entries are freed once they writeback. The 717291Sgblack@eecs.umich.edu * LSQUnit tracks if there are memory ordering violations, and also 727291Sgblack@eecs.umich.edu * detects partial load to store forwarding cases (a store only has 737291Sgblack@eecs.umich.edu * part of a load's data) that requires the load to wait until the 747291Sgblack@eecs.umich.edu * store writes back. In the former case it holds onto the instruction 757291Sgblack@eecs.umich.edu * until the dependence unit looks at it, and in the latter it stalls 767291Sgblack@eecs.umich.edu * the LSQ until the store writes back. At that point the load is 777291Sgblack@eecs.umich.edu * replayed. 787291Sgblack@eecs.umich.edu */ 797291Sgblack@eecs.umich.edutemplate <class Impl> 807291Sgblack@eecs.umich.educlass LSQUnit { 817291Sgblack@eecs.umich.edu public: 827291Sgblack@eecs.umich.edu typedef typename Impl::O3CPU O3CPU; 837291Sgblack@eecs.umich.edu typedef typename Impl::DynInstPtr DynInstPtr; 847291Sgblack@eecs.umich.edu typedef typename Impl::CPUPol::IEW IEW; 857291Sgblack@eecs.umich.edu typedef typename Impl::CPUPol::LSQ LSQ; 867291Sgblack@eecs.umich.edu typedef typename Impl::CPUPol::IssueStruct IssueStruct; 877291Sgblack@eecs.umich.edu 887291Sgblack@eecs.umich.edu public: 897291Sgblack@eecs.umich.edu /** Constructs an LSQ unit. init() must be called prior to use. */ 907132Sgblack@eecs.umich.edu LSQUnit(); 917118Sgblack@eecs.umich.edu 927118Sgblack@eecs.umich.edu /** Initializes the LSQ unit with the specified number of entries. */ 937118Sgblack@eecs.umich.edu void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 947118Sgblack@eecs.umich.edu LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 957118Sgblack@eecs.umich.edu unsigned id); 967118Sgblack@eecs.umich.edu 977118Sgblack@eecs.umich.edu /** Returns the name of the LSQ unit. */ 987118Sgblack@eecs.umich.edu std::string name() const; 997118Sgblack@eecs.umich.edu 1007118Sgblack@eecs.umich.edu /** Registers statistics. */ 1017118Sgblack@eecs.umich.edu void regStats(); 1027118Sgblack@eecs.umich.edu 1037118Sgblack@eecs.umich.edu /** Sets the pointer to the dcache port. */ 1047118Sgblack@eecs.umich.edu void setDcachePort(MasterPort *dcache_port); 1057132Sgblack@eecs.umich.edu 1067132Sgblack@eecs.umich.edu /** Perform sanity checks after a drain. */ 1077118Sgblack@eecs.umich.edu void drainSanityCheck() const; 1087118Sgblack@eecs.umich.edu 1097118Sgblack@eecs.umich.edu /** Takes over from another CPU's thread. */ 1107118Sgblack@eecs.umich.edu void takeOverFrom(); 1117118Sgblack@eecs.umich.edu 1127118Sgblack@eecs.umich.edu /** Ticks the LSQ unit, which in this case only resets the number of 1137118Sgblack@eecs.umich.edu * used cache ports. 1147118Sgblack@eecs.umich.edu * @todo: Move the number of used ports up to the LSQ level so it can 1157279Sgblack@eecs.umich.edu * be shared by all LSQ units. 1167279Sgblack@eecs.umich.edu */ 1177279Sgblack@eecs.umich.edu void tick() { usedPorts = 0; } 1187279Sgblack@eecs.umich.edu 1197279Sgblack@eecs.umich.edu /** Inserts an instruction. */ 1207279Sgblack@eecs.umich.edu void insert(DynInstPtr &inst); 1217118Sgblack@eecs.umich.edu /** Inserts a load instruction. */ 1227118Sgblack@eecs.umich.edu void insertLoad(DynInstPtr &load_inst); 1237118Sgblack@eecs.umich.edu /** Inserts a store instruction. */ 1247118Sgblack@eecs.umich.edu void insertStore(DynInstPtr &store_inst); 1257132Sgblack@eecs.umich.edu 1267118Sgblack@eecs.umich.edu /** Check for ordering violations in the LSQ. For a store squash if we 1277118Sgblack@eecs.umich.edu * ever find a conflicting load. For a load, only squash if we 1287118Sgblack@eecs.umich.edu * an external snoop invalidate has been seen for that load address 1297118Sgblack@eecs.umich.edu * @param load_idx index to start checking at 1307132Sgblack@eecs.umich.edu * @param inst the instruction to check 1317132Sgblack@eecs.umich.edu */ 1327132Sgblack@eecs.umich.edu Fault checkViolations(int load_idx, DynInstPtr &inst); 1337118Sgblack@eecs.umich.edu 1347118Sgblack@eecs.umich.edu /** Check if an incoming invalidate hits in the lsq on a load 1357118Sgblack@eecs.umich.edu * that might have issued out of order wrt another load beacuse 1367118Sgblack@eecs.umich.edu * of the intermediate invalidate. 1377118Sgblack@eecs.umich.edu */ 1387118Sgblack@eecs.umich.edu void checkSnoop(PacketPtr pkt); 1397118Sgblack@eecs.umich.edu 1407118Sgblack@eecs.umich.edu /** Executes a load instruction. */ 1417118Sgblack@eecs.umich.edu Fault executeLoad(DynInstPtr &inst); 1427118Sgblack@eecs.umich.edu 1437118Sgblack@eecs.umich.edu Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 1447118Sgblack@eecs.umich.edu /** Executes a store instruction. */ 1457279Sgblack@eecs.umich.edu Fault executeStore(DynInstPtr &inst); 1467279Sgblack@eecs.umich.edu 1477279Sgblack@eecs.umich.edu /** Commits the head load. */ 1487279Sgblack@eecs.umich.edu void commitLoad(); 1497279Sgblack@eecs.umich.edu /** Commits loads older than a specific sequence number. */ 1507279Sgblack@eecs.umich.edu void commitLoads(InstSeqNum &youngest_inst); 1517279Sgblack@eecs.umich.edu 1527279Sgblack@eecs.umich.edu /** Commits stores older than a specific sequence number. */ 1537279Sgblack@eecs.umich.edu void commitStores(InstSeqNum &youngest_inst); 1547279Sgblack@eecs.umich.edu 1557279Sgblack@eecs.umich.edu /** Writes back stores. */ 1567279Sgblack@eecs.umich.edu void writebackStores(); 1577279Sgblack@eecs.umich.edu 1587279Sgblack@eecs.umich.edu /** Completes the data access that has been returned from the 1597279Sgblack@eecs.umich.edu * memory system. */ 1607279Sgblack@eecs.umich.edu void completeDataAccess(PacketPtr pkt); 1617279Sgblack@eecs.umich.edu 1627279Sgblack@eecs.umich.edu /** Clears all the entries in the LQ. */ 1637279Sgblack@eecs.umich.edu void clearLQ(); 1647279Sgblack@eecs.umich.edu 1657279Sgblack@eecs.umich.edu /** Clears all the entries in the SQ. */ 1667279Sgblack@eecs.umich.edu void clearSQ(); 1677118Sgblack@eecs.umich.edu 1687132Sgblack@eecs.umich.edu /** Resizes the LQ to a given size. */ 1697118Sgblack@eecs.umich.edu void resizeLQ(unsigned size); 1707118Sgblack@eecs.umich.edu 1717118Sgblack@eecs.umich.edu /** Resizes the SQ to a given size. */ 1727118Sgblack@eecs.umich.edu void resizeSQ(unsigned size); 1737118Sgblack@eecs.umich.edu 1747118Sgblack@eecs.umich.edu /** Squashes all instructions younger than a specific sequence number. */ 1757132Sgblack@eecs.umich.edu void squash(const InstSeqNum &squashed_num); 1767132Sgblack@eecs.umich.edu 1777132Sgblack@eecs.umich.edu /** Returns if there is a memory ordering violation. Value is reset upon 1787132Sgblack@eecs.umich.edu * call to getMemDepViolator(). 1797132Sgblack@eecs.umich.edu */ 1807118Sgblack@eecs.umich.edu bool violation() { return memDepViolator; } 1817118Sgblack@eecs.umich.edu 1827118Sgblack@eecs.umich.edu /** Returns the memory ordering violator. */ 1837118Sgblack@eecs.umich.edu DynInstPtr getMemDepViolator(); 1847118Sgblack@eecs.umich.edu 1857118Sgblack@eecs.umich.edu /** Returns the number of free LQ entries. */ 1867118Sgblack@eecs.umich.edu unsigned numFreeLoadEntries(); 1877118Sgblack@eecs.umich.edu 1887118Sgblack@eecs.umich.edu /** Returns the number of free SQ entries. */ 1897118Sgblack@eecs.umich.edu unsigned numFreeStoreEntries(); 1907118Sgblack@eecs.umich.edu 1917118Sgblack@eecs.umich.edu /** Returns the number of loads in the LQ. */ 1927118Sgblack@eecs.umich.edu int numLoads() { return loads; } 1937118Sgblack@eecs.umich.edu 1947118Sgblack@eecs.umich.edu /** Returns the number of stores in the SQ. */ 1957118Sgblack@eecs.umich.edu int numStores() { return stores; } 1967118Sgblack@eecs.umich.edu 1977118Sgblack@eecs.umich.edu /** Returns if either the LQ or SQ is full. */ 1987118Sgblack@eecs.umich.edu bool isFull() { return lqFull() || sqFull(); } 1997118Sgblack@eecs.umich.edu 2007118Sgblack@eecs.umich.edu /** Returns if both the LQ and SQ are empty. */ 2017118Sgblack@eecs.umich.edu bool isEmpty() const { return lqEmpty() && sqEmpty(); } 2027118Sgblack@eecs.umich.edu 2037118Sgblack@eecs.umich.edu /** Returns if the LQ is full. */ 2047118Sgblack@eecs.umich.edu bool lqFull() { return loads >= (LQEntries - 1); } 2057118Sgblack@eecs.umich.edu 2067118Sgblack@eecs.umich.edu /** Returns if the SQ is full. */ 2077118Sgblack@eecs.umich.edu bool sqFull() { return stores >= (SQEntries - 1); } 2087118Sgblack@eecs.umich.edu 2097118Sgblack@eecs.umich.edu /** Returns if the LQ is empty. */ 2107118Sgblack@eecs.umich.edu bool lqEmpty() const { return loads == 0; } 2117118Sgblack@eecs.umich.edu 2127118Sgblack@eecs.umich.edu /** Returns if the SQ is empty. */ 2137118Sgblack@eecs.umich.edu bool sqEmpty() const { return stores == 0; } 2147118Sgblack@eecs.umich.edu 2157118Sgblack@eecs.umich.edu /** Returns the number of instructions in the LSQ. */ 2167118Sgblack@eecs.umich.edu unsigned getCount() { return loads + stores; } 2177118Sgblack@eecs.umich.edu 2187118Sgblack@eecs.umich.edu /** Returns if there are any stores to writeback. */ 2197118Sgblack@eecs.umich.edu bool hasStoresToWB() { return storesToWB; } 2207279Sgblack@eecs.umich.edu 2217279Sgblack@eecs.umich.edu /** Returns the number of stores to writeback. */ 2227279Sgblack@eecs.umich.edu int numStoresToWB() { return storesToWB; } 2237279Sgblack@eecs.umich.edu 2247279Sgblack@eecs.umich.edu /** Returns if the LSQ unit will writeback on this cycle. */ 2257279Sgblack@eecs.umich.edu bool willWB() { return storeQueue[storeWBIdx].canWB && 2267279Sgblack@eecs.umich.edu !storeQueue[storeWBIdx].completed && 2277279Sgblack@eecs.umich.edu !isStoreBlocked; } 2287279Sgblack@eecs.umich.edu 2297279Sgblack@eecs.umich.edu /** Handles doing the retry. */ 2307279Sgblack@eecs.umich.edu void recvRetry(); 2317279Sgblack@eecs.umich.edu 2327279Sgblack@eecs.umich.edu private: 2337279Sgblack@eecs.umich.edu /** Reset the LSQ state */ 2347279Sgblack@eecs.umich.edu void resetState(); 2357279Sgblack@eecs.umich.edu 2367279Sgblack@eecs.umich.edu /** Writes back the instruction, sending it to IEW. */ 2377279Sgblack@eecs.umich.edu void writeback(DynInstPtr &inst, PacketPtr pkt); 2387279Sgblack@eecs.umich.edu 2397279Sgblack@eecs.umich.edu /** Writes back a store that couldn't be completed the previous cycle. */ 2407279Sgblack@eecs.umich.edu void writebackPendingStore(); 2417279Sgblack@eecs.umich.edu 2427279Sgblack@eecs.umich.edu /** Handles completing the send of a store to memory. */ 2437279Sgblack@eecs.umich.edu void storePostSend(PacketPtr pkt); 2447118Sgblack@eecs.umich.edu 2457132Sgblack@eecs.umich.edu /** Completes the store at the specified index. */ 2467118Sgblack@eecs.umich.edu void completeStore(int store_idx); 2477118Sgblack@eecs.umich.edu 2487132Sgblack@eecs.umich.edu /** Attempts to send a store to the cache. */ 2497132Sgblack@eecs.umich.edu bool sendStore(PacketPtr data_pkt); 2507132Sgblack@eecs.umich.edu 2517132Sgblack@eecs.umich.edu /** Increments the given store index (circular queue). */ 2527132Sgblack@eecs.umich.edu inline void incrStIdx(int &store_idx) const; 2537132Sgblack@eecs.umich.edu /** Decrements the given store index (circular queue). */ 2547132Sgblack@eecs.umich.edu inline void decrStIdx(int &store_idx) const; 2557132Sgblack@eecs.umich.edu /** Increments the given load index (circular queue). */ 2567132Sgblack@eecs.umich.edu inline void incrLdIdx(int &load_idx) const; 2577132Sgblack@eecs.umich.edu /** Decrements the given load index (circular queue). */ 2587132Sgblack@eecs.umich.edu inline void decrLdIdx(int &load_idx) const; 2597132Sgblack@eecs.umich.edu 2607132Sgblack@eecs.umich.edu public: 2617132Sgblack@eecs.umich.edu /** Debugging function to dump instructions in the LSQ. */ 2627279Sgblack@eecs.umich.edu void dumpInsts() const; 2637279Sgblack@eecs.umich.edu 2647279Sgblack@eecs.umich.edu private: 2657279Sgblack@eecs.umich.edu /** Pointer to the CPU. */ 2667279Sgblack@eecs.umich.edu O3CPU *cpu; 2677279Sgblack@eecs.umich.edu 2687279Sgblack@eecs.umich.edu /** Pointer to the IEW stage. */ 2697279Sgblack@eecs.umich.edu IEW *iewStage; 2707279Sgblack@eecs.umich.edu 2717279Sgblack@eecs.umich.edu /** Pointer to the LSQ. */ 2727279Sgblack@eecs.umich.edu LSQ *lsq; 2737279Sgblack@eecs.umich.edu 2747279Sgblack@eecs.umich.edu /** Pointer to the dcache port. Used only for sending. */ 2757279Sgblack@eecs.umich.edu MasterPort *dcachePort; 2767279Sgblack@eecs.umich.edu 2777132Sgblack@eecs.umich.edu /** Derived class to hold any sender state the LSQ needs. */ 2787132Sgblack@eecs.umich.edu class LSQSenderState : public Packet::SenderState 2797132Sgblack@eecs.umich.edu { 2807132Sgblack@eecs.umich.edu public: 2817132Sgblack@eecs.umich.edu /** Default constructor. */ 2827132Sgblack@eecs.umich.edu LSQSenderState() 2837132Sgblack@eecs.umich.edu : mainPkt(NULL), pendingPacket(NULL), idx(0), outstanding(1), 2847132Sgblack@eecs.umich.edu isLoad(false), noWB(false), isSplit(false), 2857132Sgblack@eecs.umich.edu pktToSend(false), cacheBlocked(false) 2867132Sgblack@eecs.umich.edu { } 2877132Sgblack@eecs.umich.edu 2887132Sgblack@eecs.umich.edu /** Instruction who initiated the access to memory. */ 2897132Sgblack@eecs.umich.edu DynInstPtr inst; 2907132Sgblack@eecs.umich.edu /** The main packet from a split load, used during writeback. */ 2917132Sgblack@eecs.umich.edu PacketPtr mainPkt; 2927132Sgblack@eecs.umich.edu /** A second packet from a split store that needs sending. */ 2937132Sgblack@eecs.umich.edu PacketPtr pendingPacket; 2947132Sgblack@eecs.umich.edu /** The LQ/SQ index of the instruction. */ 2957132Sgblack@eecs.umich.edu uint8_t idx; 2967132Sgblack@eecs.umich.edu /** Number of outstanding packets to complete. */ 2977132Sgblack@eecs.umich.edu uint8_t outstanding; 2987132Sgblack@eecs.umich.edu /** Whether or not it is a load. */ 2997132Sgblack@eecs.umich.edu bool isLoad; 3007132Sgblack@eecs.umich.edu /** Whether or not the instruction will need to writeback. */ 3017132Sgblack@eecs.umich.edu bool noWB; 3027132Sgblack@eecs.umich.edu /** Whether or not this access is split in two. */ 3037132Sgblack@eecs.umich.edu bool isSplit; 3047279Sgblack@eecs.umich.edu /** Whether or not there is a packet that needs sending. */ 3057279Sgblack@eecs.umich.edu bool pktToSend; 3067279Sgblack@eecs.umich.edu /** Whether or not the second packet of this split load was blocked */ 3077279Sgblack@eecs.umich.edu bool cacheBlocked; 3087279Sgblack@eecs.umich.edu 3097279Sgblack@eecs.umich.edu /** Completes a packet and returns whether the access is finished. */ 3107279Sgblack@eecs.umich.edu inline bool complete() { return --outstanding == 0; } 3117279Sgblack@eecs.umich.edu }; 3127279Sgblack@eecs.umich.edu 3137279Sgblack@eecs.umich.edu /** Writeback event, specifically for when stores forward data to loads. */ 3147279Sgblack@eecs.umich.edu class WritebackEvent : public Event { 3157279Sgblack@eecs.umich.edu public: 3167279Sgblack@eecs.umich.edu /** Constructs a writeback event. */ 3177279Sgblack@eecs.umich.edu WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 3187279Sgblack@eecs.umich.edu 3197132Sgblack@eecs.umich.edu /** Processes the writeback event. */ 3207132Sgblack@eecs.umich.edu void process(); 3217132Sgblack@eecs.umich.edu 3227132Sgblack@eecs.umich.edu /** Returns the description of this event. */ 3237132Sgblack@eecs.umich.edu const char *description() const; 3247132Sgblack@eecs.umich.edu 3257132Sgblack@eecs.umich.edu private: 3267132Sgblack@eecs.umich.edu /** Instruction whose results are being written back. */ 3277132Sgblack@eecs.umich.edu DynInstPtr inst; 3287132Sgblack@eecs.umich.edu 3297132Sgblack@eecs.umich.edu /** The packet that would have been sent to memory. */ 3307132Sgblack@eecs.umich.edu PacketPtr pkt; 3317132Sgblack@eecs.umich.edu 3327132Sgblack@eecs.umich.edu /** The pointer to the LSQ unit that issued the store. */ 3337118Sgblack@eecs.umich.edu LSQUnit<Impl> *lsqPtr; 3347118Sgblack@eecs.umich.edu }; 3357118Sgblack@eecs.umich.edu 3367118Sgblack@eecs.umich.edu public: 3377118Sgblack@eecs.umich.edu struct SQEntry { 3387132Sgblack@eecs.umich.edu /** Constructs an empty store queue entry. */ 3397118Sgblack@eecs.umich.edu SQEntry() 3407118Sgblack@eecs.umich.edu : inst(NULL), req(NULL), size(0), 3417118Sgblack@eecs.umich.edu canWB(0), committed(0), completed(0) 3427118Sgblack@eecs.umich.edu { 3437118Sgblack@eecs.umich.edu std::memset(data, 0, sizeof(data)); 3447118Sgblack@eecs.umich.edu } 3457118Sgblack@eecs.umich.edu 3467279Sgblack@eecs.umich.edu ~SQEntry() 3477279Sgblack@eecs.umich.edu { 3487279Sgblack@eecs.umich.edu inst = NULL; 3497279Sgblack@eecs.umich.edu } 3507279Sgblack@eecs.umich.edu 3517279Sgblack@eecs.umich.edu /** Constructs a store queue entry for a given instruction. */ 3527279Sgblack@eecs.umich.edu SQEntry(DynInstPtr &_inst) 3537279Sgblack@eecs.umich.edu : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 3547279Sgblack@eecs.umich.edu isSplit(0), canWB(0), committed(0), completed(0), isAllZeros(0) 3557279Sgblack@eecs.umich.edu { 3567279Sgblack@eecs.umich.edu std::memset(data, 0, sizeof(data)); 3577279Sgblack@eecs.umich.edu } 3587279Sgblack@eecs.umich.edu /** The store data. */ 3597279Sgblack@eecs.umich.edu char data[16]; 3607279Sgblack@eecs.umich.edu /** The store instruction. */ 3617118Sgblack@eecs.umich.edu DynInstPtr inst; 3627118Sgblack@eecs.umich.edu /** The request for the store. */ 3637118Sgblack@eecs.umich.edu RequestPtr req; 3647118Sgblack@eecs.umich.edu /** The split requests for the store. */ 3657132Sgblack@eecs.umich.edu RequestPtr sreqLow; 3667118Sgblack@eecs.umich.edu RequestPtr sreqHigh; 3677118Sgblack@eecs.umich.edu /** The size of the store. */ 3687118Sgblack@eecs.umich.edu uint8_t size; 3696253Sgblack@eecs.umich.edu /** Whether or not the store is split into two requests. */ 3706253Sgblack@eecs.umich.edu bool isSplit; 3716253Sgblack@eecs.umich.edu /** Whether or not the store can writeback. */ 372 bool canWB; 373 /** Whether or not the store is committed. */ 374 bool committed; 375 /** Whether or not the store is completed. */ 376 bool completed; 377 /** Does this request write all zeros and thus doesn't 378 * have any data attached to it. Used for cache block zero 379 * style instructs (ARM DC ZVA; ALPHA WH64) 380 */ 381 bool isAllZeros; 382 }; 383 384 private: 385 /** The LSQUnit thread id. */ 386 ThreadID lsqID; 387 388 /** The store queue. */ 389 std::vector<SQEntry> storeQueue; 390 391 /** The load queue. */ 392 std::vector<DynInstPtr> loadQueue; 393 394 /** The number of LQ entries, plus a sentinel entry (circular queue). 395 * @todo: Consider having var that records the true number of LQ entries. 396 */ 397 unsigned LQEntries; 398 /** The number of SQ entries, plus a sentinel entry (circular queue). 399 * @todo: Consider having var that records the true number of SQ entries. 400 */ 401 unsigned SQEntries; 402 403 /** The number of places to shift addresses in the LSQ before checking 404 * for dependency violations 405 */ 406 unsigned depCheckShift; 407 408 /** Should loads be checked for dependency issues */ 409 bool checkLoads; 410 411 /** The number of load instructions in the LQ. */ 412 int loads; 413 /** The number of store instructions in the SQ. */ 414 int stores; 415 /** The number of store instructions in the SQ waiting to writeback. */ 416 int storesToWB; 417 418 /** The index of the head instruction in the LQ. */ 419 int loadHead; 420 /** The index of the tail instruction in the LQ. */ 421 int loadTail; 422 423 /** The index of the head instruction in the SQ. */ 424 int storeHead; 425 /** The index of the first instruction that may be ready to be 426 * written back, and has not yet been written back. 427 */ 428 int storeWBIdx; 429 /** The index of the tail instruction in the SQ. */ 430 int storeTail; 431 432 /// @todo Consider moving to a more advanced model with write vs read ports 433 /** The number of cache ports available each cycle. */ 434 int cachePorts; 435 436 /** The number of used cache ports in this cycle. */ 437 int usedPorts; 438 439 //list<InstSeqNum> mshrSeqNums; 440 441 /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */ 442 Addr cacheBlockMask; 443 444 /** Wire to read information from the issue stage time queue. */ 445 typename TimeBuffer<IssueStruct>::wire fromIssue; 446 447 /** Whether or not the LSQ is stalled. */ 448 bool stalled; 449 /** The store that causes the stall due to partial store to load 450 * forwarding. 451 */ 452 InstSeqNum stallingStoreIsn; 453 /** The index of the above store. */ 454 int stallingLoadIdx; 455 456 /** The packet that needs to be retried. */ 457 PacketPtr retryPkt; 458 459 /** Whehter or not a store is blocked due to the memory system. */ 460 bool isStoreBlocked; 461 462 /** Whether or not a store is in flight. */ 463 bool storeInFlight; 464 465 /** The oldest load that caused a memory ordering violation. */ 466 DynInstPtr memDepViolator; 467 468 /** Whether or not there is a packet that couldn't be sent because of 469 * a lack of cache ports. */ 470 bool hasPendingPkt; 471 472 /** The packet that is pending free cache ports. */ 473 PacketPtr pendingPkt; 474 475 /** Flag for memory model. */ 476 bool needsTSO; 477 478 // Will also need how many read/write ports the Dcache has. Or keep track 479 // of that in stage that is one level up, and only call executeLoad/Store 480 // the appropriate number of times. 481 /** Total number of loads forwaded from LSQ stores. */ 482 Stats::Scalar lsqForwLoads; 483 484 /** Total number of loads ignored due to invalid addresses. */ 485 Stats::Scalar invAddrLoads; 486 487 /** Total number of squashed loads. */ 488 Stats::Scalar lsqSquashedLoads; 489 490 /** Total number of responses from the memory system that are 491 * ignored due to the instruction already being squashed. */ 492 Stats::Scalar lsqIgnoredResponses; 493 494 /** Tota number of memory ordering violations. */ 495 Stats::Scalar lsqMemOrderViolation; 496 497 /** Total number of squashed stores. */ 498 Stats::Scalar lsqSquashedStores; 499 500 /** Total number of software prefetches ignored due to invalid addresses. */ 501 Stats::Scalar invAddrSwpfs; 502 503 /** Ready loads blocked due to partial store-forwarding. */ 504 Stats::Scalar lsqBlockedLoads; 505 506 /** Number of loads that were rescheduled. */ 507 Stats::Scalar lsqRescheduledLoads; 508 509 /** Number of times the LSQ is blocked due to the cache. */ 510 Stats::Scalar lsqCacheBlocked; 511 512 public: 513 /** Executes the load at the given index. */ 514 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 515 uint8_t *data, int load_idx); 516 517 /** Executes the store at the given index. */ 518 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 519 uint8_t *data, int store_idx); 520 521 /** Returns the index of the head load instruction. */ 522 int getLoadHead() { return loadHead; } 523 /** Returns the sequence number of the head load instruction. */ 524 InstSeqNum getLoadHeadSeqNum() 525 { 526 if (loadQueue[loadHead]) { 527 return loadQueue[loadHead]->seqNum; 528 } else { 529 return 0; 530 } 531 532 } 533 534 /** Returns the index of the head store instruction. */ 535 int getStoreHead() { return storeHead; } 536 /** Returns the sequence number of the head store instruction. */ 537 InstSeqNum getStoreHeadSeqNum() 538 { 539 if (storeQueue[storeHead].inst) { 540 return storeQueue[storeHead].inst->seqNum; 541 } else { 542 return 0; 543 } 544 545 } 546 547 /** Returns whether or not the LSQ unit is stalled. */ 548 bool isStalled() { return stalled; } 549}; 550 551template <class Impl> 552Fault 553LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 554 uint8_t *data, int load_idx) 555{ 556 DynInstPtr load_inst = loadQueue[load_idx]; 557 558 assert(load_inst); 559 560 assert(!load_inst->isExecuted()); 561 562 // Make sure this isn't a strictly ordered load 563 // A bit of a hackish way to get strictly ordered accesses to work 564 // only if they're at the head of the LSQ and are ready to commit 565 // (at the head of the ROB too). 566 if (req->isStrictlyOrdered() && 567 (load_idx != loadHead || !load_inst->isAtCommit())) { 568 iewStage->rescheduleMemInst(load_inst); 569 ++lsqRescheduledLoads; 570 DPRINTF(LSQUnit, "Strictly ordered load [sn:%lli] PC %s\n", 571 load_inst->seqNum, load_inst->pcState()); 572 573 // Must delete request now that it wasn't handed off to 574 // memory. This is quite ugly. @todo: Figure out the proper 575 // place to really handle request deletes. 576 delete req; 577 if (TheISA::HasUnalignedMemAcc && sreqLow) { 578 delete sreqLow; 579 delete sreqHigh; 580 } 581 return std::make_shared<GenericISA::M5PanicFault>( 582 "Strictly ordered load [sn:%llx] PC %s\n", 583 load_inst->seqNum, load_inst->pcState()); 584 } 585 586 // Check the SQ for any previous stores that might lead to forwarding 587 int store_idx = load_inst->sqIdx; 588 589 int store_size = 0; 590 591 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 592 "storeHead: %i addr: %#x%s\n", 593 load_idx, store_idx, storeHead, req->getPaddr(), 594 sreqLow ? " split" : ""); 595 596 if (req->isLLSC()) { 597 assert(!sreqLow); 598 // Disable recording the result temporarily. Writing to misc 599 // regs normally updates the result, but this is not the 600 // desired behavior when handling store conditionals. 601 load_inst->recordResult(false); 602 TheISA::handleLockedRead(load_inst.get(), req); 603 load_inst->recordResult(true); 604 } 605 606 if (req->isMmappedIpr()) { 607 assert(!load_inst->memData); 608 load_inst->memData = new uint8_t[64]; 609 610 ThreadContext *thread = cpu->tcBase(lsqID); 611 Cycles delay(0); 612 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 613 614 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 615 data_pkt->dataStatic(load_inst->memData); 616 delay = TheISA::handleIprRead(thread, data_pkt); 617 } else { 618 assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr()); 619 PacketPtr fst_data_pkt = new Packet(sreqLow, MemCmd::ReadReq); 620 PacketPtr snd_data_pkt = new Packet(sreqHigh, MemCmd::ReadReq); 621 622 fst_data_pkt->dataStatic(load_inst->memData); 623 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 624 625 delay = TheISA::handleIprRead(thread, fst_data_pkt); 626 Cycles delay2 = TheISA::handleIprRead(thread, snd_data_pkt); 627 if (delay2 > delay) 628 delay = delay2; 629 630 delete sreqLow; 631 delete sreqHigh; 632 delete fst_data_pkt; 633 delete snd_data_pkt; 634 } 635 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 636 cpu->schedule(wb, cpu->clockEdge(delay)); 637 return NoFault; 638 } 639 640 while (store_idx != -1) { 641 // End once we've reached the top of the LSQ 642 if (store_idx == storeWBIdx) { 643 break; 644 } 645 646 // Move the index to one younger 647 if (--store_idx < 0) 648 store_idx += SQEntries; 649 650 assert(storeQueue[store_idx].inst); 651 652 store_size = storeQueue[store_idx].size; 653 654 if (store_size == 0) 655 continue; 656 else if (storeQueue[store_idx].inst->strictlyOrdered()) 657 continue; 658 659 assert(storeQueue[store_idx].inst->effAddrValid()); 660 661 // Check if the store data is within the lower and upper bounds of 662 // addresses that the request needs. 663 bool store_has_lower_limit = 664 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 665 bool store_has_upper_limit = 666 (req->getVaddr() + req->getSize()) <= 667 (storeQueue[store_idx].inst->effAddr + store_size); 668 bool lower_load_has_store_part = 669 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 670 store_size); 671 bool upper_load_has_store_part = 672 (req->getVaddr() + req->getSize()) > 673 storeQueue[store_idx].inst->effAddr; 674 675 // If the store's data has all of the data needed, we can forward. 676 if ((store_has_lower_limit && store_has_upper_limit)) { 677 // Get shift amount for offset into the store's data. 678 int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr; 679 680 if (storeQueue[store_idx].isAllZeros) 681 memset(data, 0, req->getSize()); 682 else 683 memcpy(data, storeQueue[store_idx].data + shift_amt, 684 req->getSize()); 685 686 // Allocate memory if this is the first time a load is issued. 687 if (!load_inst->memData) { 688 load_inst->memData = new uint8_t[req->getSize()]; 689 } 690 if (storeQueue[store_idx].isAllZeros) 691 memset(load_inst->memData, 0, req->getSize()); 692 else 693 memcpy(load_inst->memData, 694 storeQueue[store_idx].data + shift_amt, req->getSize()); 695 696 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 697 "addr %#x\n", store_idx, req->getVaddr()); 698 699 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 700 data_pkt->dataStatic(load_inst->memData); 701 702 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 703 704 // We'll say this has a 1 cycle load-store forwarding latency 705 // for now. 706 // @todo: Need to make this a parameter. 707 cpu->schedule(wb, curTick()); 708 709 // Don't need to do anything special for split loads. 710 if (TheISA::HasUnalignedMemAcc && sreqLow) { 711 delete sreqLow; 712 delete sreqHigh; 713 } 714 715 ++lsqForwLoads; 716 return NoFault; 717 } else if ((store_has_lower_limit && lower_load_has_store_part) || 718 (store_has_upper_limit && upper_load_has_store_part) || 719 (lower_load_has_store_part && upper_load_has_store_part)) { 720 // This is the partial store-load forwarding case where a store 721 // has only part of the load's data. 722 723 // If it's already been written back, then don't worry about 724 // stalling on it. 725 if (storeQueue[store_idx].completed) { 726 panic("Should not check one of these"); 727 continue; 728 } 729 730 // Must stall load and force it to retry, so long as it's the oldest 731 // load that needs to do so. 732 if (!stalled || 733 (stalled && 734 load_inst->seqNum < 735 loadQueue[stallingLoadIdx]->seqNum)) { 736 stalled = true; 737 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 738 stallingLoadIdx = load_idx; 739 } 740 741 // Tell IQ/mem dep unit that this instruction will need to be 742 // rescheduled eventually 743 iewStage->rescheduleMemInst(load_inst); 744 load_inst->clearIssued(); 745 ++lsqRescheduledLoads; 746 747 // Do not generate a writeback event as this instruction is not 748 // complete. 749 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 750 "Store idx %i to load addr %#x\n", 751 store_idx, req->getVaddr()); 752 753 // Must delete request now that it wasn't handed off to 754 // memory. This is quite ugly. @todo: Figure out the 755 // proper place to really handle request deletes. 756 delete req; 757 if (TheISA::HasUnalignedMemAcc && sreqLow) { 758 delete sreqLow; 759 delete sreqHigh; 760 } 761 762 return NoFault; 763 } 764 } 765 766 // If there's no forwarding case, then go access memory 767 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 768 load_inst->seqNum, load_inst->pcState()); 769 770 // Allocate memory if this is the first time a load is issued. 771 if (!load_inst->memData) { 772 load_inst->memData = new uint8_t[req->getSize()]; 773 } 774 775 ++usedPorts; 776 777 // if we the cache is not blocked, do cache access 778 bool completedFirst = false; 779 PacketPtr data_pkt = Packet::createRead(req); 780 PacketPtr fst_data_pkt = NULL; 781 PacketPtr snd_data_pkt = NULL; 782 783 data_pkt->dataStatic(load_inst->memData); 784 785 LSQSenderState *state = new LSQSenderState; 786 state->isLoad = true; 787 state->idx = load_idx; 788 state->inst = load_inst; 789 data_pkt->senderState = state; 790 791 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 792 // Point the first packet at the main data packet. 793 fst_data_pkt = data_pkt; 794 } else { 795 // Create the split packets. 796 fst_data_pkt = Packet::createRead(sreqLow); 797 snd_data_pkt = Packet::createRead(sreqHigh); 798 799 fst_data_pkt->dataStatic(load_inst->memData); 800 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 801 802 fst_data_pkt->senderState = state; 803 snd_data_pkt->senderState = state; 804 805 state->isSplit = true; 806 state->outstanding = 2; 807 state->mainPkt = data_pkt; 808 } 809 810 bool successful_load = true; 811 if (!dcachePort->sendTimingReq(fst_data_pkt)) { 812 successful_load = false; 813 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 814 completedFirst = true; 815 816 // The first packet was sent without problems, so send this one 817 // too. If there is a problem with this packet then the whole 818 // load will be squashed, so indicate this to the state object. 819 // The first packet will return in completeDataAccess and be 820 // handled there. 821 ++usedPorts; 822 if (!dcachePort->sendTimingReq(snd_data_pkt)) { 823 // The main packet will be deleted in completeDataAccess. 824 state->complete(); 825 // Signify to 1st half that the 2nd half was blocked via state 826 state->cacheBlocked = true; 827 successful_load = false; 828 } 829 } 830 831 // If the cache was blocked, or has become blocked due to the access, 832 // handle it. 833 if (!successful_load) { 834 if (!sreqLow) { 835 // Packet wasn't split, just delete main packet info 836 delete state; 837 delete req; 838 delete data_pkt; 839 } 840 841 if (TheISA::HasUnalignedMemAcc && sreqLow) { 842 if (!completedFirst) { 843 // Split packet, but first failed. Delete all state. 844 delete state; 845 delete req; 846 delete data_pkt; 847 delete fst_data_pkt; 848 delete snd_data_pkt; 849 delete sreqLow; 850 delete sreqHigh; 851 sreqLow = NULL; 852 sreqHigh = NULL; 853 } else { 854 // Can't delete main packet data or state because first packet 855 // was sent to the memory system 856 delete data_pkt; 857 delete req; 858 delete sreqHigh; 859 delete snd_data_pkt; 860 sreqHigh = NULL; 861 } 862 } 863 864 ++lsqCacheBlocked; 865 866 iewStage->blockMemInst(load_inst); 867 868 // No fault occurred, even though the interface is blocked. 869 return NoFault; 870 } 871 872 return NoFault; 873} 874 875template <class Impl> 876Fault 877LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 878 uint8_t *data, int store_idx) 879{ 880 assert(storeQueue[store_idx].inst); 881 882 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x" 883 " | storeHead:%i [sn:%i]\n", 884 store_idx, req->getPaddr(), storeHead, 885 storeQueue[store_idx].inst->seqNum); 886 887 storeQueue[store_idx].req = req; 888 storeQueue[store_idx].sreqLow = sreqLow; 889 storeQueue[store_idx].sreqHigh = sreqHigh; 890 unsigned size = req->getSize(); 891 storeQueue[store_idx].size = size; 892 storeQueue[store_idx].isAllZeros = req->getFlags() & Request::CACHE_BLOCK_ZERO; 893 assert(size <= sizeof(storeQueue[store_idx].data) || 894 (req->getFlags() & Request::CACHE_BLOCK_ZERO)); 895 896 // Split stores can only occur in ISAs with unaligned memory accesses. If 897 // a store request has been split, sreqLow and sreqHigh will be non-null. 898 if (TheISA::HasUnalignedMemAcc && sreqLow) { 899 storeQueue[store_idx].isSplit = true; 900 } 901 902 if (!(req->getFlags() & Request::CACHE_BLOCK_ZERO)) 903 memcpy(storeQueue[store_idx].data, data, size); 904 905 // This function only writes the data to the store queue, so no fault 906 // can happen here. 907 return NoFault; 908} 909 910#endif // __CPU_O3_LSQ_UNIT_HH__ 911