1/* 2 * Copyright (c) 2004-2006 The Regents of The University of Michigan 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are 7 * met: redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer; 9 * redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution; 12 * neither the name of the copyright holders nor the names of its 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Authors: Kevin Lim 29 * Korey Sewell 30 */ 31 32#ifndef __CPU_O3_LSQ_UNIT_HH__ 33#define __CPU_O3_LSQ_UNIT_HH__ 34 35#include <algorithm> 36#include <cstring> 37#include <map> 38#include <queue> 39 40#include "arch/faults.hh" 41#include "arch/locked_mem.hh" 42#include "config/full_system.hh" 43#include "config/the_isa.hh" 44#include "base/fast_alloc.hh" 45#include "base/hashmap.hh" 46#include "cpu/inst_seq.hh" 47#include "mem/packet.hh" 48#include "mem/port.hh" 49 50class DerivO3CPUParams; 51 52/** 53 * Class that implements the actual LQ and SQ for each specific 54 * thread. Both are circular queues; load entries are freed upon 55 * committing, while store entries are freed once they writeback. The 56 * LSQUnit tracks if there are memory ordering violations, and also 57 * detects partial load to store forwarding cases (a store only has 58 * part of a load's data) that requires the load to wait until the 59 * store writes back. In the former case it holds onto the instruction 60 * until the dependence unit looks at it, and in the latter it stalls 61 * the LSQ until the store writes back. At that point the load is 62 * replayed. 63 */ 64template <class Impl> 65class LSQUnit { 66 public: 67 typedef typename Impl::O3CPU O3CPU; 68 typedef typename Impl::DynInstPtr DynInstPtr; 69 typedef typename Impl::CPUPol::IEW IEW; 70 typedef typename Impl::CPUPol::LSQ LSQ; 71 typedef typename Impl::CPUPol::IssueStruct IssueStruct; 72 73 public: 74 /** Constructs an LSQ unit. init() must be called prior to use. */ 75 LSQUnit(); 76 77 /** Initializes the LSQ unit with the specified number of entries. */ 78 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 79 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 80 unsigned id); 81 82 /** Returns the name of the LSQ unit. */ 83 std::string name() const; 84 85 /** Registers statistics. */ 86 void regStats(); 87 88 /** Sets the pointer to the dcache port. */ 89 void setDcachePort(Port *dcache_port); 90 91 /** Switches out LSQ unit. */ 92 void switchOut(); 93 94 /** Takes over from another CPU's thread. */ 95 void takeOverFrom(); 96 97 /** Returns if the LSQ is switched out. */ 98 bool isSwitchedOut() { return switchedOut; } 99 100 /** Ticks the LSQ unit, which in this case only resets the number of 101 * used cache ports. 102 * @todo: Move the number of used ports up to the LSQ level so it can 103 * be shared by all LSQ units. 104 */ 105 void tick() { usedPorts = 0; } 106 107 /** Inserts an instruction. */ 108 void insert(DynInstPtr &inst); 109 /** Inserts a load instruction. */ 110 void insertLoad(DynInstPtr &load_inst); 111 /** Inserts a store instruction. */ 112 void insertStore(DynInstPtr &store_inst); 113
| 1/* 2 * Copyright (c) 2004-2006 The Regents of The University of Michigan 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are 7 * met: redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer; 9 * redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution; 12 * neither the name of the copyright holders nor the names of its 13 * contributors may be used to endorse or promote products derived from 14 * this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Authors: Kevin Lim 29 * Korey Sewell 30 */ 31 32#ifndef __CPU_O3_LSQ_UNIT_HH__ 33#define __CPU_O3_LSQ_UNIT_HH__ 34 35#include <algorithm> 36#include <cstring> 37#include <map> 38#include <queue> 39 40#include "arch/faults.hh" 41#include "arch/locked_mem.hh" 42#include "config/full_system.hh" 43#include "config/the_isa.hh" 44#include "base/fast_alloc.hh" 45#include "base/hashmap.hh" 46#include "cpu/inst_seq.hh" 47#include "mem/packet.hh" 48#include "mem/port.hh" 49 50class DerivO3CPUParams; 51 52/** 53 * Class that implements the actual LQ and SQ for each specific 54 * thread. Both are circular queues; load entries are freed upon 55 * committing, while store entries are freed once they writeback. The 56 * LSQUnit tracks if there are memory ordering violations, and also 57 * detects partial load to store forwarding cases (a store only has 58 * part of a load's data) that requires the load to wait until the 59 * store writes back. In the former case it holds onto the instruction 60 * until the dependence unit looks at it, and in the latter it stalls 61 * the LSQ until the store writes back. At that point the load is 62 * replayed. 63 */ 64template <class Impl> 65class LSQUnit { 66 public: 67 typedef typename Impl::O3CPU O3CPU; 68 typedef typename Impl::DynInstPtr DynInstPtr; 69 typedef typename Impl::CPUPol::IEW IEW; 70 typedef typename Impl::CPUPol::LSQ LSQ; 71 typedef typename Impl::CPUPol::IssueStruct IssueStruct; 72 73 public: 74 /** Constructs an LSQ unit. init() must be called prior to use. */ 75 LSQUnit(); 76 77 /** Initializes the LSQ unit with the specified number of entries. */ 78 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 79 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 80 unsigned id); 81 82 /** Returns the name of the LSQ unit. */ 83 std::string name() const; 84 85 /** Registers statistics. */ 86 void regStats(); 87 88 /** Sets the pointer to the dcache port. */ 89 void setDcachePort(Port *dcache_port); 90 91 /** Switches out LSQ unit. */ 92 void switchOut(); 93 94 /** Takes over from another CPU's thread. */ 95 void takeOverFrom(); 96 97 /** Returns if the LSQ is switched out. */ 98 bool isSwitchedOut() { return switchedOut; } 99 100 /** Ticks the LSQ unit, which in this case only resets the number of 101 * used cache ports. 102 * @todo: Move the number of used ports up to the LSQ level so it can 103 * be shared by all LSQ units. 104 */ 105 void tick() { usedPorts = 0; } 106 107 /** Inserts an instruction. */ 108 void insert(DynInstPtr &inst); 109 /** Inserts a load instruction. */ 110 void insertLoad(DynInstPtr &load_inst); 111 /** Inserts a store instruction. */ 112 void insertStore(DynInstPtr &store_inst); 113
|
114 /** Executes a load instruction. */ 115 Fault executeLoad(DynInstPtr &inst); 116 117 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 118 /** Executes a store instruction. */ 119 Fault executeStore(DynInstPtr &inst); 120 121 /** Commits the head load. */ 122 void commitLoad(); 123 /** Commits loads older than a specific sequence number. */ 124 void commitLoads(InstSeqNum &youngest_inst); 125 126 /** Commits stores older than a specific sequence number. */ 127 void commitStores(InstSeqNum &youngest_inst); 128 129 /** Writes back stores. */ 130 void writebackStores(); 131 132 /** Completes the data access that has been returned from the 133 * memory system. */ 134 void completeDataAccess(PacketPtr pkt); 135 136 /** Clears all the entries in the LQ. */ 137 void clearLQ(); 138 139 /** Clears all the entries in the SQ. */ 140 void clearSQ(); 141 142 /** Resizes the LQ to a given size. */ 143 void resizeLQ(unsigned size); 144 145 /** Resizes the SQ to a given size. */ 146 void resizeSQ(unsigned size); 147 148 /** Squashes all instructions younger than a specific sequence number. */ 149 void squash(const InstSeqNum &squashed_num); 150 151 /** Returns if there is a memory ordering violation. Value is reset upon 152 * call to getMemDepViolator(). 153 */ 154 bool violation() { return memDepViolator; } 155 156 /** Returns the memory ordering violator. */ 157 DynInstPtr getMemDepViolator(); 158 159 /** Returns if a load became blocked due to the memory system. */ 160 bool loadBlocked() 161 { return isLoadBlocked; } 162 163 /** Clears the signal that a load became blocked. */ 164 void clearLoadBlocked() 165 { isLoadBlocked = false; } 166 167 /** Returns if the blocked load was handled. */ 168 bool isLoadBlockedHandled() 169 { return loadBlockedHandled; } 170 171 /** Records the blocked load as being handled. */ 172 void setLoadBlockedHandled() 173 { loadBlockedHandled = true; } 174 175 /** Returns the number of free entries (min of free LQ and SQ entries). */ 176 unsigned numFreeEntries(); 177 178 /** Returns the number of loads ready to execute. */ 179 int numLoadsReady(); 180 181 /** Returns the number of loads in the LQ. */ 182 int numLoads() { return loads; } 183 184 /** Returns the number of stores in the SQ. */ 185 int numStores() { return stores; } 186 187 /** Returns if either the LQ or SQ is full. */ 188 bool isFull() { return lqFull() || sqFull(); } 189 190 /** Returns if the LQ is full. */ 191 bool lqFull() { return loads >= (LQEntries - 1); } 192 193 /** Returns if the SQ is full. */ 194 bool sqFull() { return stores >= (SQEntries - 1); } 195 196 /** Returns the number of instructions in the LSQ. */ 197 unsigned getCount() { return loads + stores; } 198 199 /** Returns if there are any stores to writeback. */ 200 bool hasStoresToWB() { return storesToWB; } 201 202 /** Returns the number of stores to writeback. */ 203 int numStoresToWB() { return storesToWB; } 204 205 /** Returns if the LSQ unit will writeback on this cycle. */ 206 bool willWB() { return storeQueue[storeWBIdx].canWB && 207 !storeQueue[storeWBIdx].completed && 208 !isStoreBlocked; } 209 210 /** Handles doing the retry. */ 211 void recvRetry(); 212 213 private: 214 /** Writes back the instruction, sending it to IEW. */ 215 void writeback(DynInstPtr &inst, PacketPtr pkt); 216 217 /** Writes back a store that couldn't be completed the previous cycle. */ 218 void writebackPendingStore(); 219 220 /** Handles completing the send of a store to memory. */ 221 void storePostSend(PacketPtr pkt); 222 223 /** Completes the store at the specified index. */ 224 void completeStore(int store_idx); 225 226 /** Attempts to send a store to the cache. */ 227 bool sendStore(PacketPtr data_pkt); 228 229 /** Increments the given store index (circular queue). */ 230 inline void incrStIdx(int &store_idx); 231 /** Decrements the given store index (circular queue). */ 232 inline void decrStIdx(int &store_idx); 233 /** Increments the given load index (circular queue). */ 234 inline void incrLdIdx(int &load_idx); 235 /** Decrements the given load index (circular queue). */ 236 inline void decrLdIdx(int &load_idx); 237 238 public: 239 /** Debugging function to dump instructions in the LSQ. */ 240 void dumpInsts(); 241 242 private: 243 /** Pointer to the CPU. */ 244 O3CPU *cpu; 245 246 /** Pointer to the IEW stage. */ 247 IEW *iewStage; 248 249 /** Pointer to the LSQ. */ 250 LSQ *lsq; 251 252 /** Pointer to the dcache port. Used only for sending. */ 253 Port *dcachePort; 254 255 /** Derived class to hold any sender state the LSQ needs. */ 256 class LSQSenderState : public Packet::SenderState, public FastAlloc 257 { 258 public: 259 /** Default constructor. */ 260 LSQSenderState() 261 : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 262 mainPkt(NULL), pendingPacket(NULL) 263 { } 264 265 /** Instruction who initiated the access to memory. */ 266 DynInstPtr inst; 267 /** Whether or not it is a load. */ 268 bool isLoad; 269 /** The LQ/SQ index of the instruction. */ 270 int idx; 271 /** Whether or not the instruction will need to writeback. */ 272 bool noWB; 273 /** Whether or not this access is split in two. */ 274 bool isSplit; 275 /** Whether or not there is a packet that needs sending. */ 276 bool pktToSend; 277 /** Number of outstanding packets to complete. */ 278 int outstanding; 279 /** The main packet from a split load, used during writeback. */ 280 PacketPtr mainPkt; 281 /** A second packet from a split store that needs sending. */ 282 PacketPtr pendingPacket; 283 284 /** Completes a packet and returns whether the access is finished. */ 285 inline bool complete() { return --outstanding == 0; } 286 }; 287 288 /** Writeback event, specifically for when stores forward data to loads. */ 289 class WritebackEvent : public Event { 290 public: 291 /** Constructs a writeback event. */ 292 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 293 294 /** Processes the writeback event. */ 295 void process(); 296 297 /** Returns the description of this event. */ 298 const char *description() const; 299 300 private: 301 /** Instruction whose results are being written back. */ 302 DynInstPtr inst; 303 304 /** The packet that would have been sent to memory. */ 305 PacketPtr pkt; 306 307 /** The pointer to the LSQ unit that issued the store. */ 308 LSQUnit<Impl> *lsqPtr; 309 }; 310 311 public: 312 struct SQEntry { 313 /** Constructs an empty store queue entry. */ 314 SQEntry() 315 : inst(NULL), req(NULL), size(0), 316 canWB(0), committed(0), completed(0) 317 { 318 std::memset(data, 0, sizeof(data)); 319 } 320 321 /** Constructs a store queue entry for a given instruction. */ 322 SQEntry(DynInstPtr &_inst) 323 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 324 isSplit(0), canWB(0), committed(0), completed(0) 325 { 326 std::memset(data, 0, sizeof(data)); 327 } 328 329 /** The store instruction. */ 330 DynInstPtr inst; 331 /** The request for the store. */ 332 RequestPtr req; 333 /** The split requests for the store. */ 334 RequestPtr sreqLow; 335 RequestPtr sreqHigh; 336 /** The size of the store. */ 337 int size; 338 /** The store data. */ 339 char data[16]; 340 /** Whether or not the store is split into two requests. */ 341 bool isSplit; 342 /** Whether or not the store can writeback. */ 343 bool canWB; 344 /** Whether or not the store is committed. */ 345 bool committed; 346 /** Whether or not the store is completed. */ 347 bool completed; 348 }; 349 350 private: 351 /** The LSQUnit thread id. */ 352 ThreadID lsqID; 353 354 /** The store queue. */ 355 std::vector<SQEntry> storeQueue; 356 357 /** The load queue. */ 358 std::vector<DynInstPtr> loadQueue; 359 360 /** The number of LQ entries, plus a sentinel entry (circular queue). 361 * @todo: Consider having var that records the true number of LQ entries. 362 */ 363 unsigned LQEntries; 364 /** The number of SQ entries, plus a sentinel entry (circular queue). 365 * @todo: Consider having var that records the true number of SQ entries. 366 */ 367 unsigned SQEntries; 368
| 120 /** Executes a load instruction. */ 121 Fault executeLoad(DynInstPtr &inst); 122 123 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 124 /** Executes a store instruction. */ 125 Fault executeStore(DynInstPtr &inst); 126 127 /** Commits the head load. */ 128 void commitLoad(); 129 /** Commits loads older than a specific sequence number. */ 130 void commitLoads(InstSeqNum &youngest_inst); 131 132 /** Commits stores older than a specific sequence number. */ 133 void commitStores(InstSeqNum &youngest_inst); 134 135 /** Writes back stores. */ 136 void writebackStores(); 137 138 /** Completes the data access that has been returned from the 139 * memory system. */ 140 void completeDataAccess(PacketPtr pkt); 141 142 /** Clears all the entries in the LQ. */ 143 void clearLQ(); 144 145 /** Clears all the entries in the SQ. */ 146 void clearSQ(); 147 148 /** Resizes the LQ to a given size. */ 149 void resizeLQ(unsigned size); 150 151 /** Resizes the SQ to a given size. */ 152 void resizeSQ(unsigned size); 153 154 /** Squashes all instructions younger than a specific sequence number. */ 155 void squash(const InstSeqNum &squashed_num); 156 157 /** Returns if there is a memory ordering violation. Value is reset upon 158 * call to getMemDepViolator(). 159 */ 160 bool violation() { return memDepViolator; } 161 162 /** Returns the memory ordering violator. */ 163 DynInstPtr getMemDepViolator(); 164 165 /** Returns if a load became blocked due to the memory system. */ 166 bool loadBlocked() 167 { return isLoadBlocked; } 168 169 /** Clears the signal that a load became blocked. */ 170 void clearLoadBlocked() 171 { isLoadBlocked = false; } 172 173 /** Returns if the blocked load was handled. */ 174 bool isLoadBlockedHandled() 175 { return loadBlockedHandled; } 176 177 /** Records the blocked load as being handled. */ 178 void setLoadBlockedHandled() 179 { loadBlockedHandled = true; } 180 181 /** Returns the number of free entries (min of free LQ and SQ entries). */ 182 unsigned numFreeEntries(); 183 184 /** Returns the number of loads ready to execute. */ 185 int numLoadsReady(); 186 187 /** Returns the number of loads in the LQ. */ 188 int numLoads() { return loads; } 189 190 /** Returns the number of stores in the SQ. */ 191 int numStores() { return stores; } 192 193 /** Returns if either the LQ or SQ is full. */ 194 bool isFull() { return lqFull() || sqFull(); } 195 196 /** Returns if the LQ is full. */ 197 bool lqFull() { return loads >= (LQEntries - 1); } 198 199 /** Returns if the SQ is full. */ 200 bool sqFull() { return stores >= (SQEntries - 1); } 201 202 /** Returns the number of instructions in the LSQ. */ 203 unsigned getCount() { return loads + stores; } 204 205 /** Returns if there are any stores to writeback. */ 206 bool hasStoresToWB() { return storesToWB; } 207 208 /** Returns the number of stores to writeback. */ 209 int numStoresToWB() { return storesToWB; } 210 211 /** Returns if the LSQ unit will writeback on this cycle. */ 212 bool willWB() { return storeQueue[storeWBIdx].canWB && 213 !storeQueue[storeWBIdx].completed && 214 !isStoreBlocked; } 215 216 /** Handles doing the retry. */ 217 void recvRetry(); 218 219 private: 220 /** Writes back the instruction, sending it to IEW. */ 221 void writeback(DynInstPtr &inst, PacketPtr pkt); 222 223 /** Writes back a store that couldn't be completed the previous cycle. */ 224 void writebackPendingStore(); 225 226 /** Handles completing the send of a store to memory. */ 227 void storePostSend(PacketPtr pkt); 228 229 /** Completes the store at the specified index. */ 230 void completeStore(int store_idx); 231 232 /** Attempts to send a store to the cache. */ 233 bool sendStore(PacketPtr data_pkt); 234 235 /** Increments the given store index (circular queue). */ 236 inline void incrStIdx(int &store_idx); 237 /** Decrements the given store index (circular queue). */ 238 inline void decrStIdx(int &store_idx); 239 /** Increments the given load index (circular queue). */ 240 inline void incrLdIdx(int &load_idx); 241 /** Decrements the given load index (circular queue). */ 242 inline void decrLdIdx(int &load_idx); 243 244 public: 245 /** Debugging function to dump instructions in the LSQ. */ 246 void dumpInsts(); 247 248 private: 249 /** Pointer to the CPU. */ 250 O3CPU *cpu; 251 252 /** Pointer to the IEW stage. */ 253 IEW *iewStage; 254 255 /** Pointer to the LSQ. */ 256 LSQ *lsq; 257 258 /** Pointer to the dcache port. Used only for sending. */ 259 Port *dcachePort; 260 261 /** Derived class to hold any sender state the LSQ needs. */ 262 class LSQSenderState : public Packet::SenderState, public FastAlloc 263 { 264 public: 265 /** Default constructor. */ 266 LSQSenderState() 267 : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 268 mainPkt(NULL), pendingPacket(NULL) 269 { } 270 271 /** Instruction who initiated the access to memory. */ 272 DynInstPtr inst; 273 /** Whether or not it is a load. */ 274 bool isLoad; 275 /** The LQ/SQ index of the instruction. */ 276 int idx; 277 /** Whether or not the instruction will need to writeback. */ 278 bool noWB; 279 /** Whether or not this access is split in two. */ 280 bool isSplit; 281 /** Whether or not there is a packet that needs sending. */ 282 bool pktToSend; 283 /** Number of outstanding packets to complete. */ 284 int outstanding; 285 /** The main packet from a split load, used during writeback. */ 286 PacketPtr mainPkt; 287 /** A second packet from a split store that needs sending. */ 288 PacketPtr pendingPacket; 289 290 /** Completes a packet and returns whether the access is finished. */ 291 inline bool complete() { return --outstanding == 0; } 292 }; 293 294 /** Writeback event, specifically for when stores forward data to loads. */ 295 class WritebackEvent : public Event { 296 public: 297 /** Constructs a writeback event. */ 298 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 299 300 /** Processes the writeback event. */ 301 void process(); 302 303 /** Returns the description of this event. */ 304 const char *description() const; 305 306 private: 307 /** Instruction whose results are being written back. */ 308 DynInstPtr inst; 309 310 /** The packet that would have been sent to memory. */ 311 PacketPtr pkt; 312 313 /** The pointer to the LSQ unit that issued the store. */ 314 LSQUnit<Impl> *lsqPtr; 315 }; 316 317 public: 318 struct SQEntry { 319 /** Constructs an empty store queue entry. */ 320 SQEntry() 321 : inst(NULL), req(NULL), size(0), 322 canWB(0), committed(0), completed(0) 323 { 324 std::memset(data, 0, sizeof(data)); 325 } 326 327 /** Constructs a store queue entry for a given instruction. */ 328 SQEntry(DynInstPtr &_inst) 329 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 330 isSplit(0), canWB(0), committed(0), completed(0) 331 { 332 std::memset(data, 0, sizeof(data)); 333 } 334 335 /** The store instruction. */ 336 DynInstPtr inst; 337 /** The request for the store. */ 338 RequestPtr req; 339 /** The split requests for the store. */ 340 RequestPtr sreqLow; 341 RequestPtr sreqHigh; 342 /** The size of the store. */ 343 int size; 344 /** The store data. */ 345 char data[16]; 346 /** Whether or not the store is split into two requests. */ 347 bool isSplit; 348 /** Whether or not the store can writeback. */ 349 bool canWB; 350 /** Whether or not the store is committed. */ 351 bool committed; 352 /** Whether or not the store is completed. */ 353 bool completed; 354 }; 355 356 private: 357 /** The LSQUnit thread id. */ 358 ThreadID lsqID; 359 360 /** The store queue. */ 361 std::vector<SQEntry> storeQueue; 362 363 /** The load queue. */ 364 std::vector<DynInstPtr> loadQueue; 365 366 /** The number of LQ entries, plus a sentinel entry (circular queue). 367 * @todo: Consider having var that records the true number of LQ entries. 368 */ 369 unsigned LQEntries; 370 /** The number of SQ entries, plus a sentinel entry (circular queue). 371 * @todo: Consider having var that records the true number of SQ entries. 372 */ 373 unsigned SQEntries; 374
|
369 /** The number of load instructions in the LQ. */ 370 int loads; 371 /** The number of store instructions in the SQ. */ 372 int stores; 373 /** The number of store instructions in the SQ waiting to writeback. */ 374 int storesToWB; 375 376 /** The index of the head instruction in the LQ. */ 377 int loadHead; 378 /** The index of the tail instruction in the LQ. */ 379 int loadTail; 380 381 /** The index of the head instruction in the SQ. */ 382 int storeHead; 383 /** The index of the first instruction that may be ready to be 384 * written back, and has not yet been written back. 385 */ 386 int storeWBIdx; 387 /** The index of the tail instruction in the SQ. */ 388 int storeTail; 389 390 /// @todo Consider moving to a more advanced model with write vs read ports 391 /** The number of cache ports available each cycle. */ 392 int cachePorts; 393 394 /** The number of used cache ports in this cycle. */ 395 int usedPorts; 396 397 /** Is the LSQ switched out. */ 398 bool switchedOut; 399 400 //list<InstSeqNum> mshrSeqNums; 401 402 /** Wire to read information from the issue stage time queue. */ 403 typename TimeBuffer<IssueStruct>::wire fromIssue; 404 405 /** Whether or not the LSQ is stalled. */ 406 bool stalled; 407 /** The store that causes the stall due to partial store to load 408 * forwarding. 409 */ 410 InstSeqNum stallingStoreIsn; 411 /** The index of the above store. */ 412 int stallingLoadIdx; 413 414 /** The packet that needs to be retried. */ 415 PacketPtr retryPkt; 416 417 /** Whehter or not a store is blocked due to the memory system. */ 418 bool isStoreBlocked; 419 420 /** Whether or not a load is blocked due to the memory system. */ 421 bool isLoadBlocked; 422 423 /** Has the blocked load been handled. */ 424 bool loadBlockedHandled; 425 426 /** The sequence number of the blocked load. */ 427 InstSeqNum blockedLoadSeqNum; 428 429 /** The oldest load that caused a memory ordering violation. */ 430 DynInstPtr memDepViolator; 431 432 /** Whether or not there is a packet that couldn't be sent because of 433 * a lack of cache ports. */ 434 bool hasPendingPkt; 435 436 /** The packet that is pending free cache ports. */ 437 PacketPtr pendingPkt; 438 439 // Will also need how many read/write ports the Dcache has. Or keep track 440 // of that in stage that is one level up, and only call executeLoad/Store 441 // the appropriate number of times. 442 /** Total number of loads forwaded from LSQ stores. */ 443 Stats::Scalar lsqForwLoads; 444 445 /** Total number of loads ignored due to invalid addresses. */ 446 Stats::Scalar invAddrLoads; 447 448 /** Total number of squashed loads. */ 449 Stats::Scalar lsqSquashedLoads; 450 451 /** Total number of responses from the memory system that are 452 * ignored due to the instruction already being squashed. */ 453 Stats::Scalar lsqIgnoredResponses; 454 455 /** Tota number of memory ordering violations. */ 456 Stats::Scalar lsqMemOrderViolation; 457 458 /** Total number of squashed stores. */ 459 Stats::Scalar lsqSquashedStores; 460 461 /** Total number of software prefetches ignored due to invalid addresses. */ 462 Stats::Scalar invAddrSwpfs; 463 464 /** Ready loads blocked due to partial store-forwarding. */ 465 Stats::Scalar lsqBlockedLoads; 466 467 /** Number of loads that were rescheduled. */ 468 Stats::Scalar lsqRescheduledLoads; 469 470 /** Number of times the LSQ is blocked due to the cache. */ 471 Stats::Scalar lsqCacheBlocked; 472 473 public: 474 /** Executes the load at the given index. */ 475 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 476 uint8_t *data, int load_idx); 477 478 /** Executes the store at the given index. */ 479 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 480 uint8_t *data, int store_idx); 481 482 /** Returns the index of the head load instruction. */ 483 int getLoadHead() { return loadHead; } 484 /** Returns the sequence number of the head load instruction. */ 485 InstSeqNum getLoadHeadSeqNum() 486 { 487 if (loadQueue[loadHead]) { 488 return loadQueue[loadHead]->seqNum; 489 } else { 490 return 0; 491 } 492 493 } 494 495 /** Returns the index of the head store instruction. */ 496 int getStoreHead() { return storeHead; } 497 /** Returns the sequence number of the head store instruction. */ 498 InstSeqNum getStoreHeadSeqNum() 499 { 500 if (storeQueue[storeHead].inst) { 501 return storeQueue[storeHead].inst->seqNum; 502 } else { 503 return 0; 504 } 505 506 } 507 508 /** Returns whether or not the LSQ unit is stalled. */ 509 bool isStalled() { return stalled; } 510}; 511 512template <class Impl> 513Fault 514LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 515 uint8_t *data, int load_idx) 516{ 517 DynInstPtr load_inst = loadQueue[load_idx]; 518 519 assert(load_inst); 520 521 assert(!load_inst->isExecuted()); 522 523 // Make sure this isn't an uncacheable access 524 // A bit of a hackish way to get uncached accesses to work only if they're 525 // at the head of the LSQ and are ready to commit (at the head of the ROB 526 // too). 527 if (req->isUncacheable() && 528 (load_idx != loadHead || !load_inst->isAtCommit())) { 529 iewStage->rescheduleMemInst(load_inst); 530 ++lsqRescheduledLoads; 531 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n", 532 load_inst->seqNum, load_inst->pcState()); 533 534 // Must delete request now that it wasn't handed off to 535 // memory. This is quite ugly. @todo: Figure out the proper 536 // place to really handle request deletes. 537 delete req; 538 if (TheISA::HasUnalignedMemAcc && sreqLow) { 539 delete sreqLow; 540 delete sreqHigh; 541 } 542 return TheISA::genMachineCheckFault(); 543 } 544 545 // Check the SQ for any previous stores that might lead to forwarding 546 int store_idx = load_inst->sqIdx; 547 548 int store_size = 0; 549 550 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 551 "storeHead: %i addr: %#x%s\n", 552 load_idx, store_idx, storeHead, req->getPaddr(), 553 sreqLow ? " split" : ""); 554 555 if (req->isLLSC()) { 556 assert(!sreqLow); 557 // Disable recording the result temporarily. Writing to misc 558 // regs normally updates the result, but this is not the 559 // desired behavior when handling store conditionals. 560 load_inst->recordResult = false; 561 TheISA::handleLockedRead(load_inst.get(), req); 562 load_inst->recordResult = true; 563 } 564 565 while (store_idx != -1) { 566 // End once we've reached the top of the LSQ 567 if (store_idx == storeWBIdx) { 568 break; 569 } 570 571 // Move the index to one younger 572 if (--store_idx < 0) 573 store_idx += SQEntries; 574 575 assert(storeQueue[store_idx].inst); 576 577 store_size = storeQueue[store_idx].size; 578 579 if (store_size == 0) 580 continue; 581 else if (storeQueue[store_idx].inst->uncacheable()) 582 continue; 583 584 assert(storeQueue[store_idx].inst->effAddrValid); 585 586 // Check if the store data is within the lower and upper bounds of 587 // addresses that the request needs. 588 bool store_has_lower_limit = 589 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 590 bool store_has_upper_limit = 591 (req->getVaddr() + req->getSize()) <= 592 (storeQueue[store_idx].inst->effAddr + store_size); 593 bool lower_load_has_store_part = 594 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 595 store_size); 596 bool upper_load_has_store_part = 597 (req->getVaddr() + req->getSize()) > 598 storeQueue[store_idx].inst->effAddr; 599 600 // If the store's data has all of the data needed, we can forward. 601 if ((store_has_lower_limit && store_has_upper_limit)) { 602 // Get shift amount for offset into the store's data. 603 int shift_amt = req->getVaddr() & (store_size - 1); 604 605 memcpy(data, storeQueue[store_idx].data + shift_amt, 606 req->getSize()); 607 608 assert(!load_inst->memData); 609 load_inst->memData = new uint8_t[64]; 610 611 memcpy(load_inst->memData, 612 storeQueue[store_idx].data + shift_amt, req->getSize()); 613 614 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 615 "addr %#x, data %#x\n", 616 store_idx, req->getVaddr(), data); 617 618 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 619 Packet::Broadcast); 620 data_pkt->dataStatic(load_inst->memData); 621 622 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 623 624 // We'll say this has a 1 cycle load-store forwarding latency 625 // for now. 626 // @todo: Need to make this a parameter. 627 cpu->schedule(wb, curTick()); 628 629 // Don't need to do anything special for split loads. 630 if (TheISA::HasUnalignedMemAcc && sreqLow) { 631 delete sreqLow; 632 delete sreqHigh; 633 } 634 635 ++lsqForwLoads; 636 return NoFault; 637 } else if ((store_has_lower_limit && lower_load_has_store_part) || 638 (store_has_upper_limit && upper_load_has_store_part) || 639 (lower_load_has_store_part && upper_load_has_store_part)) { 640 // This is the partial store-load forwarding case where a store 641 // has only part of the load's data. 642 643 // If it's already been written back, then don't worry about 644 // stalling on it. 645 if (storeQueue[store_idx].completed) { 646 panic("Should not check one of these"); 647 continue; 648 } 649 650 // Must stall load and force it to retry, so long as it's the oldest 651 // load that needs to do so. 652 if (!stalled || 653 (stalled && 654 load_inst->seqNum < 655 loadQueue[stallingLoadIdx]->seqNum)) { 656 stalled = true; 657 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 658 stallingLoadIdx = load_idx; 659 } 660 661 // Tell IQ/mem dep unit that this instruction will need to be 662 // rescheduled eventually 663 iewStage->rescheduleMemInst(load_inst); 664 iewStage->decrWb(load_inst->seqNum); 665 load_inst->clearIssued(); 666 ++lsqRescheduledLoads; 667 668 // Do not generate a writeback event as this instruction is not 669 // complete. 670 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 671 "Store idx %i to load addr %#x\n", 672 store_idx, req->getVaddr()); 673 674 // Must delete request now that it wasn't handed off to 675 // memory. This is quite ugly. @todo: Figure out the 676 // proper place to really handle request deletes. 677 delete req; 678 if (TheISA::HasUnalignedMemAcc && sreqLow) { 679 delete sreqLow; 680 delete sreqHigh; 681 } 682 683 return NoFault; 684 } 685 } 686 687 // If there's no forwarding case, then go access memory 688 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 689 load_inst->seqNum, load_inst->pcState()); 690 691 assert(!load_inst->memData); 692 load_inst->memData = new uint8_t[64]; 693 694 ++usedPorts; 695 696 // if we the cache is not blocked, do cache access 697 bool completedFirst = false; 698 if (!lsq->cacheBlocked()) { 699 MemCmd command = 700 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 701 PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 702 PacketPtr fst_data_pkt = NULL; 703 PacketPtr snd_data_pkt = NULL; 704 705 data_pkt->dataStatic(load_inst->memData); 706 707 LSQSenderState *state = new LSQSenderState; 708 state->isLoad = true; 709 state->idx = load_idx; 710 state->inst = load_inst; 711 data_pkt->senderState = state; 712 713 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 714 715 // Point the first packet at the main data packet. 716 fst_data_pkt = data_pkt; 717 } else { 718 719 // Create the split packets. 720 fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 721 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 722 723 fst_data_pkt->dataStatic(load_inst->memData); 724 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 725 726 fst_data_pkt->senderState = state; 727 snd_data_pkt->senderState = state; 728 729 state->isSplit = true; 730 state->outstanding = 2; 731 state->mainPkt = data_pkt; 732 } 733 734 if (!dcachePort->sendTiming(fst_data_pkt)) { 735 // Delete state and data packet because a load retry 736 // initiates a pipeline restart; it does not retry. 737 delete state; 738 delete data_pkt->req; 739 delete data_pkt; 740 if (TheISA::HasUnalignedMemAcc && sreqLow) { 741 delete fst_data_pkt->req; 742 delete fst_data_pkt; 743 delete snd_data_pkt->req; 744 delete snd_data_pkt; 745 sreqLow = NULL; 746 sreqHigh = NULL; 747 } 748 749 req = NULL; 750 751 // If the access didn't succeed, tell the LSQ by setting 752 // the retry thread id. 753 lsq->setRetryTid(lsqID); 754 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 755 completedFirst = true; 756 757 // The first packet was sent without problems, so send this one 758 // too. If there is a problem with this packet then the whole 759 // load will be squashed, so indicate this to the state object. 760 // The first packet will return in completeDataAccess and be 761 // handled there. 762 ++usedPorts; 763 if (!dcachePort->sendTiming(snd_data_pkt)) { 764 765 // The main packet will be deleted in completeDataAccess. 766 delete snd_data_pkt->req; 767 delete snd_data_pkt; 768 769 state->complete(); 770 771 req = NULL; 772 sreqHigh = NULL; 773 774 lsq->setRetryTid(lsqID); 775 } 776 } 777 } 778 779 // If the cache was blocked, or has become blocked due to the access, 780 // handle it. 781 if (lsq->cacheBlocked()) { 782 if (req) 783 delete req; 784 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 785 delete sreqLow; 786 delete sreqHigh; 787 } 788 789 ++lsqCacheBlocked; 790 791 iewStage->decrWb(load_inst->seqNum); 792 // There's an older load that's already going to squash. 793 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 794 return NoFault; 795 796 // Record that the load was blocked due to memory. This 797 // load will squash all instructions after it, be 798 // refetched, and re-executed. 799 isLoadBlocked = true; 800 loadBlockedHandled = false; 801 blockedLoadSeqNum = load_inst->seqNum; 802 // No fault occurred, even though the interface is blocked. 803 return NoFault; 804 } 805 806 return NoFault; 807} 808 809template <class Impl> 810Fault 811LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 812 uint8_t *data, int store_idx) 813{ 814 assert(storeQueue[store_idx].inst); 815 816 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 817 " | storeHead:%i [sn:%i]\n", 818 store_idx, req->getPaddr(), data, storeHead, 819 storeQueue[store_idx].inst->seqNum); 820 821 storeQueue[store_idx].req = req; 822 storeQueue[store_idx].sreqLow = sreqLow; 823 storeQueue[store_idx].sreqHigh = sreqHigh; 824 unsigned size = req->getSize(); 825 storeQueue[store_idx].size = size; 826 assert(size <= sizeof(storeQueue[store_idx].data)); 827 828 // Split stores can only occur in ISAs with unaligned memory accesses. If 829 // a store request has been split, sreqLow and sreqHigh will be non-null. 830 if (TheISA::HasUnalignedMemAcc && sreqLow) { 831 storeQueue[store_idx].isSplit = true; 832 } 833 834 memcpy(storeQueue[store_idx].data, data, size); 835 836 // This function only writes the data to the store queue, so no fault 837 // can happen here. 838 return NoFault; 839} 840 841#endif // __CPU_O3_LSQ_UNIT_HH__
| 383 /** The number of load instructions in the LQ. */ 384 int loads; 385 /** The number of store instructions in the SQ. */ 386 int stores; 387 /** The number of store instructions in the SQ waiting to writeback. */ 388 int storesToWB; 389 390 /** The index of the head instruction in the LQ. */ 391 int loadHead; 392 /** The index of the tail instruction in the LQ. */ 393 int loadTail; 394 395 /** The index of the head instruction in the SQ. */ 396 int storeHead; 397 /** The index of the first instruction that may be ready to be 398 * written back, and has not yet been written back. 399 */ 400 int storeWBIdx; 401 /** The index of the tail instruction in the SQ. */ 402 int storeTail; 403 404 /// @todo Consider moving to a more advanced model with write vs read ports 405 /** The number of cache ports available each cycle. */ 406 int cachePorts; 407 408 /** The number of used cache ports in this cycle. */ 409 int usedPorts; 410 411 /** Is the LSQ switched out. */ 412 bool switchedOut; 413 414 //list<InstSeqNum> mshrSeqNums; 415 416 /** Wire to read information from the issue stage time queue. */ 417 typename TimeBuffer<IssueStruct>::wire fromIssue; 418 419 /** Whether or not the LSQ is stalled. */ 420 bool stalled; 421 /** The store that causes the stall due to partial store to load 422 * forwarding. 423 */ 424 InstSeqNum stallingStoreIsn; 425 /** The index of the above store. */ 426 int stallingLoadIdx; 427 428 /** The packet that needs to be retried. */ 429 PacketPtr retryPkt; 430 431 /** Whehter or not a store is blocked due to the memory system. */ 432 bool isStoreBlocked; 433 434 /** Whether or not a load is blocked due to the memory system. */ 435 bool isLoadBlocked; 436 437 /** Has the blocked load been handled. */ 438 bool loadBlockedHandled; 439 440 /** The sequence number of the blocked load. */ 441 InstSeqNum blockedLoadSeqNum; 442 443 /** The oldest load that caused a memory ordering violation. */ 444 DynInstPtr memDepViolator; 445 446 /** Whether or not there is a packet that couldn't be sent because of 447 * a lack of cache ports. */ 448 bool hasPendingPkt; 449 450 /** The packet that is pending free cache ports. */ 451 PacketPtr pendingPkt; 452 453 // Will also need how many read/write ports the Dcache has. Or keep track 454 // of that in stage that is one level up, and only call executeLoad/Store 455 // the appropriate number of times. 456 /** Total number of loads forwaded from LSQ stores. */ 457 Stats::Scalar lsqForwLoads; 458 459 /** Total number of loads ignored due to invalid addresses. */ 460 Stats::Scalar invAddrLoads; 461 462 /** Total number of squashed loads. */ 463 Stats::Scalar lsqSquashedLoads; 464 465 /** Total number of responses from the memory system that are 466 * ignored due to the instruction already being squashed. */ 467 Stats::Scalar lsqIgnoredResponses; 468 469 /** Tota number of memory ordering violations. */ 470 Stats::Scalar lsqMemOrderViolation; 471 472 /** Total number of squashed stores. */ 473 Stats::Scalar lsqSquashedStores; 474 475 /** Total number of software prefetches ignored due to invalid addresses. */ 476 Stats::Scalar invAddrSwpfs; 477 478 /** Ready loads blocked due to partial store-forwarding. */ 479 Stats::Scalar lsqBlockedLoads; 480 481 /** Number of loads that were rescheduled. */ 482 Stats::Scalar lsqRescheduledLoads; 483 484 /** Number of times the LSQ is blocked due to the cache. */ 485 Stats::Scalar lsqCacheBlocked; 486 487 public: 488 /** Executes the load at the given index. */ 489 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 490 uint8_t *data, int load_idx); 491 492 /** Executes the store at the given index. */ 493 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 494 uint8_t *data, int store_idx); 495 496 /** Returns the index of the head load instruction. */ 497 int getLoadHead() { return loadHead; } 498 /** Returns the sequence number of the head load instruction. */ 499 InstSeqNum getLoadHeadSeqNum() 500 { 501 if (loadQueue[loadHead]) { 502 return loadQueue[loadHead]->seqNum; 503 } else { 504 return 0; 505 } 506 507 } 508 509 /** Returns the index of the head store instruction. */ 510 int getStoreHead() { return storeHead; } 511 /** Returns the sequence number of the head store instruction. */ 512 InstSeqNum getStoreHeadSeqNum() 513 { 514 if (storeQueue[storeHead].inst) { 515 return storeQueue[storeHead].inst->seqNum; 516 } else { 517 return 0; 518 } 519 520 } 521 522 /** Returns whether or not the LSQ unit is stalled. */ 523 bool isStalled() { return stalled; } 524}; 525 526template <class Impl> 527Fault 528LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 529 uint8_t *data, int load_idx) 530{ 531 DynInstPtr load_inst = loadQueue[load_idx]; 532 533 assert(load_inst); 534 535 assert(!load_inst->isExecuted()); 536 537 // Make sure this isn't an uncacheable access 538 // A bit of a hackish way to get uncached accesses to work only if they're 539 // at the head of the LSQ and are ready to commit (at the head of the ROB 540 // too). 541 if (req->isUncacheable() && 542 (load_idx != loadHead || !load_inst->isAtCommit())) { 543 iewStage->rescheduleMemInst(load_inst); 544 ++lsqRescheduledLoads; 545 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n", 546 load_inst->seqNum, load_inst->pcState()); 547 548 // Must delete request now that it wasn't handed off to 549 // memory. This is quite ugly. @todo: Figure out the proper 550 // place to really handle request deletes. 551 delete req; 552 if (TheISA::HasUnalignedMemAcc && sreqLow) { 553 delete sreqLow; 554 delete sreqHigh; 555 } 556 return TheISA::genMachineCheckFault(); 557 } 558 559 // Check the SQ for any previous stores that might lead to forwarding 560 int store_idx = load_inst->sqIdx; 561 562 int store_size = 0; 563 564 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 565 "storeHead: %i addr: %#x%s\n", 566 load_idx, store_idx, storeHead, req->getPaddr(), 567 sreqLow ? " split" : ""); 568 569 if (req->isLLSC()) { 570 assert(!sreqLow); 571 // Disable recording the result temporarily. Writing to misc 572 // regs normally updates the result, but this is not the 573 // desired behavior when handling store conditionals. 574 load_inst->recordResult = false; 575 TheISA::handleLockedRead(load_inst.get(), req); 576 load_inst->recordResult = true; 577 } 578 579 while (store_idx != -1) { 580 // End once we've reached the top of the LSQ 581 if (store_idx == storeWBIdx) { 582 break; 583 } 584 585 // Move the index to one younger 586 if (--store_idx < 0) 587 store_idx += SQEntries; 588 589 assert(storeQueue[store_idx].inst); 590 591 store_size = storeQueue[store_idx].size; 592 593 if (store_size == 0) 594 continue; 595 else if (storeQueue[store_idx].inst->uncacheable()) 596 continue; 597 598 assert(storeQueue[store_idx].inst->effAddrValid); 599 600 // Check if the store data is within the lower and upper bounds of 601 // addresses that the request needs. 602 bool store_has_lower_limit = 603 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 604 bool store_has_upper_limit = 605 (req->getVaddr() + req->getSize()) <= 606 (storeQueue[store_idx].inst->effAddr + store_size); 607 bool lower_load_has_store_part = 608 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 609 store_size); 610 bool upper_load_has_store_part = 611 (req->getVaddr() + req->getSize()) > 612 storeQueue[store_idx].inst->effAddr; 613 614 // If the store's data has all of the data needed, we can forward. 615 if ((store_has_lower_limit && store_has_upper_limit)) { 616 // Get shift amount for offset into the store's data. 617 int shift_amt = req->getVaddr() & (store_size - 1); 618 619 memcpy(data, storeQueue[store_idx].data + shift_amt, 620 req->getSize()); 621 622 assert(!load_inst->memData); 623 load_inst->memData = new uint8_t[64]; 624 625 memcpy(load_inst->memData, 626 storeQueue[store_idx].data + shift_amt, req->getSize()); 627 628 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 629 "addr %#x, data %#x\n", 630 store_idx, req->getVaddr(), data); 631 632 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 633 Packet::Broadcast); 634 data_pkt->dataStatic(load_inst->memData); 635 636 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 637 638 // We'll say this has a 1 cycle load-store forwarding latency 639 // for now. 640 // @todo: Need to make this a parameter. 641 cpu->schedule(wb, curTick()); 642 643 // Don't need to do anything special for split loads. 644 if (TheISA::HasUnalignedMemAcc && sreqLow) { 645 delete sreqLow; 646 delete sreqHigh; 647 } 648 649 ++lsqForwLoads; 650 return NoFault; 651 } else if ((store_has_lower_limit && lower_load_has_store_part) || 652 (store_has_upper_limit && upper_load_has_store_part) || 653 (lower_load_has_store_part && upper_load_has_store_part)) { 654 // This is the partial store-load forwarding case where a store 655 // has only part of the load's data. 656 657 // If it's already been written back, then don't worry about 658 // stalling on it. 659 if (storeQueue[store_idx].completed) { 660 panic("Should not check one of these"); 661 continue; 662 } 663 664 // Must stall load and force it to retry, so long as it's the oldest 665 // load that needs to do so. 666 if (!stalled || 667 (stalled && 668 load_inst->seqNum < 669 loadQueue[stallingLoadIdx]->seqNum)) { 670 stalled = true; 671 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 672 stallingLoadIdx = load_idx; 673 } 674 675 // Tell IQ/mem dep unit that this instruction will need to be 676 // rescheduled eventually 677 iewStage->rescheduleMemInst(load_inst); 678 iewStage->decrWb(load_inst->seqNum); 679 load_inst->clearIssued(); 680 ++lsqRescheduledLoads; 681 682 // Do not generate a writeback event as this instruction is not 683 // complete. 684 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 685 "Store idx %i to load addr %#x\n", 686 store_idx, req->getVaddr()); 687 688 // Must delete request now that it wasn't handed off to 689 // memory. This is quite ugly. @todo: Figure out the 690 // proper place to really handle request deletes. 691 delete req; 692 if (TheISA::HasUnalignedMemAcc && sreqLow) { 693 delete sreqLow; 694 delete sreqHigh; 695 } 696 697 return NoFault; 698 } 699 } 700 701 // If there's no forwarding case, then go access memory 702 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 703 load_inst->seqNum, load_inst->pcState()); 704 705 assert(!load_inst->memData); 706 load_inst->memData = new uint8_t[64]; 707 708 ++usedPorts; 709 710 // if we the cache is not blocked, do cache access 711 bool completedFirst = false; 712 if (!lsq->cacheBlocked()) { 713 MemCmd command = 714 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 715 PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 716 PacketPtr fst_data_pkt = NULL; 717 PacketPtr snd_data_pkt = NULL; 718 719 data_pkt->dataStatic(load_inst->memData); 720 721 LSQSenderState *state = new LSQSenderState; 722 state->isLoad = true; 723 state->idx = load_idx; 724 state->inst = load_inst; 725 data_pkt->senderState = state; 726 727 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 728 729 // Point the first packet at the main data packet. 730 fst_data_pkt = data_pkt; 731 } else { 732 733 // Create the split packets. 734 fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 735 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 736 737 fst_data_pkt->dataStatic(load_inst->memData); 738 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 739 740 fst_data_pkt->senderState = state; 741 snd_data_pkt->senderState = state; 742 743 state->isSplit = true; 744 state->outstanding = 2; 745 state->mainPkt = data_pkt; 746 } 747 748 if (!dcachePort->sendTiming(fst_data_pkt)) { 749 // Delete state and data packet because a load retry 750 // initiates a pipeline restart; it does not retry. 751 delete state; 752 delete data_pkt->req; 753 delete data_pkt; 754 if (TheISA::HasUnalignedMemAcc && sreqLow) { 755 delete fst_data_pkt->req; 756 delete fst_data_pkt; 757 delete snd_data_pkt->req; 758 delete snd_data_pkt; 759 sreqLow = NULL; 760 sreqHigh = NULL; 761 } 762 763 req = NULL; 764 765 // If the access didn't succeed, tell the LSQ by setting 766 // the retry thread id. 767 lsq->setRetryTid(lsqID); 768 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 769 completedFirst = true; 770 771 // The first packet was sent without problems, so send this one 772 // too. If there is a problem with this packet then the whole 773 // load will be squashed, so indicate this to the state object. 774 // The first packet will return in completeDataAccess and be 775 // handled there. 776 ++usedPorts; 777 if (!dcachePort->sendTiming(snd_data_pkt)) { 778 779 // The main packet will be deleted in completeDataAccess. 780 delete snd_data_pkt->req; 781 delete snd_data_pkt; 782 783 state->complete(); 784 785 req = NULL; 786 sreqHigh = NULL; 787 788 lsq->setRetryTid(lsqID); 789 } 790 } 791 } 792 793 // If the cache was blocked, or has become blocked due to the access, 794 // handle it. 795 if (lsq->cacheBlocked()) { 796 if (req) 797 delete req; 798 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 799 delete sreqLow; 800 delete sreqHigh; 801 } 802 803 ++lsqCacheBlocked; 804 805 iewStage->decrWb(load_inst->seqNum); 806 // There's an older load that's already going to squash. 807 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 808 return NoFault; 809 810 // Record that the load was blocked due to memory. This 811 // load will squash all instructions after it, be 812 // refetched, and re-executed. 813 isLoadBlocked = true; 814 loadBlockedHandled = false; 815 blockedLoadSeqNum = load_inst->seqNum; 816 // No fault occurred, even though the interface is blocked. 817 return NoFault; 818 } 819 820 return NoFault; 821} 822 823template <class Impl> 824Fault 825LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 826 uint8_t *data, int store_idx) 827{ 828 assert(storeQueue[store_idx].inst); 829 830 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 831 " | storeHead:%i [sn:%i]\n", 832 store_idx, req->getPaddr(), data, storeHead, 833 storeQueue[store_idx].inst->seqNum); 834 835 storeQueue[store_idx].req = req; 836 storeQueue[store_idx].sreqLow = sreqLow; 837 storeQueue[store_idx].sreqHigh = sreqHigh; 838 unsigned size = req->getSize(); 839 storeQueue[store_idx].size = size; 840 assert(size <= sizeof(storeQueue[store_idx].data)); 841 842 // Split stores can only occur in ISAs with unaligned memory accesses. If 843 // a store request has been split, sreqLow and sreqHigh will be non-null. 844 if (TheISA::HasUnalignedMemAcc && sreqLow) { 845 storeQueue[store_idx].isSplit = true; 846 } 847 848 memcpy(storeQueue[store_idx].data, data, size); 849 850 // This function only writes the data to the store queue, so no fault 851 // can happen here. 852 return NoFault; 853} 854 855#endif // __CPU_O3_LSQ_UNIT_HH__
|