lsq_unit.hh revision 3876:127c71cfe21a
110695SAli.Saidi@ARM.com/* 210695SAli.Saidi@ARM.com * Copyright (c) 2004-2006 The Regents of The University of Michigan 310695SAli.Saidi@ARM.com * All rights reserved. 410695SAli.Saidi@ARM.com * 510695SAli.Saidi@ARM.com * Redistribution and use in source and binary forms, with or without 610695SAli.Saidi@ARM.com * modification, are permitted provided that the following conditions are 710695SAli.Saidi@ARM.com * met: redistributions of source code must retain the above copyright 810695SAli.Saidi@ARM.com * notice, this list of conditions and the following disclaimer; 910695SAli.Saidi@ARM.com * redistributions in binary form must reproduce the above copyright 1010695SAli.Saidi@ARM.com * notice, this list of conditions and the following disclaimer in the 1110695SAli.Saidi@ARM.com * documentation and/or other materials provided with the distribution; 1210695SAli.Saidi@ARM.com * neither the name of the copyright holders nor the names of its 1310695SAli.Saidi@ARM.com * contributors may be used to endorse or promote products derived from 1410695SAli.Saidi@ARM.com * this software without specific prior written permission. 1510695SAli.Saidi@ARM.com * 1610695SAli.Saidi@ARM.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1710695SAli.Saidi@ARM.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1810695SAli.Saidi@ARM.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1910695SAli.Saidi@ARM.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2010695SAli.Saidi@ARM.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2110695SAli.Saidi@ARM.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2210695SAli.Saidi@ARM.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2310695SAli.Saidi@ARM.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2410695SAli.Saidi@ARM.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2510695SAli.Saidi@ARM.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2610695SAli.Saidi@ARM.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2710695SAli.Saidi@ARM.com * 2810695SAli.Saidi@ARM.com * Authors: Kevin Lim 2910695SAli.Saidi@ARM.com * Korey Sewell 3010695SAli.Saidi@ARM.com */ 3110695SAli.Saidi@ARM.com 3210695SAli.Saidi@ARM.com#ifndef __CPU_O3_LSQ_UNIT_HH__ 3310695SAli.Saidi@ARM.com#define __CPU_O3_LSQ_UNIT_HH__ 3410695SAli.Saidi@ARM.com 3510695SAli.Saidi@ARM.com#include <algorithm> 3610695SAli.Saidi@ARM.com#include <map> 3710695SAli.Saidi@ARM.com#include <queue> 3810695SAli.Saidi@ARM.com 3910695SAli.Saidi@ARM.com#include "arch/faults.hh" 4010695SAli.Saidi@ARM.com#include "arch/locked_mem.hh" 4110695SAli.Saidi@ARM.com#include "config/full_system.hh" 4210695SAli.Saidi@ARM.com#include "base/hashmap.hh" 4310695SAli.Saidi@ARM.com#include "cpu/inst_seq.hh" 4410695SAli.Saidi@ARM.com#include "mem/packet.hh" 4510695SAli.Saidi@ARM.com#include "mem/port.hh" 4610695SAli.Saidi@ARM.com 4710695SAli.Saidi@ARM.com/** 4810695SAli.Saidi@ARM.com * Class that implements the actual LQ and SQ for each specific 4910695SAli.Saidi@ARM.com * thread. Both are circular queues; load entries are freed upon 5010695SAli.Saidi@ARM.com * committing, while store entries are freed once they writeback. The 5110695SAli.Saidi@ARM.com * LSQUnit tracks if there are memory ordering violations, and also 5210695SAli.Saidi@ARM.com * detects partial load to store forwarding cases (a store only has 5310695SAli.Saidi@ARM.com * part of a load's data) that requires the load to wait until the 5410695SAli.Saidi@ARM.com * store writes back. In the former case it holds onto the instruction 5510695SAli.Saidi@ARM.com * until the dependence unit looks at it, and in the latter it stalls 5610695SAli.Saidi@ARM.com * the LSQ until the store writes back. At that point the load is 5710695SAli.Saidi@ARM.com * replayed. 5810695SAli.Saidi@ARM.com */ 5910695SAli.Saidi@ARM.comtemplate <class Impl> 6010695SAli.Saidi@ARM.comclass LSQUnit { 6110695SAli.Saidi@ARM.com protected: 6210695SAli.Saidi@ARM.com typedef TheISA::IntReg IntReg; 6310695SAli.Saidi@ARM.com public: 6410695SAli.Saidi@ARM.com typedef typename Impl::Params Params; 6510695SAli.Saidi@ARM.com typedef typename Impl::O3CPU O3CPU; 6610695SAli.Saidi@ARM.com typedef typename Impl::DynInstPtr DynInstPtr; 6710695SAli.Saidi@ARM.com typedef typename Impl::CPUPol::IEW IEW; 6810695SAli.Saidi@ARM.com typedef typename Impl::CPUPol::LSQ LSQ; 6910695SAli.Saidi@ARM.com typedef typename Impl::CPUPol::IssueStruct IssueStruct; 7010695SAli.Saidi@ARM.com 7110695SAli.Saidi@ARM.com public: 7210695SAli.Saidi@ARM.com /** Constructs an LSQ unit. init() must be called prior to use. */ 7310695SAli.Saidi@ARM.com LSQUnit(); 7410695SAli.Saidi@ARM.com 7510695SAli.Saidi@ARM.com /** Initializes the LSQ unit with the specified number of entries. */ 7610695SAli.Saidi@ARM.com void init(Params *params, LSQ *lsq_ptr, unsigned maxLQEntries, 7710695SAli.Saidi@ARM.com unsigned maxSQEntries, unsigned id); 7810695SAli.Saidi@ARM.com 7910695SAli.Saidi@ARM.com /** Returns the name of the LSQ unit. */ 8010695SAli.Saidi@ARM.com std::string name() const; 8110695SAli.Saidi@ARM.com 8210695SAli.Saidi@ARM.com /** Registers statistics. */ 8310695SAli.Saidi@ARM.com void regStats(); 8410695SAli.Saidi@ARM.com 8510695SAli.Saidi@ARM.com /** Sets the CPU pointer. */ 8610695SAli.Saidi@ARM.com void setCPU(O3CPU *cpu_ptr); 8710695SAli.Saidi@ARM.com 8810695SAli.Saidi@ARM.com /** Sets the IEW stage pointer. */ 8910695SAli.Saidi@ARM.com void setIEW(IEW *iew_ptr) 9010695SAli.Saidi@ARM.com { iewStage = iew_ptr; } 9110695SAli.Saidi@ARM.com 9210695SAli.Saidi@ARM.com /** Sets the pointer to the dcache port. */ 9310695SAli.Saidi@ARM.com void setDcachePort(Port *dcache_port) 9410695SAli.Saidi@ARM.com { dcachePort = dcache_port; } 9510695SAli.Saidi@ARM.com 9610695SAli.Saidi@ARM.com /** Switches out LSQ unit. */ 9710695SAli.Saidi@ARM.com void switchOut(); 9810695SAli.Saidi@ARM.com 9910695SAli.Saidi@ARM.com /** Takes over from another CPU's thread. */ 10010695SAli.Saidi@ARM.com void takeOverFrom(); 10110695SAli.Saidi@ARM.com 10210695SAli.Saidi@ARM.com /** Returns if the LSQ is switched out. */ 10310695SAli.Saidi@ARM.com bool isSwitchedOut() { return switchedOut; } 10410695SAli.Saidi@ARM.com 10510695SAli.Saidi@ARM.com /** Ticks the LSQ unit, which in this case only resets the number of 10610695SAli.Saidi@ARM.com * used cache ports. 10710695SAli.Saidi@ARM.com * @todo: Move the number of used ports up to the LSQ level so it can 10810695SAli.Saidi@ARM.com * be shared by all LSQ units. 10910695SAli.Saidi@ARM.com */ 11010695SAli.Saidi@ARM.com void tick() { usedPorts = 0; } 11110695SAli.Saidi@ARM.com 11210695SAli.Saidi@ARM.com /** Inserts an instruction. */ 11310695SAli.Saidi@ARM.com void insert(DynInstPtr &inst); 11410695SAli.Saidi@ARM.com /** Inserts a load instruction. */ 11510695SAli.Saidi@ARM.com void insertLoad(DynInstPtr &load_inst); 11610695SAli.Saidi@ARM.com /** Inserts a store instruction. */ 11710695SAli.Saidi@ARM.com void insertStore(DynInstPtr &store_inst); 11810695SAli.Saidi@ARM.com 11910695SAli.Saidi@ARM.com /** Executes a load instruction. */ 12010695SAli.Saidi@ARM.com Fault executeLoad(DynInstPtr &inst); 12110695SAli.Saidi@ARM.com 12210695SAli.Saidi@ARM.com Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 12310695SAli.Saidi@ARM.com /** Executes a store instruction. */ 12410695SAli.Saidi@ARM.com Fault executeStore(DynInstPtr &inst); 12510695SAli.Saidi@ARM.com 12610695SAli.Saidi@ARM.com /** Commits the head load. */ 12710695SAli.Saidi@ARM.com void commitLoad(); 12810695SAli.Saidi@ARM.com /** Commits loads older than a specific sequence number. */ 12910695SAli.Saidi@ARM.com void commitLoads(InstSeqNum &youngest_inst); 13010695SAli.Saidi@ARM.com 13110695SAli.Saidi@ARM.com /** Commits stores older than a specific sequence number. */ 13210695SAli.Saidi@ARM.com void commitStores(InstSeqNum &youngest_inst); 13310695SAli.Saidi@ARM.com 13410695SAli.Saidi@ARM.com /** Writes back stores. */ 13510695SAli.Saidi@ARM.com void writebackStores(); 136 137 /** Completes the data access that has been returned from the 138 * memory system. */ 139 void completeDataAccess(PacketPtr pkt); 140 141 /** Clears all the entries in the LQ. */ 142 void clearLQ(); 143 144 /** Clears all the entries in the SQ. */ 145 void clearSQ(); 146 147 /** Resizes the LQ to a given size. */ 148 void resizeLQ(unsigned size); 149 150 /** Resizes the SQ to a given size. */ 151 void resizeSQ(unsigned size); 152 153 /** Squashes all instructions younger than a specific sequence number. */ 154 void squash(const InstSeqNum &squashed_num); 155 156 /** Returns if there is a memory ordering violation. Value is reset upon 157 * call to getMemDepViolator(). 158 */ 159 bool violation() { return memDepViolator; } 160 161 /** Returns the memory ordering violator. */ 162 DynInstPtr getMemDepViolator(); 163 164 /** Returns if a load became blocked due to the memory system. */ 165 bool loadBlocked() 166 { return isLoadBlocked; } 167 168 /** Clears the signal that a load became blocked. */ 169 void clearLoadBlocked() 170 { isLoadBlocked = false; } 171 172 /** Returns if the blocked load was handled. */ 173 bool isLoadBlockedHandled() 174 { return loadBlockedHandled; } 175 176 /** Records the blocked load as being handled. */ 177 void setLoadBlockedHandled() 178 { loadBlockedHandled = true; } 179 180 /** Returns the number of free entries (min of free LQ and SQ entries). */ 181 unsigned numFreeEntries(); 182 183 /** Returns the number of loads ready to execute. */ 184 int numLoadsReady(); 185 186 /** Returns the number of loads in the LQ. */ 187 int numLoads() { return loads; } 188 189 /** Returns the number of stores in the SQ. */ 190 int numStores() { return stores; } 191 192 /** Returns if either the LQ or SQ is full. */ 193 bool isFull() { return lqFull() || sqFull(); } 194 195 /** Returns if the LQ is full. */ 196 bool lqFull() { return loads >= (LQEntries - 1); } 197 198 /** Returns if the SQ is full. */ 199 bool sqFull() { return stores >= (SQEntries - 1); } 200 201 /** Returns the number of instructions in the LSQ. */ 202 unsigned getCount() { return loads + stores; } 203 204 /** Returns if there are any stores to writeback. */ 205 bool hasStoresToWB() { return storesToWB; } 206 207 /** Returns the number of stores to writeback. */ 208 int numStoresToWB() { return storesToWB; } 209 210 /** Returns if the LSQ unit will writeback on this cycle. */ 211 bool willWB() { return storeQueue[storeWBIdx].canWB && 212 !storeQueue[storeWBIdx].completed && 213 !isStoreBlocked; } 214 215 /** Handles doing the retry. */ 216 void recvRetry(); 217 218 private: 219 /** Writes back the instruction, sending it to IEW. */ 220 void writeback(DynInstPtr &inst, PacketPtr pkt); 221 222 /** Handles completing the send of a store to memory. */ 223 void storePostSend(PacketPtr pkt); 224 225 /** Completes the store at the specified index. */ 226 void completeStore(int store_idx); 227 228 /** Increments the given store index (circular queue). */ 229 inline void incrStIdx(int &store_idx); 230 /** Decrements the given store index (circular queue). */ 231 inline void decrStIdx(int &store_idx); 232 /** Increments the given load index (circular queue). */ 233 inline void incrLdIdx(int &load_idx); 234 /** Decrements the given load index (circular queue). */ 235 inline void decrLdIdx(int &load_idx); 236 237 public: 238 /** Debugging function to dump instructions in the LSQ. */ 239 void dumpInsts(); 240 241 private: 242 /** Pointer to the CPU. */ 243 O3CPU *cpu; 244 245 /** Pointer to the IEW stage. */ 246 IEW *iewStage; 247 248 /** Pointer to the LSQ. */ 249 LSQ *lsq; 250 251 /** Pointer to the dcache port. Used only for sending. */ 252 Port *dcachePort; 253 254 /** Derived class to hold any sender state the LSQ needs. */ 255 class LSQSenderState : public Packet::SenderState 256 { 257 public: 258 /** Default constructor. */ 259 LSQSenderState() 260 : noWB(false) 261 { } 262 263 /** Instruction who initiated the access to memory. */ 264 DynInstPtr inst; 265 /** Whether or not it is a load. */ 266 bool isLoad; 267 /** The LQ/SQ index of the instruction. */ 268 int idx; 269 /** Whether or not the instruction will need to writeback. */ 270 bool noWB; 271 }; 272 273 /** Writeback event, specifically for when stores forward data to loads. */ 274 class WritebackEvent : public Event { 275 public: 276 /** Constructs a writeback event. */ 277 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 278 279 /** Processes the writeback event. */ 280 void process(); 281 282 /** Returns the description of this event. */ 283 const char *description(); 284 285 private: 286 /** Instruction whose results are being written back. */ 287 DynInstPtr inst; 288 289 /** The packet that would have been sent to memory. */ 290 PacketPtr pkt; 291 292 /** The pointer to the LSQ unit that issued the store. */ 293 LSQUnit<Impl> *lsqPtr; 294 }; 295 296 public: 297 struct SQEntry { 298 /** Constructs an empty store queue entry. */ 299 SQEntry() 300 : inst(NULL), req(NULL), size(0), data(0), 301 canWB(0), committed(0), completed(0) 302 { } 303 304 /** Constructs a store queue entry for a given instruction. */ 305 SQEntry(DynInstPtr &_inst) 306 : inst(_inst), req(NULL), size(0), data(0), 307 canWB(0), committed(0), completed(0) 308 { } 309 310 /** The store instruction. */ 311 DynInstPtr inst; 312 /** The request for the store. */ 313 RequestPtr req; 314 /** The size of the store. */ 315 int size; 316 /** The store data. */ 317 IntReg data; 318 /** Whether or not the store can writeback. */ 319 bool canWB; 320 /** Whether or not the store is committed. */ 321 bool committed; 322 /** Whether or not the store is completed. */ 323 bool completed; 324 }; 325 326 private: 327 /** The LSQUnit thread id. */ 328 unsigned lsqID; 329 330 /** The store queue. */ 331 std::vector<SQEntry> storeQueue; 332 333 /** The load queue. */ 334 std::vector<DynInstPtr> loadQueue; 335 336 /** The number of LQ entries, plus a sentinel entry (circular queue). 337 * @todo: Consider having var that records the true number of LQ entries. 338 */ 339 unsigned LQEntries; 340 /** The number of SQ entries, plus a sentinel entry (circular queue). 341 * @todo: Consider having var that records the true number of SQ entries. 342 */ 343 unsigned SQEntries; 344 345 /** The number of load instructions in the LQ. */ 346 int loads; 347 /** The number of store instructions in the SQ. */ 348 int stores; 349 /** The number of store instructions in the SQ waiting to writeback. */ 350 int storesToWB; 351 352 /** The index of the head instruction in the LQ. */ 353 int loadHead; 354 /** The index of the tail instruction in the LQ. */ 355 int loadTail; 356 357 /** The index of the head instruction in the SQ. */ 358 int storeHead; 359 /** The index of the first instruction that may be ready to be 360 * written back, and has not yet been written back. 361 */ 362 int storeWBIdx; 363 /** The index of the tail instruction in the SQ. */ 364 int storeTail; 365 366 /// @todo Consider moving to a more advanced model with write vs read ports 367 /** The number of cache ports available each cycle. */ 368 int cachePorts; 369 370 /** The number of used cache ports in this cycle. */ 371 int usedPorts; 372 373 /** Is the LSQ switched out. */ 374 bool switchedOut; 375 376 //list<InstSeqNum> mshrSeqNums; 377 378 /** Wire to read information from the issue stage time queue. */ 379 typename TimeBuffer<IssueStruct>::wire fromIssue; 380 381 /** Whether or not the LSQ is stalled. */ 382 bool stalled; 383 /** The store that causes the stall due to partial store to load 384 * forwarding. 385 */ 386 InstSeqNum stallingStoreIsn; 387 /** The index of the above store. */ 388 int stallingLoadIdx; 389 390 /** The packet that needs to be retried. */ 391 PacketPtr retryPkt; 392 393 /** Whehter or not a store is blocked due to the memory system. */ 394 bool isStoreBlocked; 395 396 /** Whether or not a load is blocked due to the memory system. */ 397 bool isLoadBlocked; 398 399 /** Has the blocked load been handled. */ 400 bool loadBlockedHandled; 401 402 /** The sequence number of the blocked load. */ 403 InstSeqNum blockedLoadSeqNum; 404 405 /** The oldest load that caused a memory ordering violation. */ 406 DynInstPtr memDepViolator; 407 408 // Will also need how many read/write ports the Dcache has. Or keep track 409 // of that in stage that is one level up, and only call executeLoad/Store 410 // the appropriate number of times. 411 /** Total number of loads forwaded from LSQ stores. */ 412 Stats::Scalar<> lsqForwLoads; 413 414 /** Total number of loads ignored due to invalid addresses. */ 415 Stats::Scalar<> invAddrLoads; 416 417 /** Total number of squashed loads. */ 418 Stats::Scalar<> lsqSquashedLoads; 419 420 /** Total number of responses from the memory system that are 421 * ignored due to the instruction already being squashed. */ 422 Stats::Scalar<> lsqIgnoredResponses; 423 424 /** Tota number of memory ordering violations. */ 425 Stats::Scalar<> lsqMemOrderViolation; 426 427 /** Total number of squashed stores. */ 428 Stats::Scalar<> lsqSquashedStores; 429 430 /** Total number of software prefetches ignored due to invalid addresses. */ 431 Stats::Scalar<> invAddrSwpfs; 432 433 /** Ready loads blocked due to partial store-forwarding. */ 434 Stats::Scalar<> lsqBlockedLoads; 435 436 /** Number of loads that were rescheduled. */ 437 Stats::Scalar<> lsqRescheduledLoads; 438 439 /** Number of times the LSQ is blocked due to the cache. */ 440 Stats::Scalar<> lsqCacheBlocked; 441 442 public: 443 /** Executes the load at the given index. */ 444 template <class T> 445 Fault read(Request *req, T &data, int load_idx); 446 447 /** Executes the store at the given index. */ 448 template <class T> 449 Fault write(Request *req, T &data, int store_idx); 450 451 /** Returns the index of the head load instruction. */ 452 int getLoadHead() { return loadHead; } 453 /** Returns the sequence number of the head load instruction. */ 454 InstSeqNum getLoadHeadSeqNum() 455 { 456 if (loadQueue[loadHead]) { 457 return loadQueue[loadHead]->seqNum; 458 } else { 459 return 0; 460 } 461 462 } 463 464 /** Returns the index of the head store instruction. */ 465 int getStoreHead() { return storeHead; } 466 /** Returns the sequence number of the head store instruction. */ 467 InstSeqNum getStoreHeadSeqNum() 468 { 469 if (storeQueue[storeHead].inst) { 470 return storeQueue[storeHead].inst->seqNum; 471 } else { 472 return 0; 473 } 474 475 } 476 477 /** Returns whether or not the LSQ unit is stalled. */ 478 bool isStalled() { return stalled; } 479}; 480 481template <class Impl> 482template <class T> 483Fault 484LSQUnit<Impl>::read(Request *req, T &data, int load_idx) 485{ 486 DynInstPtr load_inst = loadQueue[load_idx]; 487 488 assert(load_inst); 489 490 assert(!load_inst->isExecuted()); 491 492 // Make sure this isn't an uncacheable access 493 // A bit of a hackish way to get uncached accesses to work only if they're 494 // at the head of the LSQ and are ready to commit (at the head of the ROB 495 // too). 496 if (req->isUncacheable() && 497 (load_idx != loadHead || !load_inst->isAtCommit())) { 498 iewStage->rescheduleMemInst(load_inst); 499 ++lsqRescheduledLoads; 500 return TheISA::genMachineCheckFault(); 501 } 502 503 // Check the SQ for any previous stores that might lead to forwarding 504 int store_idx = load_inst->sqIdx; 505 506 int store_size = 0; 507 508 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 509 "storeHead: %i addr: %#x\n", 510 load_idx, store_idx, storeHead, req->getPaddr()); 511 512 if (req->isLocked()) { 513 // Disable recording the result temporarily. Writing to misc 514 // regs normally updates the result, but this is not the 515 // desired behavior when handling store conditionals. 516 load_inst->recordResult = false; 517 TheISA::handleLockedRead(load_inst.get(), req); 518 load_inst->recordResult = true; 519 } 520 521 while (store_idx != -1) { 522 // End once we've reached the top of the LSQ 523 if (store_idx == storeWBIdx) { 524 break; 525 } 526 527 // Move the index to one younger 528 if (--store_idx < 0) 529 store_idx += SQEntries; 530 531 assert(storeQueue[store_idx].inst); 532 533 store_size = storeQueue[store_idx].size; 534 535 if (store_size == 0) 536 continue; 537 538 // Check if the store data is within the lower and upper bounds of 539 // addresses that the request needs. 540 bool store_has_lower_limit = 541 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 542 bool store_has_upper_limit = 543 (req->getVaddr() + req->getSize()) <= 544 (storeQueue[store_idx].inst->effAddr + store_size); 545 bool lower_load_has_store_part = 546 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 547 store_size); 548 bool upper_load_has_store_part = 549 (req->getVaddr() + req->getSize()) > 550 storeQueue[store_idx].inst->effAddr; 551 552 // If the store's data has all of the data needed, we can forward. 553 if (store_has_lower_limit && store_has_upper_limit) { 554 // Get shift amount for offset into the store's data. 555 int shift_amt = req->getVaddr() & (store_size - 1); 556 // @todo: Magic number, assumes byte addressing 557 shift_amt = shift_amt << 3; 558 559 // Cast this to type T? 560 data = storeQueue[store_idx].data >> shift_amt; 561 562 assert(!load_inst->memData); 563 load_inst->memData = new uint8_t[64]; 564 565 memcpy(load_inst->memData, &data, req->getSize()); 566 567 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 568 "addr %#x, data %#x\n", 569 store_idx, req->getVaddr(), data); 570 571 PacketPtr data_pkt = new Packet(req, Packet::ReadReq, Packet::Broadcast); 572 data_pkt->dataStatic(load_inst->memData); 573 574 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 575 576 // We'll say this has a 1 cycle load-store forwarding latency 577 // for now. 578 // @todo: Need to make this a parameter. 579 wb->schedule(curTick); 580 581 ++lsqForwLoads; 582 return NoFault; 583 } else if ((store_has_lower_limit && lower_load_has_store_part) || 584 (store_has_upper_limit && upper_load_has_store_part) || 585 (lower_load_has_store_part && upper_load_has_store_part)) { 586 // This is the partial store-load forwarding case where a store 587 // has only part of the load's data. 588 589 // If it's already been written back, then don't worry about 590 // stalling on it. 591 if (storeQueue[store_idx].completed) { 592 continue; 593 } 594 595 // Must stall load and force it to retry, so long as it's the oldest 596 // load that needs to do so. 597 if (!stalled || 598 (stalled && 599 load_inst->seqNum < 600 loadQueue[stallingLoadIdx]->seqNum)) { 601 stalled = true; 602 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 603 stallingLoadIdx = load_idx; 604 } 605 606 // Tell IQ/mem dep unit that this instruction will need to be 607 // rescheduled eventually 608 iewStage->rescheduleMemInst(load_inst); 609 iewStage->decrWb(load_inst->seqNum); 610 ++lsqRescheduledLoads; 611 612 // Do not generate a writeback event as this instruction is not 613 // complete. 614 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 615 "Store idx %i to load addr %#x\n", 616 store_idx, req->getVaddr()); 617 618 ++lsqBlockedLoads; 619 return NoFault; 620 } 621 } 622 623 // If there's no forwarding case, then go access memory 624 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %#x\n", 625 load_inst->seqNum, load_inst->readPC()); 626 627 assert(!load_inst->memData); 628 load_inst->memData = new uint8_t[64]; 629 630 ++usedPorts; 631 632 // if we the cache is not blocked, do cache access 633 if (!lsq->cacheBlocked()) { 634 PacketPtr data_pkt = 635 new Packet(req, Packet::ReadReq, Packet::Broadcast); 636 data_pkt->dataStatic(load_inst->memData); 637 638 LSQSenderState *state = new LSQSenderState; 639 state->isLoad = true; 640 state->idx = load_idx; 641 state->inst = load_inst; 642 data_pkt->senderState = state; 643 644 if (!dcachePort->sendTiming(data_pkt)) { 645 Packet::Result result = data_pkt->result; 646 647 // Delete state and data packet because a load retry 648 // initiates a pipeline restart; it does not retry. 649 delete state; 650 delete data_pkt; 651 652 if (result == Packet::BadAddress) { 653 return TheISA::genMachineCheckFault(); 654 } 655 656 // If the access didn't succeed, tell the LSQ by setting 657 // the retry thread id. 658 lsq->setRetryTid(lsqID); 659 } 660 } 661 662 // If the cache was blocked, or has become blocked due to the access, 663 // handle it. 664 if (lsq->cacheBlocked()) { 665 ++lsqCacheBlocked; 666 667 iewStage->decrWb(load_inst->seqNum); 668 // There's an older load that's already going to squash. 669 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 670 return NoFault; 671 672 // Record that the load was blocked due to memory. This 673 // load will squash all instructions after it, be 674 // refetched, and re-executed. 675 isLoadBlocked = true; 676 loadBlockedHandled = false; 677 blockedLoadSeqNum = load_inst->seqNum; 678 // No fault occurred, even though the interface is blocked. 679 return NoFault; 680 } 681 682 return NoFault; 683} 684 685template <class Impl> 686template <class T> 687Fault 688LSQUnit<Impl>::write(Request *req, T &data, int store_idx) 689{ 690 assert(storeQueue[store_idx].inst); 691 692 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 693 " | storeHead:%i [sn:%i]\n", 694 store_idx, req->getPaddr(), data, storeHead, 695 storeQueue[store_idx].inst->seqNum); 696 697 storeQueue[store_idx].req = req; 698 storeQueue[store_idx].size = sizeof(T); 699 storeQueue[store_idx].data = data; 700 701 // This function only writes the data to the store queue, so no fault 702 // can happen here. 703 return NoFault; 704} 705 706#endif // __CPU_O3_LSQ_UNIT_HH__ 707