47#include "mem/packet.hh" 48#include "mem/port.hh" 49 50class DerivO3CPUParams; 51 52/** 53 * Class that implements the actual LQ and SQ for each specific 54 * thread. Both are circular queues; load entries are freed upon 55 * committing, while store entries are freed once they writeback. The 56 * LSQUnit tracks if there are memory ordering violations, and also 57 * detects partial load to store forwarding cases (a store only has 58 * part of a load's data) that requires the load to wait until the 59 * store writes back. In the former case it holds onto the instruction 60 * until the dependence unit looks at it, and in the latter it stalls 61 * the LSQ until the store writes back. At that point the load is 62 * replayed. 63 */ 64template <class Impl> 65class LSQUnit { 66 public: 67 typedef typename Impl::O3CPU O3CPU; 68 typedef typename Impl::DynInstPtr DynInstPtr; 69 typedef typename Impl::CPUPol::IEW IEW; 70 typedef typename Impl::CPUPol::LSQ LSQ; 71 typedef typename Impl::CPUPol::IssueStruct IssueStruct; 72 73 public: 74 /** Constructs an LSQ unit. init() must be called prior to use. */ 75 LSQUnit(); 76 77 /** Initializes the LSQ unit with the specified number of entries. */ 78 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 79 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 80 unsigned id); 81 82 /** Returns the name of the LSQ unit. */ 83 std::string name() const; 84 85 /** Registers statistics. */ 86 void regStats(); 87 88 /** Sets the pointer to the dcache port. */ 89 void setDcachePort(Port *dcache_port); 90 91 /** Switches out LSQ unit. */ 92 void switchOut(); 93 94 /** Takes over from another CPU's thread. */ 95 void takeOverFrom(); 96 97 /** Returns if the LSQ is switched out. */ 98 bool isSwitchedOut() { return switchedOut; } 99 100 /** Ticks the LSQ unit, which in this case only resets the number of 101 * used cache ports. 102 * @todo: Move the number of used ports up to the LSQ level so it can 103 * be shared by all LSQ units. 104 */ 105 void tick() { usedPorts = 0; } 106 107 /** Inserts an instruction. */ 108 void insert(DynInstPtr &inst); 109 /** Inserts a load instruction. */ 110 void insertLoad(DynInstPtr &load_inst); 111 /** Inserts a store instruction. */ 112 void insertStore(DynInstPtr &store_inst); 113 114 /** Check for ordering violations in the LSQ 115 * @param load_idx index to start checking at 116 * @param inst the instruction to check 117 */ 118 Fault checkViolations(int load_idx, DynInstPtr &inst); 119 120 /** Executes a load instruction. */ 121 Fault executeLoad(DynInstPtr &inst); 122 123 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 124 /** Executes a store instruction. */ 125 Fault executeStore(DynInstPtr &inst); 126 127 /** Commits the head load. */ 128 void commitLoad(); 129 /** Commits loads older than a specific sequence number. */ 130 void commitLoads(InstSeqNum &youngest_inst); 131 132 /** Commits stores older than a specific sequence number. */ 133 void commitStores(InstSeqNum &youngest_inst); 134 135 /** Writes back stores. */ 136 void writebackStores(); 137 138 /** Completes the data access that has been returned from the 139 * memory system. */ 140 void completeDataAccess(PacketPtr pkt); 141 142 /** Clears all the entries in the LQ. */ 143 void clearLQ(); 144 145 /** Clears all the entries in the SQ. */ 146 void clearSQ(); 147 148 /** Resizes the LQ to a given size. */ 149 void resizeLQ(unsigned size); 150 151 /** Resizes the SQ to a given size. */ 152 void resizeSQ(unsigned size); 153 154 /** Squashes all instructions younger than a specific sequence number. */ 155 void squash(const InstSeqNum &squashed_num); 156 157 /** Returns if there is a memory ordering violation. Value is reset upon 158 * call to getMemDepViolator(). 159 */ 160 bool violation() { return memDepViolator; } 161 162 /** Returns the memory ordering violator. */ 163 DynInstPtr getMemDepViolator(); 164 165 /** Returns if a load became blocked due to the memory system. */ 166 bool loadBlocked() 167 { return isLoadBlocked; } 168 169 /** Clears the signal that a load became blocked. */ 170 void clearLoadBlocked() 171 { isLoadBlocked = false; } 172 173 /** Returns if the blocked load was handled. */ 174 bool isLoadBlockedHandled() 175 { return loadBlockedHandled; } 176 177 /** Records the blocked load as being handled. */ 178 void setLoadBlockedHandled() 179 { loadBlockedHandled = true; } 180 181 /** Returns the number of free entries (min of free LQ and SQ entries). */ 182 unsigned numFreeEntries(); 183 184 /** Returns the number of loads ready to execute. */ 185 int numLoadsReady(); 186 187 /** Returns the number of loads in the LQ. */ 188 int numLoads() { return loads; } 189 190 /** Returns the number of stores in the SQ. */ 191 int numStores() { return stores; } 192 193 /** Returns if either the LQ or SQ is full. */ 194 bool isFull() { return lqFull() || sqFull(); } 195 196 /** Returns if the LQ is full. */ 197 bool lqFull() { return loads >= (LQEntries - 1); } 198 199 /** Returns if the SQ is full. */ 200 bool sqFull() { return stores >= (SQEntries - 1); } 201 202 /** Returns the number of instructions in the LSQ. */ 203 unsigned getCount() { return loads + stores; } 204 205 /** Returns if there are any stores to writeback. */ 206 bool hasStoresToWB() { return storesToWB; } 207 208 /** Returns the number of stores to writeback. */ 209 int numStoresToWB() { return storesToWB; } 210 211 /** Returns if the LSQ unit will writeback on this cycle. */ 212 bool willWB() { return storeQueue[storeWBIdx].canWB && 213 !storeQueue[storeWBIdx].completed && 214 !isStoreBlocked; } 215 216 /** Handles doing the retry. */ 217 void recvRetry(); 218 219 private: 220 /** Writes back the instruction, sending it to IEW. */ 221 void writeback(DynInstPtr &inst, PacketPtr pkt); 222 223 /** Writes back a store that couldn't be completed the previous cycle. */ 224 void writebackPendingStore(); 225 226 /** Handles completing the send of a store to memory. */ 227 void storePostSend(PacketPtr pkt); 228 229 /** Completes the store at the specified index. */ 230 void completeStore(int store_idx); 231 232 /** Attempts to send a store to the cache. */ 233 bool sendStore(PacketPtr data_pkt); 234 235 /** Increments the given store index (circular queue). */ 236 inline void incrStIdx(int &store_idx); 237 /** Decrements the given store index (circular queue). */ 238 inline void decrStIdx(int &store_idx); 239 /** Increments the given load index (circular queue). */ 240 inline void incrLdIdx(int &load_idx); 241 /** Decrements the given load index (circular queue). */ 242 inline void decrLdIdx(int &load_idx); 243 244 public: 245 /** Debugging function to dump instructions in the LSQ. */ 246 void dumpInsts(); 247 248 private: 249 /** Pointer to the CPU. */ 250 O3CPU *cpu; 251 252 /** Pointer to the IEW stage. */ 253 IEW *iewStage; 254 255 /** Pointer to the LSQ. */ 256 LSQ *lsq; 257 258 /** Pointer to the dcache port. Used only for sending. */ 259 Port *dcachePort; 260 261 /** Derived class to hold any sender state the LSQ needs. */ 262 class LSQSenderState : public Packet::SenderState, public FastAlloc 263 { 264 public: 265 /** Default constructor. */ 266 LSQSenderState() 267 : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 268 mainPkt(NULL), pendingPacket(NULL) 269 { } 270 271 /** Instruction who initiated the access to memory. */ 272 DynInstPtr inst; 273 /** Whether or not it is a load. */ 274 bool isLoad; 275 /** The LQ/SQ index of the instruction. */ 276 int idx; 277 /** Whether or not the instruction will need to writeback. */ 278 bool noWB; 279 /** Whether or not this access is split in two. */ 280 bool isSplit; 281 /** Whether or not there is a packet that needs sending. */ 282 bool pktToSend; 283 /** Number of outstanding packets to complete. */ 284 int outstanding; 285 /** The main packet from a split load, used during writeback. */ 286 PacketPtr mainPkt; 287 /** A second packet from a split store that needs sending. */ 288 PacketPtr pendingPacket; 289 290 /** Completes a packet and returns whether the access is finished. */ 291 inline bool complete() { return --outstanding == 0; } 292 }; 293 294 /** Writeback event, specifically for when stores forward data to loads. */ 295 class WritebackEvent : public Event { 296 public: 297 /** Constructs a writeback event. */ 298 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 299 300 /** Processes the writeback event. */ 301 void process(); 302 303 /** Returns the description of this event. */ 304 const char *description() const; 305 306 private: 307 /** Instruction whose results are being written back. */ 308 DynInstPtr inst; 309 310 /** The packet that would have been sent to memory. */ 311 PacketPtr pkt; 312 313 /** The pointer to the LSQ unit that issued the store. */ 314 LSQUnit<Impl> *lsqPtr; 315 }; 316 317 public: 318 struct SQEntry { 319 /** Constructs an empty store queue entry. */ 320 SQEntry() 321 : inst(NULL), req(NULL), size(0), 322 canWB(0), committed(0), completed(0) 323 { 324 std::memset(data, 0, sizeof(data)); 325 } 326 327 /** Constructs a store queue entry for a given instruction. */ 328 SQEntry(DynInstPtr &_inst) 329 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 330 isSplit(0), canWB(0), committed(0), completed(0) 331 { 332 std::memset(data, 0, sizeof(data)); 333 } 334 335 /** The store instruction. */ 336 DynInstPtr inst; 337 /** The request for the store. */ 338 RequestPtr req; 339 /** The split requests for the store. */ 340 RequestPtr sreqLow; 341 RequestPtr sreqHigh; 342 /** The size of the store. */ 343 int size; 344 /** The store data. */ 345 char data[16]; 346 /** Whether or not the store is split into two requests. */ 347 bool isSplit; 348 /** Whether or not the store can writeback. */ 349 bool canWB; 350 /** Whether or not the store is committed. */ 351 bool committed; 352 /** Whether or not the store is completed. */ 353 bool completed; 354 }; 355 356 private: 357 /** The LSQUnit thread id. */ 358 ThreadID lsqID; 359 360 /** The store queue. */ 361 std::vector<SQEntry> storeQueue; 362 363 /** The load queue. */ 364 std::vector<DynInstPtr> loadQueue; 365 366 /** The number of LQ entries, plus a sentinel entry (circular queue). 367 * @todo: Consider having var that records the true number of LQ entries. 368 */ 369 unsigned LQEntries; 370 /** The number of SQ entries, plus a sentinel entry (circular queue). 371 * @todo: Consider having var that records the true number of SQ entries. 372 */ 373 unsigned SQEntries; 374 375 /** The number of places to shift addresses in the LSQ before checking 376 * for dependency violations 377 */ 378 unsigned depCheckShift; 379 380 /** Should loads be checked for dependency issues */ 381 bool checkLoads; 382 383 /** The number of load instructions in the LQ. */ 384 int loads; 385 /** The number of store instructions in the SQ. */ 386 int stores; 387 /** The number of store instructions in the SQ waiting to writeback. */ 388 int storesToWB; 389 390 /** The index of the head instruction in the LQ. */ 391 int loadHead; 392 /** The index of the tail instruction in the LQ. */ 393 int loadTail; 394 395 /** The index of the head instruction in the SQ. */ 396 int storeHead; 397 /** The index of the first instruction that may be ready to be 398 * written back, and has not yet been written back. 399 */ 400 int storeWBIdx; 401 /** The index of the tail instruction in the SQ. */ 402 int storeTail; 403 404 /// @todo Consider moving to a more advanced model with write vs read ports 405 /** The number of cache ports available each cycle. */ 406 int cachePorts; 407 408 /** The number of used cache ports in this cycle. */ 409 int usedPorts; 410 411 /** Is the LSQ switched out. */ 412 bool switchedOut; 413 414 //list<InstSeqNum> mshrSeqNums; 415 416 /** Wire to read information from the issue stage time queue. */ 417 typename TimeBuffer<IssueStruct>::wire fromIssue; 418 419 /** Whether or not the LSQ is stalled. */ 420 bool stalled; 421 /** The store that causes the stall due to partial store to load 422 * forwarding. 423 */ 424 InstSeqNum stallingStoreIsn; 425 /** The index of the above store. */ 426 int stallingLoadIdx; 427 428 /** The packet that needs to be retried. */ 429 PacketPtr retryPkt; 430 431 /** Whehter or not a store is blocked due to the memory system. */ 432 bool isStoreBlocked; 433 434 /** Whether or not a load is blocked due to the memory system. */ 435 bool isLoadBlocked; 436 437 /** Has the blocked load been handled. */ 438 bool loadBlockedHandled; 439 440 /** The sequence number of the blocked load. */ 441 InstSeqNum blockedLoadSeqNum; 442 443 /** The oldest load that caused a memory ordering violation. */ 444 DynInstPtr memDepViolator; 445 446 /** Whether or not there is a packet that couldn't be sent because of 447 * a lack of cache ports. */ 448 bool hasPendingPkt; 449 450 /** The packet that is pending free cache ports. */ 451 PacketPtr pendingPkt; 452 453 // Will also need how many read/write ports the Dcache has. Or keep track 454 // of that in stage that is one level up, and only call executeLoad/Store 455 // the appropriate number of times. 456 /** Total number of loads forwaded from LSQ stores. */ 457 Stats::Scalar lsqForwLoads; 458 459 /** Total number of loads ignored due to invalid addresses. */ 460 Stats::Scalar invAddrLoads; 461 462 /** Total number of squashed loads. */ 463 Stats::Scalar lsqSquashedLoads; 464 465 /** Total number of responses from the memory system that are 466 * ignored due to the instruction already being squashed. */ 467 Stats::Scalar lsqIgnoredResponses; 468 469 /** Tota number of memory ordering violations. */ 470 Stats::Scalar lsqMemOrderViolation; 471 472 /** Total number of squashed stores. */ 473 Stats::Scalar lsqSquashedStores; 474 475 /** Total number of software prefetches ignored due to invalid addresses. */ 476 Stats::Scalar invAddrSwpfs; 477 478 /** Ready loads blocked due to partial store-forwarding. */ 479 Stats::Scalar lsqBlockedLoads; 480 481 /** Number of loads that were rescheduled. */ 482 Stats::Scalar lsqRescheduledLoads; 483 484 /** Number of times the LSQ is blocked due to the cache. */ 485 Stats::Scalar lsqCacheBlocked; 486 487 public: 488 /** Executes the load at the given index. */ 489 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 490 uint8_t *data, int load_idx); 491 492 /** Executes the store at the given index. */ 493 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 494 uint8_t *data, int store_idx); 495 496 /** Returns the index of the head load instruction. */ 497 int getLoadHead() { return loadHead; } 498 /** Returns the sequence number of the head load instruction. */ 499 InstSeqNum getLoadHeadSeqNum() 500 { 501 if (loadQueue[loadHead]) { 502 return loadQueue[loadHead]->seqNum; 503 } else { 504 return 0; 505 } 506 507 } 508 509 /** Returns the index of the head store instruction. */ 510 int getStoreHead() { return storeHead; } 511 /** Returns the sequence number of the head store instruction. */ 512 InstSeqNum getStoreHeadSeqNum() 513 { 514 if (storeQueue[storeHead].inst) { 515 return storeQueue[storeHead].inst->seqNum; 516 } else { 517 return 0; 518 } 519 520 } 521 522 /** Returns whether or not the LSQ unit is stalled. */ 523 bool isStalled() { return stalled; } 524}; 525 526template <class Impl> 527Fault 528LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 529 uint8_t *data, int load_idx) 530{ 531 DynInstPtr load_inst = loadQueue[load_idx]; 532 533 assert(load_inst); 534 535 assert(!load_inst->isExecuted()); 536 537 // Make sure this isn't an uncacheable access 538 // A bit of a hackish way to get uncached accesses to work only if they're 539 // at the head of the LSQ and are ready to commit (at the head of the ROB 540 // too). 541 if (req->isUncacheable() && 542 (load_idx != loadHead || !load_inst->isAtCommit())) { 543 iewStage->rescheduleMemInst(load_inst); 544 ++lsqRescheduledLoads; 545 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n", 546 load_inst->seqNum, load_inst->pcState()); 547 548 // Must delete request now that it wasn't handed off to 549 // memory. This is quite ugly. @todo: Figure out the proper 550 // place to really handle request deletes. 551 delete req; 552 if (TheISA::HasUnalignedMemAcc && sreqLow) { 553 delete sreqLow; 554 delete sreqHigh; 555 } 556 return TheISA::genMachineCheckFault(); 557 } 558 559 // Check the SQ for any previous stores that might lead to forwarding 560 int store_idx = load_inst->sqIdx; 561 562 int store_size = 0; 563 564 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 565 "storeHead: %i addr: %#x%s\n", 566 load_idx, store_idx, storeHead, req->getPaddr(), 567 sreqLow ? " split" : ""); 568 569 if (req->isLLSC()) { 570 assert(!sreqLow); 571 // Disable recording the result temporarily. Writing to misc 572 // regs normally updates the result, but this is not the 573 // desired behavior when handling store conditionals. 574 load_inst->recordResult = false; 575 TheISA::handleLockedRead(load_inst.get(), req); 576 load_inst->recordResult = true; 577 } 578 579 while (store_idx != -1) { 580 // End once we've reached the top of the LSQ 581 if (store_idx == storeWBIdx) { 582 break; 583 } 584 585 // Move the index to one younger 586 if (--store_idx < 0) 587 store_idx += SQEntries; 588 589 assert(storeQueue[store_idx].inst); 590 591 store_size = storeQueue[store_idx].size; 592 593 if (store_size == 0) 594 continue; 595 else if (storeQueue[store_idx].inst->uncacheable()) 596 continue; 597 598 assert(storeQueue[store_idx].inst->effAddrValid); 599 600 // Check if the store data is within the lower and upper bounds of 601 // addresses that the request needs. 602 bool store_has_lower_limit = 603 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 604 bool store_has_upper_limit = 605 (req->getVaddr() + req->getSize()) <= 606 (storeQueue[store_idx].inst->effAddr + store_size); 607 bool lower_load_has_store_part = 608 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 609 store_size); 610 bool upper_load_has_store_part = 611 (req->getVaddr() + req->getSize()) > 612 storeQueue[store_idx].inst->effAddr; 613 614 // If the store's data has all of the data needed, we can forward. 615 if ((store_has_lower_limit && store_has_upper_limit)) { 616 // Get shift amount for offset into the store's data. 617 int shift_amt = req->getVaddr() & (store_size - 1); 618 619 memcpy(data, storeQueue[store_idx].data + shift_amt, 620 req->getSize()); 621 622 assert(!load_inst->memData); 623 load_inst->memData = new uint8_t[64]; 624 625 memcpy(load_inst->memData, 626 storeQueue[store_idx].data + shift_amt, req->getSize()); 627 628 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 629 "addr %#x, data %#x\n", 630 store_idx, req->getVaddr(), data); 631 632 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 633 Packet::Broadcast); 634 data_pkt->dataStatic(load_inst->memData); 635 636 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 637 638 // We'll say this has a 1 cycle load-store forwarding latency 639 // for now. 640 // @todo: Need to make this a parameter. 641 cpu->schedule(wb, curTick()); 642 643 // Don't need to do anything special for split loads. 644 if (TheISA::HasUnalignedMemAcc && sreqLow) { 645 delete sreqLow; 646 delete sreqHigh; 647 } 648 649 ++lsqForwLoads; 650 return NoFault; 651 } else if ((store_has_lower_limit && lower_load_has_store_part) || 652 (store_has_upper_limit && upper_load_has_store_part) || 653 (lower_load_has_store_part && upper_load_has_store_part)) { 654 // This is the partial store-load forwarding case where a store 655 // has only part of the load's data. 656 657 // If it's already been written back, then don't worry about 658 // stalling on it. 659 if (storeQueue[store_idx].completed) { 660 panic("Should not check one of these"); 661 continue; 662 } 663 664 // Must stall load and force it to retry, so long as it's the oldest 665 // load that needs to do so. 666 if (!stalled || 667 (stalled && 668 load_inst->seqNum < 669 loadQueue[stallingLoadIdx]->seqNum)) { 670 stalled = true; 671 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 672 stallingLoadIdx = load_idx; 673 } 674 675 // Tell IQ/mem dep unit that this instruction will need to be 676 // rescheduled eventually 677 iewStage->rescheduleMemInst(load_inst); 678 iewStage->decrWb(load_inst->seqNum); 679 load_inst->clearIssued(); 680 ++lsqRescheduledLoads; 681 682 // Do not generate a writeback event as this instruction is not 683 // complete. 684 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 685 "Store idx %i to load addr %#x\n", 686 store_idx, req->getVaddr()); 687 688 // Must delete request now that it wasn't handed off to 689 // memory. This is quite ugly. @todo: Figure out the 690 // proper place to really handle request deletes. 691 delete req; 692 if (TheISA::HasUnalignedMemAcc && sreqLow) { 693 delete sreqLow; 694 delete sreqHigh; 695 } 696 697 return NoFault; 698 } 699 } 700 701 // If there's no forwarding case, then go access memory 702 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 703 load_inst->seqNum, load_inst->pcState()); 704 705 assert(!load_inst->memData); 706 load_inst->memData = new uint8_t[64]; 707 708 ++usedPorts; 709 710 // if we the cache is not blocked, do cache access 711 bool completedFirst = false; 712 if (!lsq->cacheBlocked()) { 713 MemCmd command = 714 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 715 PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 716 PacketPtr fst_data_pkt = NULL; 717 PacketPtr snd_data_pkt = NULL; 718 719 data_pkt->dataStatic(load_inst->memData); 720 721 LSQSenderState *state = new LSQSenderState; 722 state->isLoad = true; 723 state->idx = load_idx; 724 state->inst = load_inst; 725 data_pkt->senderState = state; 726 727 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 728 729 // Point the first packet at the main data packet. 730 fst_data_pkt = data_pkt; 731 } else { 732 733 // Create the split packets. 734 fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 735 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 736 737 fst_data_pkt->dataStatic(load_inst->memData); 738 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 739 740 fst_data_pkt->senderState = state; 741 snd_data_pkt->senderState = state; 742 743 state->isSplit = true; 744 state->outstanding = 2; 745 state->mainPkt = data_pkt; 746 } 747 748 if (!dcachePort->sendTiming(fst_data_pkt)) { 749 // Delete state and data packet because a load retry 750 // initiates a pipeline restart; it does not retry. 751 delete state; 752 delete data_pkt->req; 753 delete data_pkt; 754 if (TheISA::HasUnalignedMemAcc && sreqLow) { 755 delete fst_data_pkt->req; 756 delete fst_data_pkt; 757 delete snd_data_pkt->req; 758 delete snd_data_pkt; 759 sreqLow = NULL; 760 sreqHigh = NULL; 761 } 762 763 req = NULL; 764 765 // If the access didn't succeed, tell the LSQ by setting 766 // the retry thread id. 767 lsq->setRetryTid(lsqID); 768 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 769 completedFirst = true; 770 771 // The first packet was sent without problems, so send this one 772 // too. If there is a problem with this packet then the whole 773 // load will be squashed, so indicate this to the state object. 774 // The first packet will return in completeDataAccess and be 775 // handled there. 776 ++usedPorts; 777 if (!dcachePort->sendTiming(snd_data_pkt)) { 778 779 // The main packet will be deleted in completeDataAccess. 780 delete snd_data_pkt->req; 781 delete snd_data_pkt; 782 783 state->complete(); 784 785 req = NULL; 786 sreqHigh = NULL; 787 788 lsq->setRetryTid(lsqID); 789 } 790 } 791 } 792 793 // If the cache was blocked, or has become blocked due to the access, 794 // handle it. 795 if (lsq->cacheBlocked()) { 796 if (req) 797 delete req; 798 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 799 delete sreqLow; 800 delete sreqHigh; 801 } 802 803 ++lsqCacheBlocked; 804 805 iewStage->decrWb(load_inst->seqNum); 806 // There's an older load that's already going to squash. 807 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 808 return NoFault; 809 810 // Record that the load was blocked due to memory. This 811 // load will squash all instructions after it, be 812 // refetched, and re-executed. 813 isLoadBlocked = true; 814 loadBlockedHandled = false; 815 blockedLoadSeqNum = load_inst->seqNum; 816 // No fault occurred, even though the interface is blocked. 817 return NoFault; 818 } 819 820 return NoFault; 821} 822 823template <class Impl> 824Fault 825LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 826 uint8_t *data, int store_idx) 827{ 828 assert(storeQueue[store_idx].inst); 829 830 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 831 " | storeHead:%i [sn:%i]\n", 832 store_idx, req->getPaddr(), data, storeHead, 833 storeQueue[store_idx].inst->seqNum); 834 835 storeQueue[store_idx].req = req; 836 storeQueue[store_idx].sreqLow = sreqLow; 837 storeQueue[store_idx].sreqHigh = sreqHigh; 838 unsigned size = req->getSize(); 839 storeQueue[store_idx].size = size; 840 assert(size <= sizeof(storeQueue[store_idx].data)); 841 842 // Split stores can only occur in ISAs with unaligned memory accesses. If 843 // a store request has been split, sreqLow and sreqHigh will be non-null. 844 if (TheISA::HasUnalignedMemAcc && sreqLow) { 845 storeQueue[store_idx].isSplit = true; 846 } 847 848 memcpy(storeQueue[store_idx].data, data, size); 849 850 // This function only writes the data to the store queue, so no fault 851 // can happen here. 852 return NoFault; 853} 854 855#endif // __CPU_O3_LSQ_UNIT_HH__
| 48#include "mem/packet.hh" 49#include "mem/port.hh" 50 51class DerivO3CPUParams; 52 53/** 54 * Class that implements the actual LQ and SQ for each specific 55 * thread. Both are circular queues; load entries are freed upon 56 * committing, while store entries are freed once they writeback. The 57 * LSQUnit tracks if there are memory ordering violations, and also 58 * detects partial load to store forwarding cases (a store only has 59 * part of a load's data) that requires the load to wait until the 60 * store writes back. In the former case it holds onto the instruction 61 * until the dependence unit looks at it, and in the latter it stalls 62 * the LSQ until the store writes back. At that point the load is 63 * replayed. 64 */ 65template <class Impl> 66class LSQUnit { 67 public: 68 typedef typename Impl::O3CPU O3CPU; 69 typedef typename Impl::DynInstPtr DynInstPtr; 70 typedef typename Impl::CPUPol::IEW IEW; 71 typedef typename Impl::CPUPol::LSQ LSQ; 72 typedef typename Impl::CPUPol::IssueStruct IssueStruct; 73 74 public: 75 /** Constructs an LSQ unit. init() must be called prior to use. */ 76 LSQUnit(); 77 78 /** Initializes the LSQ unit with the specified number of entries. */ 79 void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 80 LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 81 unsigned id); 82 83 /** Returns the name of the LSQ unit. */ 84 std::string name() const; 85 86 /** Registers statistics. */ 87 void regStats(); 88 89 /** Sets the pointer to the dcache port. */ 90 void setDcachePort(Port *dcache_port); 91 92 /** Switches out LSQ unit. */ 93 void switchOut(); 94 95 /** Takes over from another CPU's thread. */ 96 void takeOverFrom(); 97 98 /** Returns if the LSQ is switched out. */ 99 bool isSwitchedOut() { return switchedOut; } 100 101 /** Ticks the LSQ unit, which in this case only resets the number of 102 * used cache ports. 103 * @todo: Move the number of used ports up to the LSQ level so it can 104 * be shared by all LSQ units. 105 */ 106 void tick() { usedPorts = 0; } 107 108 /** Inserts an instruction. */ 109 void insert(DynInstPtr &inst); 110 /** Inserts a load instruction. */ 111 void insertLoad(DynInstPtr &load_inst); 112 /** Inserts a store instruction. */ 113 void insertStore(DynInstPtr &store_inst); 114 115 /** Check for ordering violations in the LSQ 116 * @param load_idx index to start checking at 117 * @param inst the instruction to check 118 */ 119 Fault checkViolations(int load_idx, DynInstPtr &inst); 120 121 /** Executes a load instruction. */ 122 Fault executeLoad(DynInstPtr &inst); 123 124 Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 125 /** Executes a store instruction. */ 126 Fault executeStore(DynInstPtr &inst); 127 128 /** Commits the head load. */ 129 void commitLoad(); 130 /** Commits loads older than a specific sequence number. */ 131 void commitLoads(InstSeqNum &youngest_inst); 132 133 /** Commits stores older than a specific sequence number. */ 134 void commitStores(InstSeqNum &youngest_inst); 135 136 /** Writes back stores. */ 137 void writebackStores(); 138 139 /** Completes the data access that has been returned from the 140 * memory system. */ 141 void completeDataAccess(PacketPtr pkt); 142 143 /** Clears all the entries in the LQ. */ 144 void clearLQ(); 145 146 /** Clears all the entries in the SQ. */ 147 void clearSQ(); 148 149 /** Resizes the LQ to a given size. */ 150 void resizeLQ(unsigned size); 151 152 /** Resizes the SQ to a given size. */ 153 void resizeSQ(unsigned size); 154 155 /** Squashes all instructions younger than a specific sequence number. */ 156 void squash(const InstSeqNum &squashed_num); 157 158 /** Returns if there is a memory ordering violation. Value is reset upon 159 * call to getMemDepViolator(). 160 */ 161 bool violation() { return memDepViolator; } 162 163 /** Returns the memory ordering violator. */ 164 DynInstPtr getMemDepViolator(); 165 166 /** Returns if a load became blocked due to the memory system. */ 167 bool loadBlocked() 168 { return isLoadBlocked; } 169 170 /** Clears the signal that a load became blocked. */ 171 void clearLoadBlocked() 172 { isLoadBlocked = false; } 173 174 /** Returns if the blocked load was handled. */ 175 bool isLoadBlockedHandled() 176 { return loadBlockedHandled; } 177 178 /** Records the blocked load as being handled. */ 179 void setLoadBlockedHandled() 180 { loadBlockedHandled = true; } 181 182 /** Returns the number of free entries (min of free LQ and SQ entries). */ 183 unsigned numFreeEntries(); 184 185 /** Returns the number of loads ready to execute. */ 186 int numLoadsReady(); 187 188 /** Returns the number of loads in the LQ. */ 189 int numLoads() { return loads; } 190 191 /** Returns the number of stores in the SQ. */ 192 int numStores() { return stores; } 193 194 /** Returns if either the LQ or SQ is full. */ 195 bool isFull() { return lqFull() || sqFull(); } 196 197 /** Returns if the LQ is full. */ 198 bool lqFull() { return loads >= (LQEntries - 1); } 199 200 /** Returns if the SQ is full. */ 201 bool sqFull() { return stores >= (SQEntries - 1); } 202 203 /** Returns the number of instructions in the LSQ. */ 204 unsigned getCount() { return loads + stores; } 205 206 /** Returns if there are any stores to writeback. */ 207 bool hasStoresToWB() { return storesToWB; } 208 209 /** Returns the number of stores to writeback. */ 210 int numStoresToWB() { return storesToWB; } 211 212 /** Returns if the LSQ unit will writeback on this cycle. */ 213 bool willWB() { return storeQueue[storeWBIdx].canWB && 214 !storeQueue[storeWBIdx].completed && 215 !isStoreBlocked; } 216 217 /** Handles doing the retry. */ 218 void recvRetry(); 219 220 private: 221 /** Writes back the instruction, sending it to IEW. */ 222 void writeback(DynInstPtr &inst, PacketPtr pkt); 223 224 /** Writes back a store that couldn't be completed the previous cycle. */ 225 void writebackPendingStore(); 226 227 /** Handles completing the send of a store to memory. */ 228 void storePostSend(PacketPtr pkt); 229 230 /** Completes the store at the specified index. */ 231 void completeStore(int store_idx); 232 233 /** Attempts to send a store to the cache. */ 234 bool sendStore(PacketPtr data_pkt); 235 236 /** Increments the given store index (circular queue). */ 237 inline void incrStIdx(int &store_idx); 238 /** Decrements the given store index (circular queue). */ 239 inline void decrStIdx(int &store_idx); 240 /** Increments the given load index (circular queue). */ 241 inline void incrLdIdx(int &load_idx); 242 /** Decrements the given load index (circular queue). */ 243 inline void decrLdIdx(int &load_idx); 244 245 public: 246 /** Debugging function to dump instructions in the LSQ. */ 247 void dumpInsts(); 248 249 private: 250 /** Pointer to the CPU. */ 251 O3CPU *cpu; 252 253 /** Pointer to the IEW stage. */ 254 IEW *iewStage; 255 256 /** Pointer to the LSQ. */ 257 LSQ *lsq; 258 259 /** Pointer to the dcache port. Used only for sending. */ 260 Port *dcachePort; 261 262 /** Derived class to hold any sender state the LSQ needs. */ 263 class LSQSenderState : public Packet::SenderState, public FastAlloc 264 { 265 public: 266 /** Default constructor. */ 267 LSQSenderState() 268 : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 269 mainPkt(NULL), pendingPacket(NULL) 270 { } 271 272 /** Instruction who initiated the access to memory. */ 273 DynInstPtr inst; 274 /** Whether or not it is a load. */ 275 bool isLoad; 276 /** The LQ/SQ index of the instruction. */ 277 int idx; 278 /** Whether or not the instruction will need to writeback. */ 279 bool noWB; 280 /** Whether or not this access is split in two. */ 281 bool isSplit; 282 /** Whether or not there is a packet that needs sending. */ 283 bool pktToSend; 284 /** Number of outstanding packets to complete. */ 285 int outstanding; 286 /** The main packet from a split load, used during writeback. */ 287 PacketPtr mainPkt; 288 /** A second packet from a split store that needs sending. */ 289 PacketPtr pendingPacket; 290 291 /** Completes a packet and returns whether the access is finished. */ 292 inline bool complete() { return --outstanding == 0; } 293 }; 294 295 /** Writeback event, specifically for when stores forward data to loads. */ 296 class WritebackEvent : public Event { 297 public: 298 /** Constructs a writeback event. */ 299 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 300 301 /** Processes the writeback event. */ 302 void process(); 303 304 /** Returns the description of this event. */ 305 const char *description() const; 306 307 private: 308 /** Instruction whose results are being written back. */ 309 DynInstPtr inst; 310 311 /** The packet that would have been sent to memory. */ 312 PacketPtr pkt; 313 314 /** The pointer to the LSQ unit that issued the store. */ 315 LSQUnit<Impl> *lsqPtr; 316 }; 317 318 public: 319 struct SQEntry { 320 /** Constructs an empty store queue entry. */ 321 SQEntry() 322 : inst(NULL), req(NULL), size(0), 323 canWB(0), committed(0), completed(0) 324 { 325 std::memset(data, 0, sizeof(data)); 326 } 327 328 /** Constructs a store queue entry for a given instruction. */ 329 SQEntry(DynInstPtr &_inst) 330 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 331 isSplit(0), canWB(0), committed(0), completed(0) 332 { 333 std::memset(data, 0, sizeof(data)); 334 } 335 336 /** The store instruction. */ 337 DynInstPtr inst; 338 /** The request for the store. */ 339 RequestPtr req; 340 /** The split requests for the store. */ 341 RequestPtr sreqLow; 342 RequestPtr sreqHigh; 343 /** The size of the store. */ 344 int size; 345 /** The store data. */ 346 char data[16]; 347 /** Whether or not the store is split into two requests. */ 348 bool isSplit; 349 /** Whether or not the store can writeback. */ 350 bool canWB; 351 /** Whether or not the store is committed. */ 352 bool committed; 353 /** Whether or not the store is completed. */ 354 bool completed; 355 }; 356 357 private: 358 /** The LSQUnit thread id. */ 359 ThreadID lsqID; 360 361 /** The store queue. */ 362 std::vector<SQEntry> storeQueue; 363 364 /** The load queue. */ 365 std::vector<DynInstPtr> loadQueue; 366 367 /** The number of LQ entries, plus a sentinel entry (circular queue). 368 * @todo: Consider having var that records the true number of LQ entries. 369 */ 370 unsigned LQEntries; 371 /** The number of SQ entries, plus a sentinel entry (circular queue). 372 * @todo: Consider having var that records the true number of SQ entries. 373 */ 374 unsigned SQEntries; 375 376 /** The number of places to shift addresses in the LSQ before checking 377 * for dependency violations 378 */ 379 unsigned depCheckShift; 380 381 /** Should loads be checked for dependency issues */ 382 bool checkLoads; 383 384 /** The number of load instructions in the LQ. */ 385 int loads; 386 /** The number of store instructions in the SQ. */ 387 int stores; 388 /** The number of store instructions in the SQ waiting to writeback. */ 389 int storesToWB; 390 391 /** The index of the head instruction in the LQ. */ 392 int loadHead; 393 /** The index of the tail instruction in the LQ. */ 394 int loadTail; 395 396 /** The index of the head instruction in the SQ. */ 397 int storeHead; 398 /** The index of the first instruction that may be ready to be 399 * written back, and has not yet been written back. 400 */ 401 int storeWBIdx; 402 /** The index of the tail instruction in the SQ. */ 403 int storeTail; 404 405 /// @todo Consider moving to a more advanced model with write vs read ports 406 /** The number of cache ports available each cycle. */ 407 int cachePorts; 408 409 /** The number of used cache ports in this cycle. */ 410 int usedPorts; 411 412 /** Is the LSQ switched out. */ 413 bool switchedOut; 414 415 //list<InstSeqNum> mshrSeqNums; 416 417 /** Wire to read information from the issue stage time queue. */ 418 typename TimeBuffer<IssueStruct>::wire fromIssue; 419 420 /** Whether or not the LSQ is stalled. */ 421 bool stalled; 422 /** The store that causes the stall due to partial store to load 423 * forwarding. 424 */ 425 InstSeqNum stallingStoreIsn; 426 /** The index of the above store. */ 427 int stallingLoadIdx; 428 429 /** The packet that needs to be retried. */ 430 PacketPtr retryPkt; 431 432 /** Whehter or not a store is blocked due to the memory system. */ 433 bool isStoreBlocked; 434 435 /** Whether or not a load is blocked due to the memory system. */ 436 bool isLoadBlocked; 437 438 /** Has the blocked load been handled. */ 439 bool loadBlockedHandled; 440 441 /** The sequence number of the blocked load. */ 442 InstSeqNum blockedLoadSeqNum; 443 444 /** The oldest load that caused a memory ordering violation. */ 445 DynInstPtr memDepViolator; 446 447 /** Whether or not there is a packet that couldn't be sent because of 448 * a lack of cache ports. */ 449 bool hasPendingPkt; 450 451 /** The packet that is pending free cache ports. */ 452 PacketPtr pendingPkt; 453 454 // Will also need how many read/write ports the Dcache has. Or keep track 455 // of that in stage that is one level up, and only call executeLoad/Store 456 // the appropriate number of times. 457 /** Total number of loads forwaded from LSQ stores. */ 458 Stats::Scalar lsqForwLoads; 459 460 /** Total number of loads ignored due to invalid addresses. */ 461 Stats::Scalar invAddrLoads; 462 463 /** Total number of squashed loads. */ 464 Stats::Scalar lsqSquashedLoads; 465 466 /** Total number of responses from the memory system that are 467 * ignored due to the instruction already being squashed. */ 468 Stats::Scalar lsqIgnoredResponses; 469 470 /** Tota number of memory ordering violations. */ 471 Stats::Scalar lsqMemOrderViolation; 472 473 /** Total number of squashed stores. */ 474 Stats::Scalar lsqSquashedStores; 475 476 /** Total number of software prefetches ignored due to invalid addresses. */ 477 Stats::Scalar invAddrSwpfs; 478 479 /** Ready loads blocked due to partial store-forwarding. */ 480 Stats::Scalar lsqBlockedLoads; 481 482 /** Number of loads that were rescheduled. */ 483 Stats::Scalar lsqRescheduledLoads; 484 485 /** Number of times the LSQ is blocked due to the cache. */ 486 Stats::Scalar lsqCacheBlocked; 487 488 public: 489 /** Executes the load at the given index. */ 490 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 491 uint8_t *data, int load_idx); 492 493 /** Executes the store at the given index. */ 494 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 495 uint8_t *data, int store_idx); 496 497 /** Returns the index of the head load instruction. */ 498 int getLoadHead() { return loadHead; } 499 /** Returns the sequence number of the head load instruction. */ 500 InstSeqNum getLoadHeadSeqNum() 501 { 502 if (loadQueue[loadHead]) { 503 return loadQueue[loadHead]->seqNum; 504 } else { 505 return 0; 506 } 507 508 } 509 510 /** Returns the index of the head store instruction. */ 511 int getStoreHead() { return storeHead; } 512 /** Returns the sequence number of the head store instruction. */ 513 InstSeqNum getStoreHeadSeqNum() 514 { 515 if (storeQueue[storeHead].inst) { 516 return storeQueue[storeHead].inst->seqNum; 517 } else { 518 return 0; 519 } 520 521 } 522 523 /** Returns whether or not the LSQ unit is stalled. */ 524 bool isStalled() { return stalled; } 525}; 526 527template <class Impl> 528Fault 529LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 530 uint8_t *data, int load_idx) 531{ 532 DynInstPtr load_inst = loadQueue[load_idx]; 533 534 assert(load_inst); 535 536 assert(!load_inst->isExecuted()); 537 538 // Make sure this isn't an uncacheable access 539 // A bit of a hackish way to get uncached accesses to work only if they're 540 // at the head of the LSQ and are ready to commit (at the head of the ROB 541 // too). 542 if (req->isUncacheable() && 543 (load_idx != loadHead || !load_inst->isAtCommit())) { 544 iewStage->rescheduleMemInst(load_inst); 545 ++lsqRescheduledLoads; 546 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n", 547 load_inst->seqNum, load_inst->pcState()); 548 549 // Must delete request now that it wasn't handed off to 550 // memory. This is quite ugly. @todo: Figure out the proper 551 // place to really handle request deletes. 552 delete req; 553 if (TheISA::HasUnalignedMemAcc && sreqLow) { 554 delete sreqLow; 555 delete sreqHigh; 556 } 557 return TheISA::genMachineCheckFault(); 558 } 559 560 // Check the SQ for any previous stores that might lead to forwarding 561 int store_idx = load_inst->sqIdx; 562 563 int store_size = 0; 564 565 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 566 "storeHead: %i addr: %#x%s\n", 567 load_idx, store_idx, storeHead, req->getPaddr(), 568 sreqLow ? " split" : ""); 569 570 if (req->isLLSC()) { 571 assert(!sreqLow); 572 // Disable recording the result temporarily. Writing to misc 573 // regs normally updates the result, but this is not the 574 // desired behavior when handling store conditionals. 575 load_inst->recordResult = false; 576 TheISA::handleLockedRead(load_inst.get(), req); 577 load_inst->recordResult = true; 578 } 579 580 while (store_idx != -1) { 581 // End once we've reached the top of the LSQ 582 if (store_idx == storeWBIdx) { 583 break; 584 } 585 586 // Move the index to one younger 587 if (--store_idx < 0) 588 store_idx += SQEntries; 589 590 assert(storeQueue[store_idx].inst); 591 592 store_size = storeQueue[store_idx].size; 593 594 if (store_size == 0) 595 continue; 596 else if (storeQueue[store_idx].inst->uncacheable()) 597 continue; 598 599 assert(storeQueue[store_idx].inst->effAddrValid); 600 601 // Check if the store data is within the lower and upper bounds of 602 // addresses that the request needs. 603 bool store_has_lower_limit = 604 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 605 bool store_has_upper_limit = 606 (req->getVaddr() + req->getSize()) <= 607 (storeQueue[store_idx].inst->effAddr + store_size); 608 bool lower_load_has_store_part = 609 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 610 store_size); 611 bool upper_load_has_store_part = 612 (req->getVaddr() + req->getSize()) > 613 storeQueue[store_idx].inst->effAddr; 614 615 // If the store's data has all of the data needed, we can forward. 616 if ((store_has_lower_limit && store_has_upper_limit)) { 617 // Get shift amount for offset into the store's data. 618 int shift_amt = req->getVaddr() & (store_size - 1); 619 620 memcpy(data, storeQueue[store_idx].data + shift_amt, 621 req->getSize()); 622 623 assert(!load_inst->memData); 624 load_inst->memData = new uint8_t[64]; 625 626 memcpy(load_inst->memData, 627 storeQueue[store_idx].data + shift_amt, req->getSize()); 628 629 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 630 "addr %#x, data %#x\n", 631 store_idx, req->getVaddr(), data); 632 633 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 634 Packet::Broadcast); 635 data_pkt->dataStatic(load_inst->memData); 636 637 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 638 639 // We'll say this has a 1 cycle load-store forwarding latency 640 // for now. 641 // @todo: Need to make this a parameter. 642 cpu->schedule(wb, curTick()); 643 644 // Don't need to do anything special for split loads. 645 if (TheISA::HasUnalignedMemAcc && sreqLow) { 646 delete sreqLow; 647 delete sreqHigh; 648 } 649 650 ++lsqForwLoads; 651 return NoFault; 652 } else if ((store_has_lower_limit && lower_load_has_store_part) || 653 (store_has_upper_limit && upper_load_has_store_part) || 654 (lower_load_has_store_part && upper_load_has_store_part)) { 655 // This is the partial store-load forwarding case where a store 656 // has only part of the load's data. 657 658 // If it's already been written back, then don't worry about 659 // stalling on it. 660 if (storeQueue[store_idx].completed) { 661 panic("Should not check one of these"); 662 continue; 663 } 664 665 // Must stall load and force it to retry, so long as it's the oldest 666 // load that needs to do so. 667 if (!stalled || 668 (stalled && 669 load_inst->seqNum < 670 loadQueue[stallingLoadIdx]->seqNum)) { 671 stalled = true; 672 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 673 stallingLoadIdx = load_idx; 674 } 675 676 // Tell IQ/mem dep unit that this instruction will need to be 677 // rescheduled eventually 678 iewStage->rescheduleMemInst(load_inst); 679 iewStage->decrWb(load_inst->seqNum); 680 load_inst->clearIssued(); 681 ++lsqRescheduledLoads; 682 683 // Do not generate a writeback event as this instruction is not 684 // complete. 685 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 686 "Store idx %i to load addr %#x\n", 687 store_idx, req->getVaddr()); 688 689 // Must delete request now that it wasn't handed off to 690 // memory. This is quite ugly. @todo: Figure out the 691 // proper place to really handle request deletes. 692 delete req; 693 if (TheISA::HasUnalignedMemAcc && sreqLow) { 694 delete sreqLow; 695 delete sreqHigh; 696 } 697 698 return NoFault; 699 } 700 } 701 702 // If there's no forwarding case, then go access memory 703 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 704 load_inst->seqNum, load_inst->pcState()); 705 706 assert(!load_inst->memData); 707 load_inst->memData = new uint8_t[64]; 708 709 ++usedPorts; 710 711 // if we the cache is not blocked, do cache access 712 bool completedFirst = false; 713 if (!lsq->cacheBlocked()) { 714 MemCmd command = 715 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 716 PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 717 PacketPtr fst_data_pkt = NULL; 718 PacketPtr snd_data_pkt = NULL; 719 720 data_pkt->dataStatic(load_inst->memData); 721 722 LSQSenderState *state = new LSQSenderState; 723 state->isLoad = true; 724 state->idx = load_idx; 725 state->inst = load_inst; 726 data_pkt->senderState = state; 727 728 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 729 730 // Point the first packet at the main data packet. 731 fst_data_pkt = data_pkt; 732 } else { 733 734 // Create the split packets. 735 fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 736 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 737 738 fst_data_pkt->dataStatic(load_inst->memData); 739 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 740 741 fst_data_pkt->senderState = state; 742 snd_data_pkt->senderState = state; 743 744 state->isSplit = true; 745 state->outstanding = 2; 746 state->mainPkt = data_pkt; 747 } 748 749 if (!dcachePort->sendTiming(fst_data_pkt)) { 750 // Delete state and data packet because a load retry 751 // initiates a pipeline restart; it does not retry. 752 delete state; 753 delete data_pkt->req; 754 delete data_pkt; 755 if (TheISA::HasUnalignedMemAcc && sreqLow) { 756 delete fst_data_pkt->req; 757 delete fst_data_pkt; 758 delete snd_data_pkt->req; 759 delete snd_data_pkt; 760 sreqLow = NULL; 761 sreqHigh = NULL; 762 } 763 764 req = NULL; 765 766 // If the access didn't succeed, tell the LSQ by setting 767 // the retry thread id. 768 lsq->setRetryTid(lsqID); 769 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 770 completedFirst = true; 771 772 // The first packet was sent without problems, so send this one 773 // too. If there is a problem with this packet then the whole 774 // load will be squashed, so indicate this to the state object. 775 // The first packet will return in completeDataAccess and be 776 // handled there. 777 ++usedPorts; 778 if (!dcachePort->sendTiming(snd_data_pkt)) { 779 780 // The main packet will be deleted in completeDataAccess. 781 delete snd_data_pkt->req; 782 delete snd_data_pkt; 783 784 state->complete(); 785 786 req = NULL; 787 sreqHigh = NULL; 788 789 lsq->setRetryTid(lsqID); 790 } 791 } 792 } 793 794 // If the cache was blocked, or has become blocked due to the access, 795 // handle it. 796 if (lsq->cacheBlocked()) { 797 if (req) 798 delete req; 799 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 800 delete sreqLow; 801 delete sreqHigh; 802 } 803 804 ++lsqCacheBlocked; 805 806 iewStage->decrWb(load_inst->seqNum); 807 // There's an older load that's already going to squash. 808 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 809 return NoFault; 810 811 // Record that the load was blocked due to memory. This 812 // load will squash all instructions after it, be 813 // refetched, and re-executed. 814 isLoadBlocked = true; 815 loadBlockedHandled = false; 816 blockedLoadSeqNum = load_inst->seqNum; 817 // No fault occurred, even though the interface is blocked. 818 return NoFault; 819 } 820 821 return NoFault; 822} 823 824template <class Impl> 825Fault 826LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 827 uint8_t *data, int store_idx) 828{ 829 assert(storeQueue[store_idx].inst); 830 831 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 832 " | storeHead:%i [sn:%i]\n", 833 store_idx, req->getPaddr(), data, storeHead, 834 storeQueue[store_idx].inst->seqNum); 835 836 storeQueue[store_idx].req = req; 837 storeQueue[store_idx].sreqLow = sreqLow; 838 storeQueue[store_idx].sreqHigh = sreqHigh; 839 unsigned size = req->getSize(); 840 storeQueue[store_idx].size = size; 841 assert(size <= sizeof(storeQueue[store_idx].data)); 842 843 // Split stores can only occur in ISAs with unaligned memory accesses. If 844 // a store request has been split, sreqLow and sreqHigh will be non-null. 845 if (TheISA::HasUnalignedMemAcc && sreqLow) { 846 storeQueue[store_idx].isSplit = true; 847 } 848 849 memcpy(storeQueue[store_idx].data, data, size); 850 851 // This function only writes the data to the store queue, so no fault 852 // can happen here. 853 return NoFault; 854} 855 856#endif // __CPU_O3_LSQ_UNIT_HH__
|