lsq_unit.hh revision 11302
12SN/A/* 21762SN/A * Copyright (c) 2012-2014 ARM Limited 32SN/A * All rights reserved 42SN/A * 52SN/A * The license below extends only to copyright in the software and shall 62SN/A * not be construed as granting a license to any other intellectual 72SN/A * property including but not limited to intellectual property relating 82SN/A * to a hardware implementation of the functionality of the software 92SN/A * licensed hereunder. You may use the software subject to the license 102SN/A * terms below provided that you ensure that this notice is replicated 112SN/A * unmodified and in its entirety in all distributions of the software, 122SN/A * modified or unmodified, in source code or in binary form. 132SN/A * 142SN/A * Copyright (c) 2004-2006 The Regents of The University of Michigan 152SN/A * Copyright (c) 2013 Advanced Micro Devices, Inc. 162SN/A * All rights reserved. 172SN/A * 182SN/A * Redistribution and use in source and binary forms, with or without 192SN/A * modification, are permitted provided that the following conditions are 202SN/A * met: redistributions of source code must retain the above copyright 212SN/A * notice, this list of conditions and the following disclaimer; 222SN/A * redistributions in binary form must reproduce the above copyright 232SN/A * notice, this list of conditions and the following disclaimer in the 242SN/A * documentation and/or other materials provided with the distribution; 252SN/A * neither the name of the copyright holders nor the names of its 262SN/A * contributors may be used to endorse or promote products derived from 272665Ssaidi@eecs.umich.edu * this software without specific prior written permission. 282665Ssaidi@eecs.umich.edu * 292665Ssaidi@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 302665Ssaidi@eecs.umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 312SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 322SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 332623SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 342623SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 352SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 361354SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 371858SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 381717SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 392683Sktlim@umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 401354SN/A * 411354SN/A * Authors: Kevin Lim 422387SN/A * Korey Sewell 432387SN/A */ 442387SN/A 4556SN/A#ifndef __CPU_O3_LSQ_UNIT_HH__ 462SN/A#define __CPU_O3_LSQ_UNIT_HH__ 472SN/A 481858SN/A#include <algorithm> 492SN/A#include <cstring> 503453Sgblack@eecs.umich.edu#include <map> 513453Sgblack@eecs.umich.edu#include <queue> 523453Sgblack@eecs.umich.edu 533453Sgblack@eecs.umich.edu#include "arch/generic/debugfaults.hh" 543453Sgblack@eecs.umich.edu#include "arch/isa_traits.hh" 552462SN/A#include "arch/locked_mem.hh" 562SN/A#include "arch/mmapped_ipr.hh" 572SN/A#include "config/the_isa.hh" 582SN/A#include "cpu/inst_seq.hh" 59715SN/A#include "cpu/timebuf.hh" 60715SN/A#include "debug/LSQUnit.hh" 61715SN/A#include "mem/packet.hh" 62715SN/A#include "mem/port.hh" 63715SN/A 642SN/Astruct DerivO3CPUParams; 652SN/A 662680Sktlim@umich.edu/** 67237SN/A * Class that implements the actual LQ and SQ for each specific 682SN/A * thread. Both are circular queues; load entries are freed upon 692SN/A * committing, while store entries are freed once they writeback. The 702SN/A * LSQUnit tracks if there are memory ordering violations, and also 712SN/A * detects partial load to store forwarding cases (a store only has 722SN/A * part of a load's data) that requires the load to wait until the 732420SN/A * store writes back. In the former case it holds onto the instruction 742623SN/A * until the dependence unit looks at it, and in the latter it stalls 752SN/A * the LSQ until the store writes back. At that point the load is 762107SN/A * replayed. 772107SN/A */ 782159SN/Atemplate <class Impl> 792455SN/Aclass LSQUnit { 802455SN/A public: 812386SN/A typedef typename Impl::O3CPU O3CPU; 822499SN/A typedef typename Impl::DynInstPtr DynInstPtr; 832386SN/A typedef typename Impl::CPUPol::IEW IEW; 842623SN/A typedef typename Impl::CPUPol::LSQ LSQ; 852SN/A typedef typename Impl::CPUPol::IssueStruct IssueStruct; 861371SN/A 872SN/A public: 882SN/A /** Constructs an LSQ unit. init() must be called prior to use. */ 892SN/A LSQUnit(); 902SN/A 912SN/A /** Initializes the LSQ unit with the specified number of entries. */ 922SN/A void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 932SN/A LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 942SN/A unsigned id); 952SN/A 962SN/A /** Returns the name of the LSQ unit. */ 972SN/A std::string name() const; 981400SN/A 991400SN/A /** Registers statistics. */ 1001400SN/A void regStats(); 1012542SN/A 1021858SN/A /** Sets the pointer to the dcache port. */ 1033453Sgblack@eecs.umich.edu void setDcachePort(MasterPort *dcache_port); 1043453Sgblack@eecs.umich.edu 1052SN/A /** Perform sanity checks after a drain. */ 1061400SN/A void drainSanityCheck() const; 1072SN/A 1081400SN/A /** Takes over from another CPU's thread. */ 1092623SN/A void takeOverFrom(); 1102623SN/A 1112SN/A /** Ticks the LSQ unit, which in this case only resets the number of 1121400SN/A * used cache ports. 1132683Sktlim@umich.edu * @todo: Move the number of used ports up to the LSQ level so it can 1142683Sktlim@umich.edu * be shared by all LSQ units. 1152190SN/A */ 1162683Sktlim@umich.edu void tick() { usedPorts = 0; } 1172683Sktlim@umich.edu 1182683Sktlim@umich.edu /** Inserts an instruction. */ 1192680Sktlim@umich.edu void insert(DynInstPtr &inst); 1202SN/A /** Inserts a load instruction. */ 1211858SN/A void insertLoad(DynInstPtr &load_inst); 1222SN/A /** Inserts a store instruction. */ 1232SN/A void insertStore(DynInstPtr &store_inst); 1242SN/A 1252SN/A /** Check for ordering violations in the LSQ. For a store squash if we 1262SN/A * ever find a conflicting load. For a load, only squash if we 1272SN/A * an external snoop invalidate has been seen for that load address 1282SN/A * @param load_idx index to start checking at 1292SN/A * @param inst the instruction to check 1302566SN/A */ 1312566SN/A Fault checkViolations(int load_idx, DynInstPtr &inst); 1322566SN/A 1332107SN/A /** Check if an incoming invalidate hits in the lsq on a load 1343276Sgblack@eecs.umich.edu * that might have issued out of order wrt another load beacuse 1351469SN/A * of the intermediate invalidate. 1362623SN/A */ 1372662Sstever@eecs.umich.edu void checkSnoop(PacketPtr pkt); 1382623SN/A 1392623SN/A /** Executes a load instruction. */ 1402623SN/A Fault executeLoad(DynInstPtr &inst); 141180SN/A 142393SN/A Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 143393SN/A /** Executes a store instruction. */ 1442SN/A Fault executeStore(DynInstPtr &inst); 1452SN/A 146334SN/A /** Commits the head load. */ 147334SN/A void commitLoad(); 1482SN/A /** Commits loads older than a specific sequence number. */ 1492SN/A void commitLoads(InstSeqNum &youngest_inst); 1502SN/A 151334SN/A /** Commits stores older than a specific sequence number. */ 152729SN/A void commitStores(InstSeqNum &youngest_inst); 153707SN/A 154707SN/A /** Writes back stores. */ 155707SN/A void writebackStores(); 156707SN/A 157707SN/A /** Completes the data access that has been returned from the 1582SN/A * memory system. */ 1592SN/A void completeDataAccess(PacketPtr pkt); 160729SN/A 1612SN/A /** Clears all the entries in the LQ. */ 162124SN/A void clearLQ(); 163124SN/A 164334SN/A /** Clears all the entries in the SQ. */ 165124SN/A void clearSQ(); 1662SN/A 167729SN/A /** Resizes the LQ to a given size. */ 168729SN/A void resizeLQ(unsigned size); 1692SN/A 1702390SN/A /** Resizes the SQ to a given size. */ 171729SN/A void resizeSQ(unsigned size); 1722SN/A 1732SN/A /** Squashes all instructions younger than a specific sequence number. */ 1742390SN/A void squash(const InstSeqNum &squashed_num); 1752390SN/A 1762390SN/A /** Returns if there is a memory ordering violation. Value is reset upon 1772390SN/A * call to getMemDepViolator(). 1782390SN/A */ 179729SN/A bool violation() { return memDepViolator; } 1802SN/A 1812SN/A /** Returns the memory ordering violator. */ 1822390SN/A DynInstPtr getMemDepViolator(); 1832390SN/A 1842390SN/A /** Returns the number of free LQ entries. */ 1852390SN/A unsigned numFreeLoadEntries(); 186217SN/A 187237SN/A /** Returns the number of free SQ entries. */ 1882SN/A unsigned numFreeStoreEntries(); 1891371SN/A 1901371SN/A /** Returns the number of loads in the LQ. */ 1912623SN/A int numLoads() { return loads; } 1922623SN/A 1931371SN/A /** Returns the number of stores in the SQ. */ 194581SN/A int numStores() { return stores; } 1952SN/A 1962SN/A /** Returns if either the LQ or SQ is full. */ 1972SN/A bool isFull() { return lqFull() || sqFull(); } 1982SN/A 199753SN/A /** Returns if both the LQ and SQ are empty. */ 2002SN/A bool isEmpty() const { return lqEmpty() && sqEmpty(); } 2012SN/A 2022SN/A /** Returns if the LQ is full. */ 203594SN/A bool lqFull() { return loads >= (LQEntries - 1); } 204595SN/A 205594SN/A /** Returns if the SQ is full. */ 206595SN/A bool sqFull() { return stores >= (SQEntries - 1); } 207705SN/A 208726SN/A /** Returns if the LQ is empty. */ 209726SN/A bool lqEmpty() const { return loads == 0; } 210726SN/A 211726SN/A /** Returns if the SQ is empty. */ 212726SN/A bool sqEmpty() const { return stores == 0; } 213726SN/A 214726SN/A /** Returns the number of instructions in the LSQ. */ 215726SN/A unsigned getCount() { return loads + stores; } 216726SN/A 217726SN/A /** Returns if there are any stores to writeback. */ 218705SN/A bool hasStoresToWB() { return storesToWB; } 2192107SN/A 220726SN/A /** Returns the number of stores to writeback. */ 2212683Sktlim@umich.edu int numStoresToWB() { return storesToWB; } 222726SN/A 223705SN/A /** Returns if the LSQ unit will writeback on this cycle. */ 2242455SN/A bool willWB() { return storeQueue[storeWBIdx].canWB && 225726SN/A !storeQueue[storeWBIdx].completed && 226726SN/A !isStoreBlocked; } 2272683Sktlim@umich.edu 228726SN/A /** Handles doing the retry. */ 229705SN/A void recvRetry(); 2302455SN/A 231726SN/A private: 232726SN/A /** Reset the LSQ state */ 2332683Sktlim@umich.edu void resetState(); 234726SN/A 235705SN/A /** Writes back the instruction, sending it to IEW. */ 2362455SN/A void writeback(DynInstPtr &inst, PacketPtr pkt); 237726SN/A 238726SN/A /** Writes back a store that couldn't be completed the previous cycle. */ 2392683Sktlim@umich.edu void writebackPendingStore(); 2402455SN/A 2412455SN/A /** Handles completing the send of a store to memory. */ 2422455SN/A void storePostSend(PacketPtr pkt); 2432455SN/A 2442455SN/A /** Completes the store at the specified index. */ 2452683Sktlim@umich.edu void completeStore(int store_idx); 246726SN/A 247705SN/A /** Attempts to send a store to the cache. */ 2482107SN/A bool sendStore(PacketPtr data_pkt); 249726SN/A 2502683Sktlim@umich.edu /** Increments the given store index (circular queue). */ 251726SN/A inline void incrStIdx(int &store_idx) const; 252705SN/A /** Decrements the given store index (circular queue). */ 2532455SN/A inline void decrStIdx(int &store_idx) const; 254726SN/A /** Increments the given load index (circular queue). */ 255726SN/A inline void incrLdIdx(int &load_idx) const; 2562683Sktlim@umich.edu /** Decrements the given load index (circular queue). */ 257726SN/A inline void decrLdIdx(int &load_idx) const; 258705SN/A 2592455SN/A public: 260726SN/A /** Debugging function to dump instructions in the LSQ. */ 261726SN/A void dumpInsts() const; 2622683Sktlim@umich.edu 263726SN/A private: 264726SN/A /** Pointer to the CPU. */ 2652455SN/A O3CPU *cpu; 2662577SN/A 267726SN/A /** Pointer to the IEW stage. */ 268726SN/A IEW *iewStage; 2692683Sktlim@umich.edu 2702455SN/A /** Pointer to the LSQ. */ 2712455SN/A LSQ *lsq; 2722455SN/A 2732455SN/A /** Pointer to the dcache port. Used only for sending. */ 2742455SN/A MasterPort *dcachePort; 2752683Sktlim@umich.edu 276726SN/A /** Derived class to hold any sender state the LSQ needs. */ 277705SN/A class LSQSenderState : public Packet::SenderState 2782683Sktlim@umich.edu { 2792683Sktlim@umich.edu public: 2802683Sktlim@umich.edu /** Default constructor. */ 2812447SN/A LSQSenderState() 2822683Sktlim@umich.edu : mainPkt(NULL), pendingPacket(NULL), idx(0), outstanding(1), 2832683Sktlim@umich.edu isLoad(false), noWB(false), isSplit(false), 2842683Sktlim@umich.edu pktToSend(false), cacheBlocked(false) 285705SN/A { } 2862159SN/A 2872159SN/A /** Instruction who initiated the access to memory. */ 2882683Sktlim@umich.edu DynInstPtr inst; 2892159SN/A /** The main packet from a split load, used during writeback. */ 290705SN/A PacketPtr mainPkt; 2913468Sgblack@eecs.umich.edu /** A second packet from a split store that needs sending. */ 2922159SN/A PacketPtr pendingPacket; 2933468Sgblack@eecs.umich.edu /** The LQ/SQ index of the instruction. */ 2942159SN/A uint8_t idx; 2952159SN/A /** Number of outstanding packets to complete. */ 2963468Sgblack@eecs.umich.edu uint8_t outstanding; 2972159SN/A /** Whether or not it is a load. */ 2982683Sktlim@umich.edu bool isLoad; 2992159SN/A /** Whether or not the instruction will need to writeback. */ 3002159SN/A bool noWB; 3013468Sgblack@eecs.umich.edu /** Whether or not this access is split in two. */ 3022159SN/A bool isSplit; 3032683Sktlim@umich.edu /** Whether or not there is a packet that needs sending. */ 3042159SN/A bool pktToSend; 305705SN/A /** Whether or not the second packet of this split load was blocked */ 3061858SN/A bool cacheBlocked; 3072683Sktlim@umich.edu 3082683Sktlim@umich.edu /** Completes a packet and returns whether the access is finished. */ 3092680Sktlim@umich.edu inline bool complete() { return --outstanding == 0; } 3102683Sktlim@umich.edu }; 311705SN/A 3122683Sktlim@umich.edu /** Writeback event, specifically for when stores forward data to loads. */ 313705SN/A class WritebackEvent : public Event { 314705SN/A public: 3152683Sktlim@umich.edu /** Constructs a writeback event. */ 3162680Sktlim@umich.edu WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 3172SN/A 3182SN/A /** Processes the writeback event. */ 3192623SN/A void process(); 320 321 /** Returns the description of this event. */ 322 const char *description() const; 323 324 private: 325 /** Instruction whose results are being written back. */ 326 DynInstPtr inst; 327 328 /** The packet that would have been sent to memory. */ 329 PacketPtr pkt; 330 331 /** The pointer to the LSQ unit that issued the store. */ 332 LSQUnit<Impl> *lsqPtr; 333 }; 334 335 public: 336 struct SQEntry { 337 /** Constructs an empty store queue entry. */ 338 SQEntry() 339 : inst(NULL), req(NULL), size(0), 340 canWB(0), committed(0), completed(0) 341 { 342 std::memset(data, 0, sizeof(data)); 343 } 344 345 ~SQEntry() 346 { 347 inst = NULL; 348 } 349 350 /** Constructs a store queue entry for a given instruction. */ 351 SQEntry(DynInstPtr &_inst) 352 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 353 isSplit(0), canWB(0), committed(0), completed(0), isAllZeros(0) 354 { 355 std::memset(data, 0, sizeof(data)); 356 } 357 /** The store data. */ 358 char data[16]; 359 /** The store instruction. */ 360 DynInstPtr inst; 361 /** The request for the store. */ 362 RequestPtr req; 363 /** The split requests for the store. */ 364 RequestPtr sreqLow; 365 RequestPtr sreqHigh; 366 /** The size of the store. */ 367 uint8_t size; 368 /** Whether or not the store is split into two requests. */ 369 bool isSplit; 370 /** Whether or not the store can writeback. */ 371 bool canWB; 372 /** Whether or not the store is committed. */ 373 bool committed; 374 /** Whether or not the store is completed. */ 375 bool completed; 376 /** Does this request write all zeros and thus doesn't 377 * have any data attached to it. Used for cache block zero 378 * style instructs (ARM DC ZVA; ALPHA WH64) 379 */ 380 bool isAllZeros; 381 }; 382 383 private: 384 /** The LSQUnit thread id. */ 385 ThreadID lsqID; 386 387 /** The store queue. */ 388 std::vector<SQEntry> storeQueue; 389 390 /** The load queue. */ 391 std::vector<DynInstPtr> loadQueue; 392 393 /** The number of LQ entries, plus a sentinel entry (circular queue). 394 * @todo: Consider having var that records the true number of LQ entries. 395 */ 396 unsigned LQEntries; 397 /** The number of SQ entries, plus a sentinel entry (circular queue). 398 * @todo: Consider having var that records the true number of SQ entries. 399 */ 400 unsigned SQEntries; 401 402 /** The number of places to shift addresses in the LSQ before checking 403 * for dependency violations 404 */ 405 unsigned depCheckShift; 406 407 /** Should loads be checked for dependency issues */ 408 bool checkLoads; 409 410 /** The number of load instructions in the LQ. */ 411 int loads; 412 /** The number of store instructions in the SQ. */ 413 int stores; 414 /** The number of store instructions in the SQ waiting to writeback. */ 415 int storesToWB; 416 417 /** The index of the head instruction in the LQ. */ 418 int loadHead; 419 /** The index of the tail instruction in the LQ. */ 420 int loadTail; 421 422 /** The index of the head instruction in the SQ. */ 423 int storeHead; 424 /** The index of the first instruction that may be ready to be 425 * written back, and has not yet been written back. 426 */ 427 int storeWBIdx; 428 /** The index of the tail instruction in the SQ. */ 429 int storeTail; 430 431 /// @todo Consider moving to a more advanced model with write vs read ports 432 /** The number of cache ports available each cycle. */ 433 int cachePorts; 434 435 /** The number of used cache ports in this cycle. */ 436 int usedPorts; 437 438 //list<InstSeqNum> mshrSeqNums; 439 440 /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */ 441 Addr cacheBlockMask; 442 443 /** Wire to read information from the issue stage time queue. */ 444 typename TimeBuffer<IssueStruct>::wire fromIssue; 445 446 /** Whether or not the LSQ is stalled. */ 447 bool stalled; 448 /** The store that causes the stall due to partial store to load 449 * forwarding. 450 */ 451 InstSeqNum stallingStoreIsn; 452 /** The index of the above store. */ 453 int stallingLoadIdx; 454 455 /** The packet that needs to be retried. */ 456 PacketPtr retryPkt; 457 458 /** Whehter or not a store is blocked due to the memory system. */ 459 bool isStoreBlocked; 460 461 /** Whether or not a store is in flight. */ 462 bool storeInFlight; 463 464 /** The oldest load that caused a memory ordering violation. */ 465 DynInstPtr memDepViolator; 466 467 /** Whether or not there is a packet that couldn't be sent because of 468 * a lack of cache ports. */ 469 bool hasPendingPkt; 470 471 /** The packet that is pending free cache ports. */ 472 PacketPtr pendingPkt; 473 474 /** Flag for memory model. */ 475 bool needsTSO; 476 477 // Will also need how many read/write ports the Dcache has. Or keep track 478 // of that in stage that is one level up, and only call executeLoad/Store 479 // the appropriate number of times. 480 /** Total number of loads forwaded from LSQ stores. */ 481 Stats::Scalar lsqForwLoads; 482 483 /** Total number of loads ignored due to invalid addresses. */ 484 Stats::Scalar invAddrLoads; 485 486 /** Total number of squashed loads. */ 487 Stats::Scalar lsqSquashedLoads; 488 489 /** Total number of responses from the memory system that are 490 * ignored due to the instruction already being squashed. */ 491 Stats::Scalar lsqIgnoredResponses; 492 493 /** Tota number of memory ordering violations. */ 494 Stats::Scalar lsqMemOrderViolation; 495 496 /** Total number of squashed stores. */ 497 Stats::Scalar lsqSquashedStores; 498 499 /** Total number of software prefetches ignored due to invalid addresses. */ 500 Stats::Scalar invAddrSwpfs; 501 502 /** Ready loads blocked due to partial store-forwarding. */ 503 Stats::Scalar lsqBlockedLoads; 504 505 /** Number of loads that were rescheduled. */ 506 Stats::Scalar lsqRescheduledLoads; 507 508 /** Number of times the LSQ is blocked due to the cache. */ 509 Stats::Scalar lsqCacheBlocked; 510 511 public: 512 /** Executes the load at the given index. */ 513 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 514 int load_idx); 515 516 /** Executes the store at the given index. */ 517 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 518 uint8_t *data, int store_idx); 519 520 /** Returns the index of the head load instruction. */ 521 int getLoadHead() { return loadHead; } 522 /** Returns the sequence number of the head load instruction. */ 523 InstSeqNum getLoadHeadSeqNum() 524 { 525 if (loadQueue[loadHead]) { 526 return loadQueue[loadHead]->seqNum; 527 } else { 528 return 0; 529 } 530 531 } 532 533 /** Returns the index of the head store instruction. */ 534 int getStoreHead() { return storeHead; } 535 /** Returns the sequence number of the head store instruction. */ 536 InstSeqNum getStoreHeadSeqNum() 537 { 538 if (storeQueue[storeHead].inst) { 539 return storeQueue[storeHead].inst->seqNum; 540 } else { 541 return 0; 542 } 543 544 } 545 546 /** Returns whether or not the LSQ unit is stalled. */ 547 bool isStalled() { return stalled; } 548}; 549 550template <class Impl> 551Fault 552LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 553 int load_idx) 554{ 555 DynInstPtr load_inst = loadQueue[load_idx]; 556 557 assert(load_inst); 558 559 assert(!load_inst->isExecuted()); 560 561 // Make sure this isn't a strictly ordered load 562 // A bit of a hackish way to get strictly ordered accesses to work 563 // only if they're at the head of the LSQ and are ready to commit 564 // (at the head of the ROB too). 565 if (req->isStrictlyOrdered() && 566 (load_idx != loadHead || !load_inst->isAtCommit())) { 567 iewStage->rescheduleMemInst(load_inst); 568 ++lsqRescheduledLoads; 569 DPRINTF(LSQUnit, "Strictly ordered load [sn:%lli] PC %s\n", 570 load_inst->seqNum, load_inst->pcState()); 571 572 // Must delete request now that it wasn't handed off to 573 // memory. This is quite ugly. @todo: Figure out the proper 574 // place to really handle request deletes. 575 delete req; 576 if (TheISA::HasUnalignedMemAcc && sreqLow) { 577 delete sreqLow; 578 delete sreqHigh; 579 } 580 return std::make_shared<GenericISA::M5PanicFault>( 581 "Strictly ordered load [sn:%llx] PC %s\n", 582 load_inst->seqNum, load_inst->pcState()); 583 } 584 585 // Check the SQ for any previous stores that might lead to forwarding 586 int store_idx = load_inst->sqIdx; 587 588 int store_size = 0; 589 590 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 591 "storeHead: %i addr: %#x%s\n", 592 load_idx, store_idx, storeHead, req->getPaddr(), 593 sreqLow ? " split" : ""); 594 595 if (req->isLLSC()) { 596 assert(!sreqLow); 597 // Disable recording the result temporarily. Writing to misc 598 // regs normally updates the result, but this is not the 599 // desired behavior when handling store conditionals. 600 load_inst->recordResult(false); 601 TheISA::handleLockedRead(load_inst.get(), req); 602 load_inst->recordResult(true); 603 } 604 605 if (req->isMmappedIpr()) { 606 assert(!load_inst->memData); 607 load_inst->memData = new uint8_t[64]; 608 609 ThreadContext *thread = cpu->tcBase(lsqID); 610 Cycles delay(0); 611 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 612 613 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 614 data_pkt->dataStatic(load_inst->memData); 615 delay = TheISA::handleIprRead(thread, data_pkt); 616 } else { 617 assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr()); 618 PacketPtr fst_data_pkt = new Packet(sreqLow, MemCmd::ReadReq); 619 PacketPtr snd_data_pkt = new Packet(sreqHigh, MemCmd::ReadReq); 620 621 fst_data_pkt->dataStatic(load_inst->memData); 622 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 623 624 delay = TheISA::handleIprRead(thread, fst_data_pkt); 625 Cycles delay2 = TheISA::handleIprRead(thread, snd_data_pkt); 626 if (delay2 > delay) 627 delay = delay2; 628 629 delete sreqLow; 630 delete sreqHigh; 631 delete fst_data_pkt; 632 delete snd_data_pkt; 633 } 634 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 635 cpu->schedule(wb, cpu->clockEdge(delay)); 636 return NoFault; 637 } 638 639 while (store_idx != -1) { 640 // End once we've reached the top of the LSQ 641 if (store_idx == storeWBIdx) { 642 break; 643 } 644 645 // Move the index to one younger 646 if (--store_idx < 0) 647 store_idx += SQEntries; 648 649 assert(storeQueue[store_idx].inst); 650 651 store_size = storeQueue[store_idx].size; 652 653 if (store_size == 0) 654 continue; 655 else if (storeQueue[store_idx].inst->strictlyOrdered()) 656 continue; 657 658 assert(storeQueue[store_idx].inst->effAddrValid()); 659 660 // Check if the store data is within the lower and upper bounds of 661 // addresses that the request needs. 662 bool store_has_lower_limit = 663 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 664 bool store_has_upper_limit = 665 (req->getVaddr() + req->getSize()) <= 666 (storeQueue[store_idx].inst->effAddr + store_size); 667 bool lower_load_has_store_part = 668 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 669 store_size); 670 bool upper_load_has_store_part = 671 (req->getVaddr() + req->getSize()) > 672 storeQueue[store_idx].inst->effAddr; 673 674 // If the store's data has all of the data needed, we can forward. 675 if ((store_has_lower_limit && store_has_upper_limit)) { 676 // Get shift amount for offset into the store's data. 677 int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr; 678 679 // Allocate memory if this is the first time a load is issued. 680 if (!load_inst->memData) { 681 load_inst->memData = new uint8_t[req->getSize()]; 682 } 683 if (storeQueue[store_idx].isAllZeros) 684 memset(load_inst->memData, 0, req->getSize()); 685 else 686 memcpy(load_inst->memData, 687 storeQueue[store_idx].data + shift_amt, req->getSize()); 688 689 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 690 "addr %#x\n", store_idx, req->getVaddr()); 691 692 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 693 data_pkt->dataStatic(load_inst->memData); 694 695 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 696 697 // We'll say this has a 1 cycle load-store forwarding latency 698 // for now. 699 // @todo: Need to make this a parameter. 700 cpu->schedule(wb, curTick()); 701 702 // Don't need to do anything special for split loads. 703 if (TheISA::HasUnalignedMemAcc && sreqLow) { 704 delete sreqLow; 705 delete sreqHigh; 706 } 707 708 ++lsqForwLoads; 709 return NoFault; 710 } else if ((store_has_lower_limit && lower_load_has_store_part) || 711 (store_has_upper_limit && upper_load_has_store_part) || 712 (lower_load_has_store_part && upper_load_has_store_part)) { 713 // This is the partial store-load forwarding case where a store 714 // has only part of the load's data. 715 716 // If it's already been written back, then don't worry about 717 // stalling on it. 718 if (storeQueue[store_idx].completed) { 719 panic("Should not check one of these"); 720 continue; 721 } 722 723 // Must stall load and force it to retry, so long as it's the oldest 724 // load that needs to do so. 725 if (!stalled || 726 (stalled && 727 load_inst->seqNum < 728 loadQueue[stallingLoadIdx]->seqNum)) { 729 stalled = true; 730 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 731 stallingLoadIdx = load_idx; 732 } 733 734 // Tell IQ/mem dep unit that this instruction will need to be 735 // rescheduled eventually 736 iewStage->rescheduleMemInst(load_inst); 737 load_inst->clearIssued(); 738 ++lsqRescheduledLoads; 739 740 // Do not generate a writeback event as this instruction is not 741 // complete. 742 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 743 "Store idx %i to load addr %#x\n", 744 store_idx, req->getVaddr()); 745 746 // Must delete request now that it wasn't handed off to 747 // memory. This is quite ugly. @todo: Figure out the 748 // proper place to really handle request deletes. 749 delete req; 750 if (TheISA::HasUnalignedMemAcc && sreqLow) { 751 delete sreqLow; 752 delete sreqHigh; 753 } 754 755 return NoFault; 756 } 757 } 758 759 // If there's no forwarding case, then go access memory 760 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 761 load_inst->seqNum, load_inst->pcState()); 762 763 // Allocate memory if this is the first time a load is issued. 764 if (!load_inst->memData) { 765 load_inst->memData = new uint8_t[req->getSize()]; 766 } 767 768 ++usedPorts; 769 770 // if we the cache is not blocked, do cache access 771 bool completedFirst = false; 772 PacketPtr data_pkt = Packet::createRead(req); 773 PacketPtr fst_data_pkt = NULL; 774 PacketPtr snd_data_pkt = NULL; 775 776 data_pkt->dataStatic(load_inst->memData); 777 778 LSQSenderState *state = new LSQSenderState; 779 state->isLoad = true; 780 state->idx = load_idx; 781 state->inst = load_inst; 782 data_pkt->senderState = state; 783 784 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 785 // Point the first packet at the main data packet. 786 fst_data_pkt = data_pkt; 787 } else { 788 // Create the split packets. 789 fst_data_pkt = Packet::createRead(sreqLow); 790 snd_data_pkt = Packet::createRead(sreqHigh); 791 792 fst_data_pkt->dataStatic(load_inst->memData); 793 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 794 795 fst_data_pkt->senderState = state; 796 snd_data_pkt->senderState = state; 797 798 state->isSplit = true; 799 state->outstanding = 2; 800 state->mainPkt = data_pkt; 801 } 802 803 bool successful_load = true; 804 if (!dcachePort->sendTimingReq(fst_data_pkt)) { 805 successful_load = false; 806 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 807 completedFirst = true; 808 809 // The first packet was sent without problems, so send this one 810 // too. If there is a problem with this packet then the whole 811 // load will be squashed, so indicate this to the state object. 812 // The first packet will return in completeDataAccess and be 813 // handled there. 814 ++usedPorts; 815 if (!dcachePort->sendTimingReq(snd_data_pkt)) { 816 // The main packet will be deleted in completeDataAccess. 817 state->complete(); 818 // Signify to 1st half that the 2nd half was blocked via state 819 state->cacheBlocked = true; 820 successful_load = false; 821 } 822 } 823 824 // If the cache was blocked, or has become blocked due to the access, 825 // handle it. 826 if (!successful_load) { 827 if (!sreqLow) { 828 // Packet wasn't split, just delete main packet info 829 delete state; 830 delete req; 831 delete data_pkt; 832 } 833 834 if (TheISA::HasUnalignedMemAcc && sreqLow) { 835 if (!completedFirst) { 836 // Split packet, but first failed. Delete all state. 837 delete state; 838 delete req; 839 delete data_pkt; 840 delete fst_data_pkt; 841 delete snd_data_pkt; 842 delete sreqLow; 843 delete sreqHigh; 844 sreqLow = NULL; 845 sreqHigh = NULL; 846 } else { 847 // Can't delete main packet data or state because first packet 848 // was sent to the memory system 849 delete data_pkt; 850 delete req; 851 delete sreqHigh; 852 delete snd_data_pkt; 853 sreqHigh = NULL; 854 } 855 } 856 857 ++lsqCacheBlocked; 858 859 iewStage->blockMemInst(load_inst); 860 861 // No fault occurred, even though the interface is blocked. 862 return NoFault; 863 } 864 865 return NoFault; 866} 867 868template <class Impl> 869Fault 870LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 871 uint8_t *data, int store_idx) 872{ 873 assert(storeQueue[store_idx].inst); 874 875 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x" 876 " | storeHead:%i [sn:%i]\n", 877 store_idx, req->getPaddr(), storeHead, 878 storeQueue[store_idx].inst->seqNum); 879 880 storeQueue[store_idx].req = req; 881 storeQueue[store_idx].sreqLow = sreqLow; 882 storeQueue[store_idx].sreqHigh = sreqHigh; 883 unsigned size = req->getSize(); 884 storeQueue[store_idx].size = size; 885 storeQueue[store_idx].isAllZeros = req->getFlags() & Request::CACHE_BLOCK_ZERO; 886 assert(size <= sizeof(storeQueue[store_idx].data) || 887 (req->getFlags() & Request::CACHE_BLOCK_ZERO)); 888 889 // Split stores can only occur in ISAs with unaligned memory accesses. If 890 // a store request has been split, sreqLow and sreqHigh will be non-null. 891 if (TheISA::HasUnalignedMemAcc && sreqLow) { 892 storeQueue[store_idx].isSplit = true; 893 } 894 895 if (!(req->getFlags() & Request::CACHE_BLOCK_ZERO)) 896 memcpy(storeQueue[store_idx].data, data, size); 897 898 // This function only writes the data to the store queue, so no fault 899 // can happen here. 900 return NoFault; 901} 902 903#endif // __CPU_O3_LSQ_UNIT_HH__ 904