lsq_unit.hh revision 10239
11689SN/A/* 210330Smitch.hayenga@arm.com * Copyright (c) 2012-2013 ARM Limited 38842Smrinmoy.ghosh@arm.com * All rights reserved 48842Smrinmoy.ghosh@arm.com * 58842Smrinmoy.ghosh@arm.com * The license below extends only to copyright in the software and shall 68842Smrinmoy.ghosh@arm.com * not be construed as granting a license to any other intellectual 78842Smrinmoy.ghosh@arm.com * property including but not limited to intellectual property relating 88842Smrinmoy.ghosh@arm.com * to a hardware implementation of the functionality of the software 98842Smrinmoy.ghosh@arm.com * licensed hereunder. You may use the software subject to the license 108842Smrinmoy.ghosh@arm.com * terms below provided that you ensure that this notice is replicated 118842Smrinmoy.ghosh@arm.com * unmodified and in its entirety in all distributions of the software, 128842Smrinmoy.ghosh@arm.com * modified or unmodified, in source code or in binary form. 138842Smrinmoy.ghosh@arm.com * 142345SN/A * Copyright (c) 2004-2006 The Regents of The University of Michigan 151689SN/A * Copyright (c) 2013 Advanced Micro Devices, Inc. 161689SN/A * All rights reserved. 171689SN/A * 181689SN/A * Redistribution and use in source and binary forms, with or without 191689SN/A * modification, are permitted provided that the following conditions are 201689SN/A * met: redistributions of source code must retain the above copyright 211689SN/A * notice, this list of conditions and the following disclaimer; 221689SN/A * redistributions in binary form must reproduce the above copyright 231689SN/A * notice, this list of conditions and the following disclaimer in the 241689SN/A * documentation and/or other materials provided with the distribution; 251689SN/A * neither the name of the copyright holders nor the names of its 261689SN/A * contributors may be used to endorse or promote products derived from 271689SN/A * this software without specific prior written permission. 281689SN/A * 291689SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 301689SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 311689SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 321689SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 331689SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 341689SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 351689SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 361689SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 371689SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 381689SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 392665SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 402665SN/A * 419480Snilay@cs.wisc.edu * Authors: Kevin Lim 429480Snilay@cs.wisc.edu * Korey Sewell 431689SN/A */ 441689SN/A 459480Snilay@cs.wisc.edu#ifndef __CPU_O3_LSQ_UNIT_HH__ 469480Snilay@cs.wisc.edu#define __CPU_O3_LSQ_UNIT_HH__ 471062SN/A 486216SN/A#include <algorithm> 496216SN/A#include <cstring> 506216SN/A#include <map> 519480Snilay@cs.wisc.edu#include <queue> 529480Snilay@cs.wisc.edu 531062SN/A#include "arch/generic/debugfaults.hh" 542345SN/A#include "arch/isa_traits.hh" 552345SN/A#include "arch/locked_mem.hh" 562345SN/A#include "arch/mmapped_ipr.hh" 572345SN/A#include "base/hashmap.hh" 582345SN/A#include "config/the_isa.hh" 592345SN/A#include "cpu/inst_seq.hh" 602345SN/A#include "cpu/timebuf.hh" 612345SN/A#include "debug/LSQUnit.hh" 622345SN/A#include "mem/packet.hh" 639480Snilay@cs.wisc.edu#include "mem/port.hh" 641062SN/A#include "sim/fault_fwd.hh" 651062SN/A 661062SN/Astruct DerivO3CPUParams; 671062SN/A 681062SN/A/** 699480Snilay@cs.wisc.edu * Class that implements the actual LQ and SQ for each specific 701062SN/A * thread. Both are circular queues; load entries are freed upon 711062SN/A * committing, while store entries are freed once they writeback. The 721062SN/A * LSQUnit tracks if there are memory ordering violations, and also 732345SN/A * detects partial load to store forwarding cases (a store only has 742345SN/A * part of a load's data) that requires the load to wait until the 751062SN/A * store writes back. In the former case it holds onto the instruction 762345SN/A * until the dependence unit looks at it, and in the latter it stalls 771062SN/A * the LSQ until the store writes back. At that point the load is 781062SN/A * replayed. 799480Snilay@cs.wisc.edu */ 802345SN/Atemplate <class Impl> 812345SN/Aclass LSQUnit { 822345SN/A public: 832345SN/A typedef typename Impl::O3CPU O3CPU; 842345SN/A typedef typename Impl::DynInstPtr DynInstPtr; 852345SN/A typedef typename Impl::CPUPol::IEW IEW; 862345SN/A typedef typename Impl::CPUPol::LSQ LSQ; 879480Snilay@cs.wisc.edu typedef typename Impl::CPUPol::IssueStruct IssueStruct; 888842Smrinmoy.ghosh@arm.com 898842Smrinmoy.ghosh@arm.com public: 908842Smrinmoy.ghosh@arm.com /** Constructs an LSQ unit. init() must be called prior to use. */ 918842Smrinmoy.ghosh@arm.com LSQUnit(); 928842Smrinmoy.ghosh@arm.com 938842Smrinmoy.ghosh@arm.com /** Initializes the LSQ unit with the specified number of entries. */ 948842Smrinmoy.ghosh@arm.com void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 959480Snilay@cs.wisc.edu LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 961062SN/A unsigned id); 971062SN/A 981062SN/A /** Returns the name of the LSQ unit. */ 991062SN/A std::string name() const; 1002345SN/A 1012345SN/A /** Registers statistics. */ 1028842Smrinmoy.ghosh@arm.com void regStats(); 1038842Smrinmoy.ghosh@arm.com 1041062SN/A /** Sets the pointer to the dcache port. */ 1059480Snilay@cs.wisc.edu void setDcachePort(MasterPort *dcache_port); 1061062SN/A 10710330Smitch.hayenga@arm.com /** Perform sanity checks after a drain. */ 10810330Smitch.hayenga@arm.com void drainSanityCheck() const; 1092345SN/A 1102345SN/A /** Takes over from another CPU's thread. */ 1112345SN/A void takeOverFrom(); 1122345SN/A 1132345SN/A /** Ticks the LSQ unit, which in this case only resets the number of 1142345SN/A * used cache ports. 1152345SN/A * @todo: Move the number of used ports up to the LSQ level so it can 1162345SN/A * be shared by all LSQ units. 1171684SN/A */ 1181062SN/A void tick() { usedPorts = 0; } 1191062SN/A 1202345SN/A /** Inserts an instruction. */ 1212345SN/A void insert(DynInstPtr &inst); 1222345SN/A /** Inserts a load instruction. */ 1232345SN/A void insertLoad(DynInstPtr &load_inst); 1242345SN/A /** Inserts a store instruction. */ 1251062SN/A void insertStore(DynInstPtr &store_inst); 1261062SN/A 1272345SN/A /** Check for ordering violations in the LSQ. For a store squash if we 1282345SN/A * ever find a conflicting load. For a load, only squash if we 1292345SN/A * an external snoop invalidate has been seen for that load address 1302345SN/A * @param load_idx index to start checking at 1311062SN/A * @param inst the instruction to check 1321062SN/A */ 1332345SN/A Fault checkViolations(int load_idx, DynInstPtr &inst); 1342345SN/A 1351062SN/A /** Check if an incoming invalidate hits in the lsq on a load 1362345SN/A * that might have issued out of order wrt another load beacuse 1372345SN/A * of the intermediate invalidate. 1382345SN/A */ 1392345SN/A void checkSnoop(PacketPtr pkt); 1402345SN/A 1412345SN/A /** Executes a load instruction. */ 1422345SN/A Fault executeLoad(DynInstPtr &inst); 1432345SN/A 1442345SN/A Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 1452345SN/A /** Executes a store instruction. */ 1462345SN/A Fault executeStore(DynInstPtr &inst); 1472345SN/A 1482345SN/A /** Commits the head load. */ 1492345SN/A void commitLoad(); 1502345SN/A /** Commits loads older than a specific sequence number. */ 1512345SN/A void commitLoads(InstSeqNum &youngest_inst); 1522345SN/A 1532345SN/A /** Commits stores older than a specific sequence number. */ 1542345SN/A void commitStores(InstSeqNum &youngest_inst); 1552345SN/A 1562345SN/A /** Writes back stores. */ 1572345SN/A void writebackStores(); 1582345SN/A 1592345SN/A /** Completes the data access that has been returned from the 1602345SN/A * memory system. */ 1612345SN/A void completeDataAccess(PacketPtr pkt); 1622345SN/A 1632345SN/A /** Clears all the entries in the LQ. */ 1642345SN/A void clearLQ(); 1652345SN/A 1662345SN/A /** Clears all the entries in the SQ. */ 1672345SN/A void clearSQ(); 1682345SN/A 1698842Smrinmoy.ghosh@arm.com /** Resizes the LQ to a given size. */ 1702345SN/A void resizeLQ(unsigned size); 1712345SN/A 1722345SN/A /** Resizes the SQ to a given size. */ 1732345SN/A void resizeSQ(unsigned size); 1741062SN/A 1758842Smrinmoy.ghosh@arm.com /** Squashes all instructions younger than a specific sequence number. */ 1768842Smrinmoy.ghosh@arm.com void squash(const InstSeqNum &squashed_num); 1771062SN/A 1782292SN/A /** Returns if there is a memory ordering violation. Value is reset upon 1791062SN/A * call to getMemDepViolator(). 1809360SE.Tomusk@sms.ed.ac.uk */ 1811684SN/A bool violation() { return memDepViolator; } 1821062SN/A 1839360SE.Tomusk@sms.ed.ac.uk /** Returns the memory ordering violator. */ 1842356SN/A DynInstPtr getMemDepViolator(); 1852356SN/A 1861062SN/A /** Returns if a load became blocked due to the memory system. */ 1871684SN/A bool loadBlocked() 1881062SN/A { return isLoadBlocked; } 1891062SN/A 1902292SN/A /** Clears the signal that a load became blocked. */ 1911062SN/A void clearLoadBlocked() 1929360SE.Tomusk@sms.ed.ac.uk { isLoadBlocked = false; } 1931684SN/A 1941062SN/A /** Returns if the blocked load was handled. */ 1959360SE.Tomusk@sms.ed.ac.uk bool isLoadBlockedHandled() 1961684SN/A { return loadBlockedHandled; } 1971062SN/A 1981062SN/A /** Records the blocked load as being handled. */ 1992292SN/A void setLoadBlockedHandled() 2001062SN/A { loadBlockedHandled = true; } 2019360SE.Tomusk@sms.ed.ac.uk 2021684SN/A /** Returns the number of free LQ entries. */ 2031062SN/A unsigned numFreeLoadEntries(); 2041062SN/A 2051684SN/A /** Returns the number of free SQ entries. */ 2061062SN/A unsigned numFreeStoreEntries(); 2079360SE.Tomusk@sms.ed.ac.uk 2089360SE.Tomusk@sms.ed.ac.uk /** Returns the number of loads in the LQ. */ 2099360SE.Tomusk@sms.ed.ac.uk int numLoads() { return loads; } 2101684SN/A 2111062SN/A /** Returns the number of stores in the SQ. */ 2129360SE.Tomusk@sms.ed.ac.uk int numStores() { return stores; } 2139360SE.Tomusk@sms.ed.ac.uk 2141684SN/A /** Returns if either the LQ or SQ is full. */ 2151062SN/A bool isFull() { return lqFull() || sqFull(); } 2169360SE.Tomusk@sms.ed.ac.uk 2179360SE.Tomusk@sms.ed.ac.uk /** Returns if both the LQ and SQ are empty. */ 2181062SN/A bool isEmpty() const { return lqEmpty() && sqEmpty(); } 2191062SN/A 2209360SE.Tomusk@sms.ed.ac.uk /** Returns if the LQ is full. */ 2219360SE.Tomusk@sms.ed.ac.uk bool lqFull() { return loads >= (LQEntries - 1); } 2229360SE.Tomusk@sms.ed.ac.uk 2239360SE.Tomusk@sms.ed.ac.uk /** Returns if the SQ is full. */ 2249360SE.Tomusk@sms.ed.ac.uk bool sqFull() { return stores >= (SQEntries - 1); } 2259360SE.Tomusk@sms.ed.ac.uk 2269360SE.Tomusk@sms.ed.ac.uk /** Returns if the LQ is empty. */ 2279360SE.Tomusk@sms.ed.ac.uk bool lqEmpty() const { return loads == 0; } 2281062SN/A 2292292SN/A /** Returns if the SQ is empty. */ 2301062SN/A bool sqEmpty() const { return stores == 0; } 2319360SE.Tomusk@sms.ed.ac.uk 2321684SN/A /** Returns the number of instructions in the LSQ. */ 2331062SN/A unsigned getCount() { return loads + stores; } 2349360SE.Tomusk@sms.ed.ac.uk 2351684SN/A /** Returns if there are any stores to writeback. */ 2361062SN/A bool hasStoresToWB() { return storesToWB; } 2371062SN/A 2381062SN/A /** Returns the number of stores to writeback. */ 2391062SN/A int numStoresToWB() { return storesToWB; } 2401062SN/A 2411062SN/A /** Returns if the LSQ unit will writeback on this cycle. */ 2429360SE.Tomusk@sms.ed.ac.uk bool willWB() { return storeQueue[storeWBIdx].canWB && 2431062SN/A !storeQueue[storeWBIdx].completed && 2441062SN/A !isStoreBlocked; } 2459360SE.Tomusk@sms.ed.ac.uk 2469360SE.Tomusk@sms.ed.ac.uk /** Handles doing the retry. */ 2479360SE.Tomusk@sms.ed.ac.uk void recvRetry(); 2481062SN/A 2491062SN/A private: 2509480Snilay@cs.wisc.edu /** Reset the LSQ state */ 251 void resetState(); 252 253 /** Writes back the instruction, sending it to IEW. */ 254 void writeback(DynInstPtr &inst, PacketPtr pkt); 255 256 /** Writes back a store that couldn't be completed the previous cycle. */ 257 void writebackPendingStore(); 258 259 /** Handles completing the send of a store to memory. */ 260 void storePostSend(PacketPtr pkt); 261 262 /** Completes the store at the specified index. */ 263 void completeStore(int store_idx); 264 265 /** Attempts to send a store to the cache. */ 266 bool sendStore(PacketPtr data_pkt); 267 268 /** Increments the given store index (circular queue). */ 269 inline void incrStIdx(int &store_idx) const; 270 /** Decrements the given store index (circular queue). */ 271 inline void decrStIdx(int &store_idx) const; 272 /** Increments the given load index (circular queue). */ 273 inline void incrLdIdx(int &load_idx) const; 274 /** Decrements the given load index (circular queue). */ 275 inline void decrLdIdx(int &load_idx) const; 276 277 public: 278 /** Debugging function to dump instructions in the LSQ. */ 279 void dumpInsts() const; 280 281 private: 282 /** Pointer to the CPU. */ 283 O3CPU *cpu; 284 285 /** Pointer to the IEW stage. */ 286 IEW *iewStage; 287 288 /** Pointer to the LSQ. */ 289 LSQ *lsq; 290 291 /** Pointer to the dcache port. Used only for sending. */ 292 MasterPort *dcachePort; 293 294 /** Derived class to hold any sender state the LSQ needs. */ 295 class LSQSenderState : public Packet::SenderState 296 { 297 public: 298 /** Default constructor. */ 299 LSQSenderState() 300 : mainPkt(NULL), pendingPacket(NULL), outstanding(1), 301 noWB(false), isSplit(false), pktToSend(false) 302 { } 303 304 /** Instruction who initiated the access to memory. */ 305 DynInstPtr inst; 306 /** The main packet from a split load, used during writeback. */ 307 PacketPtr mainPkt; 308 /** A second packet from a split store that needs sending. */ 309 PacketPtr pendingPacket; 310 /** The LQ/SQ index of the instruction. */ 311 uint8_t idx; 312 /** Number of outstanding packets to complete. */ 313 uint8_t outstanding; 314 /** Whether or not it is a load. */ 315 bool isLoad; 316 /** Whether or not the instruction will need to writeback. */ 317 bool noWB; 318 /** Whether or not this access is split in two. */ 319 bool isSplit; 320 /** Whether or not there is a packet that needs sending. */ 321 bool pktToSend; 322 323 /** Completes a packet and returns whether the access is finished. */ 324 inline bool complete() { return --outstanding == 0; } 325 }; 326 327 /** Writeback event, specifically for when stores forward data to loads. */ 328 class WritebackEvent : public Event { 329 public: 330 /** Constructs a writeback event. */ 331 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 332 333 /** Processes the writeback event. */ 334 void process(); 335 336 /** Returns the description of this event. */ 337 const char *description() const; 338 339 private: 340 /** Instruction whose results are being written back. */ 341 DynInstPtr inst; 342 343 /** The packet that would have been sent to memory. */ 344 PacketPtr pkt; 345 346 /** The pointer to the LSQ unit that issued the store. */ 347 LSQUnit<Impl> *lsqPtr; 348 }; 349 350 public: 351 struct SQEntry { 352 /** Constructs an empty store queue entry. */ 353 SQEntry() 354 : inst(NULL), req(NULL), size(0), 355 canWB(0), committed(0), completed(0) 356 { 357 std::memset(data, 0, sizeof(data)); 358 } 359 360 ~SQEntry() 361 { 362 inst = NULL; 363 } 364 365 /** Constructs a store queue entry for a given instruction. */ 366 SQEntry(DynInstPtr &_inst) 367 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 368 isSplit(0), canWB(0), committed(0), completed(0), isAllZeros(0) 369 { 370 std::memset(data, 0, sizeof(data)); 371 } 372 /** The store data. */ 373 char data[16]; 374 /** The store instruction. */ 375 DynInstPtr inst; 376 /** The request for the store. */ 377 RequestPtr req; 378 /** The split requests for the store. */ 379 RequestPtr sreqLow; 380 RequestPtr sreqHigh; 381 /** The size of the store. */ 382 uint8_t size; 383 /** Whether or not the store is split into two requests. */ 384 bool isSplit; 385 /** Whether or not the store can writeback. */ 386 bool canWB; 387 /** Whether or not the store is committed. */ 388 bool committed; 389 /** Whether or not the store is completed. */ 390 bool completed; 391 /** Does this request write all zeros and thus doesn't 392 * have any data attached to it. Used for cache block zero 393 * style instructs (ARM DC ZVA; ALPHA WH64) 394 */ 395 bool isAllZeros; 396 }; 397 398 private: 399 /** The LSQUnit thread id. */ 400 ThreadID lsqID; 401 402 /** The store queue. */ 403 std::vector<SQEntry> storeQueue; 404 405 /** The load queue. */ 406 std::vector<DynInstPtr> loadQueue; 407 408 /** The number of LQ entries, plus a sentinel entry (circular queue). 409 * @todo: Consider having var that records the true number of LQ entries. 410 */ 411 unsigned LQEntries; 412 /** The number of SQ entries, plus a sentinel entry (circular queue). 413 * @todo: Consider having var that records the true number of SQ entries. 414 */ 415 unsigned SQEntries; 416 417 /** The number of places to shift addresses in the LSQ before checking 418 * for dependency violations 419 */ 420 unsigned depCheckShift; 421 422 /** Should loads be checked for dependency issues */ 423 bool checkLoads; 424 425 /** The number of load instructions in the LQ. */ 426 int loads; 427 /** The number of store instructions in the SQ. */ 428 int stores; 429 /** The number of store instructions in the SQ waiting to writeback. */ 430 int storesToWB; 431 432 /** The index of the head instruction in the LQ. */ 433 int loadHead; 434 /** The index of the tail instruction in the LQ. */ 435 int loadTail; 436 437 /** The index of the head instruction in the SQ. */ 438 int storeHead; 439 /** The index of the first instruction that may be ready to be 440 * written back, and has not yet been written back. 441 */ 442 int storeWBIdx; 443 /** The index of the tail instruction in the SQ. */ 444 int storeTail; 445 446 /// @todo Consider moving to a more advanced model with write vs read ports 447 /** The number of cache ports available each cycle. */ 448 int cachePorts; 449 450 /** The number of used cache ports in this cycle. */ 451 int usedPorts; 452 453 //list<InstSeqNum> mshrSeqNums; 454 455 /** Address Mask for a cache block (e.g. ~(cache_block_size-1)) */ 456 Addr cacheBlockMask; 457 458 /** Wire to read information from the issue stage time queue. */ 459 typename TimeBuffer<IssueStruct>::wire fromIssue; 460 461 /** Whether or not the LSQ is stalled. */ 462 bool stalled; 463 /** The store that causes the stall due to partial store to load 464 * forwarding. 465 */ 466 InstSeqNum stallingStoreIsn; 467 /** The index of the above store. */ 468 int stallingLoadIdx; 469 470 /** The packet that needs to be retried. */ 471 PacketPtr retryPkt; 472 473 /** Whehter or not a store is blocked due to the memory system. */ 474 bool isStoreBlocked; 475 476 /** Whether or not a load is blocked due to the memory system. */ 477 bool isLoadBlocked; 478 479 /** Has the blocked load been handled. */ 480 bool loadBlockedHandled; 481 482 /** Whether or not a store is in flight. */ 483 bool storeInFlight; 484 485 /** The sequence number of the blocked load. */ 486 InstSeqNum blockedLoadSeqNum; 487 488 /** The oldest load that caused a memory ordering violation. */ 489 DynInstPtr memDepViolator; 490 491 /** Whether or not there is a packet that couldn't be sent because of 492 * a lack of cache ports. */ 493 bool hasPendingPkt; 494 495 /** The packet that is pending free cache ports. */ 496 PacketPtr pendingPkt; 497 498 /** Flag for memory model. */ 499 bool needsTSO; 500 501 // Will also need how many read/write ports the Dcache has. Or keep track 502 // of that in stage that is one level up, and only call executeLoad/Store 503 // the appropriate number of times. 504 /** Total number of loads forwaded from LSQ stores. */ 505 Stats::Scalar lsqForwLoads; 506 507 /** Total number of loads ignored due to invalid addresses. */ 508 Stats::Scalar invAddrLoads; 509 510 /** Total number of squashed loads. */ 511 Stats::Scalar lsqSquashedLoads; 512 513 /** Total number of responses from the memory system that are 514 * ignored due to the instruction already being squashed. */ 515 Stats::Scalar lsqIgnoredResponses; 516 517 /** Tota number of memory ordering violations. */ 518 Stats::Scalar lsqMemOrderViolation; 519 520 /** Total number of squashed stores. */ 521 Stats::Scalar lsqSquashedStores; 522 523 /** Total number of software prefetches ignored due to invalid addresses. */ 524 Stats::Scalar invAddrSwpfs; 525 526 /** Ready loads blocked due to partial store-forwarding. */ 527 Stats::Scalar lsqBlockedLoads; 528 529 /** Number of loads that were rescheduled. */ 530 Stats::Scalar lsqRescheduledLoads; 531 532 /** Number of times the LSQ is blocked due to the cache. */ 533 Stats::Scalar lsqCacheBlocked; 534 535 public: 536 /** Executes the load at the given index. */ 537 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, 538 uint8_t *data, int load_idx); 539 540 /** Executes the store at the given index. */ 541 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, 542 uint8_t *data, int store_idx); 543 544 /** Returns the index of the head load instruction. */ 545 int getLoadHead() { return loadHead; } 546 /** Returns the sequence number of the head load instruction. */ 547 InstSeqNum getLoadHeadSeqNum() 548 { 549 if (loadQueue[loadHead]) { 550 return loadQueue[loadHead]->seqNum; 551 } else { 552 return 0; 553 } 554 555 } 556 557 /** Returns the index of the head store instruction. */ 558 int getStoreHead() { return storeHead; } 559 /** Returns the sequence number of the head store instruction. */ 560 InstSeqNum getStoreHeadSeqNum() 561 { 562 if (storeQueue[storeHead].inst) { 563 return storeQueue[storeHead].inst->seqNum; 564 } else { 565 return 0; 566 } 567 568 } 569 570 /** Returns whether or not the LSQ unit is stalled. */ 571 bool isStalled() { return stalled; } 572}; 573 574template <class Impl> 575Fault 576LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 577 uint8_t *data, int load_idx) 578{ 579 DynInstPtr load_inst = loadQueue[load_idx]; 580 581 assert(load_inst); 582 583 assert(!load_inst->isExecuted()); 584 585 // Make sure this isn't an uncacheable access 586 // A bit of a hackish way to get uncached accesses to work only if they're 587 // at the head of the LSQ and are ready to commit (at the head of the ROB 588 // too). 589 if (req->isUncacheable() && 590 (load_idx != loadHead || !load_inst->isAtCommit())) { 591 iewStage->rescheduleMemInst(load_inst); 592 ++lsqRescheduledLoads; 593 DPRINTF(LSQUnit, "Uncachable load [sn:%lli] PC %s\n", 594 load_inst->seqNum, load_inst->pcState()); 595 596 // Must delete request now that it wasn't handed off to 597 // memory. This is quite ugly. @todo: Figure out the proper 598 // place to really handle request deletes. 599 delete req; 600 if (TheISA::HasUnalignedMemAcc && sreqLow) { 601 delete sreqLow; 602 delete sreqHigh; 603 } 604 return new GenericISA::M5PanicFault( 605 "Uncachable load [sn:%llx] PC %s\n", 606 load_inst->seqNum, load_inst->pcState()); 607 } 608 609 // Check the SQ for any previous stores that might lead to forwarding 610 int store_idx = load_inst->sqIdx; 611 612 int store_size = 0; 613 614 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 615 "storeHead: %i addr: %#x%s\n", 616 load_idx, store_idx, storeHead, req->getPaddr(), 617 sreqLow ? " split" : ""); 618 619 if (req->isLLSC()) { 620 assert(!sreqLow); 621 // Disable recording the result temporarily. Writing to misc 622 // regs normally updates the result, but this is not the 623 // desired behavior when handling store conditionals. 624 load_inst->recordResult(false); 625 TheISA::handleLockedRead(load_inst.get(), req); 626 load_inst->recordResult(true); 627 } 628 629 if (req->isMmappedIpr()) { 630 assert(!load_inst->memData); 631 load_inst->memData = new uint8_t[64]; 632 633 ThreadContext *thread = cpu->tcBase(lsqID); 634 Cycles delay(0); 635 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 636 637 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 638 data_pkt->dataStatic(load_inst->memData); 639 delay = TheISA::handleIprRead(thread, data_pkt); 640 } else { 641 assert(sreqLow->isMmappedIpr() && sreqHigh->isMmappedIpr()); 642 PacketPtr fst_data_pkt = new Packet(sreqLow, MemCmd::ReadReq); 643 PacketPtr snd_data_pkt = new Packet(sreqHigh, MemCmd::ReadReq); 644 645 fst_data_pkt->dataStatic(load_inst->memData); 646 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 647 648 delay = TheISA::handleIprRead(thread, fst_data_pkt); 649 Cycles delay2 = TheISA::handleIprRead(thread, snd_data_pkt); 650 if (delay2 > delay) 651 delay = delay2; 652 653 delete sreqLow; 654 delete sreqHigh; 655 delete fst_data_pkt; 656 delete snd_data_pkt; 657 } 658 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 659 cpu->schedule(wb, cpu->clockEdge(delay)); 660 return NoFault; 661 } 662 663 while (store_idx != -1) { 664 // End once we've reached the top of the LSQ 665 if (store_idx == storeWBIdx) { 666 break; 667 } 668 669 // Move the index to one younger 670 if (--store_idx < 0) 671 store_idx += SQEntries; 672 673 assert(storeQueue[store_idx].inst); 674 675 store_size = storeQueue[store_idx].size; 676 677 if (store_size == 0) 678 continue; 679 else if (storeQueue[store_idx].inst->uncacheable()) 680 continue; 681 682 assert(storeQueue[store_idx].inst->effAddrValid()); 683 684 // Check if the store data is within the lower and upper bounds of 685 // addresses that the request needs. 686 bool store_has_lower_limit = 687 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 688 bool store_has_upper_limit = 689 (req->getVaddr() + req->getSize()) <= 690 (storeQueue[store_idx].inst->effAddr + store_size); 691 bool lower_load_has_store_part = 692 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 693 store_size); 694 bool upper_load_has_store_part = 695 (req->getVaddr() + req->getSize()) > 696 storeQueue[store_idx].inst->effAddr; 697 698 // If the store's data has all of the data needed, we can forward. 699 if ((store_has_lower_limit && store_has_upper_limit)) { 700 // Get shift amount for offset into the store's data. 701 int shift_amt = req->getVaddr() - storeQueue[store_idx].inst->effAddr; 702 703 if (storeQueue[store_idx].isAllZeros) 704 memset(data, 0, req->getSize()); 705 else 706 memcpy(data, storeQueue[store_idx].data + shift_amt, 707 req->getSize()); 708 709 assert(!load_inst->memData); 710 load_inst->memData = new uint8_t[req->getSize()]; 711 if (storeQueue[store_idx].isAllZeros) 712 memset(load_inst->memData, 0, req->getSize()); 713 else 714 memcpy(load_inst->memData, 715 storeQueue[store_idx].data + shift_amt, req->getSize()); 716 717 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 718 "addr %#x\n", store_idx, req->getVaddr()); 719 720 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq); 721 data_pkt->dataStatic(load_inst->memData); 722 723 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 724 725 // We'll say this has a 1 cycle load-store forwarding latency 726 // for now. 727 // @todo: Need to make this a parameter. 728 cpu->schedule(wb, curTick()); 729 730 // Don't need to do anything special for split loads. 731 if (TheISA::HasUnalignedMemAcc && sreqLow) { 732 delete sreqLow; 733 delete sreqHigh; 734 } 735 736 ++lsqForwLoads; 737 return NoFault; 738 } else if ((store_has_lower_limit && lower_load_has_store_part) || 739 (store_has_upper_limit && upper_load_has_store_part) || 740 (lower_load_has_store_part && upper_load_has_store_part)) { 741 // This is the partial store-load forwarding case where a store 742 // has only part of the load's data. 743 744 // If it's already been written back, then don't worry about 745 // stalling on it. 746 if (storeQueue[store_idx].completed) { 747 panic("Should not check one of these"); 748 continue; 749 } 750 751 // Must stall load and force it to retry, so long as it's the oldest 752 // load that needs to do so. 753 if (!stalled || 754 (stalled && 755 load_inst->seqNum < 756 loadQueue[stallingLoadIdx]->seqNum)) { 757 stalled = true; 758 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 759 stallingLoadIdx = load_idx; 760 } 761 762 // Tell IQ/mem dep unit that this instruction will need to be 763 // rescheduled eventually 764 iewStage->rescheduleMemInst(load_inst); 765 iewStage->decrWb(load_inst->seqNum); 766 load_inst->clearIssued(); 767 ++lsqRescheduledLoads; 768 769 // Do not generate a writeback event as this instruction is not 770 // complete. 771 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 772 "Store idx %i to load addr %#x\n", 773 store_idx, req->getVaddr()); 774 775 // Must delete request now that it wasn't handed off to 776 // memory. This is quite ugly. @todo: Figure out the 777 // proper place to really handle request deletes. 778 delete req; 779 if (TheISA::HasUnalignedMemAcc && sreqLow) { 780 delete sreqLow; 781 delete sreqHigh; 782 } 783 784 return NoFault; 785 } 786 } 787 788 // If there's no forwarding case, then go access memory 789 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %s\n", 790 load_inst->seqNum, load_inst->pcState()); 791 792 assert(!load_inst->memData); 793 load_inst->memData = new uint8_t[req->getSize()]; 794 795 ++usedPorts; 796 797 // if we the cache is not blocked, do cache access 798 bool completedFirst = false; 799 if (!lsq->cacheBlocked()) { 800 MemCmd command = 801 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 802 PacketPtr data_pkt = new Packet(req, command); 803 PacketPtr fst_data_pkt = NULL; 804 PacketPtr snd_data_pkt = NULL; 805 806 data_pkt->dataStatic(load_inst->memData); 807 808 LSQSenderState *state = new LSQSenderState; 809 state->isLoad = true; 810 state->idx = load_idx; 811 state->inst = load_inst; 812 data_pkt->senderState = state; 813 814 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 815 816 // Point the first packet at the main data packet. 817 fst_data_pkt = data_pkt; 818 } else { 819 820 // Create the split packets. 821 fst_data_pkt = new Packet(sreqLow, command); 822 snd_data_pkt = new Packet(sreqHigh, command); 823 824 fst_data_pkt->dataStatic(load_inst->memData); 825 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 826 827 fst_data_pkt->senderState = state; 828 snd_data_pkt->senderState = state; 829 830 state->isSplit = true; 831 state->outstanding = 2; 832 state->mainPkt = data_pkt; 833 } 834 835 if (!dcachePort->sendTimingReq(fst_data_pkt)) { 836 // Delete state and data packet because a load retry 837 // initiates a pipeline restart; it does not retry. 838 delete state; 839 delete data_pkt->req; 840 delete data_pkt; 841 if (TheISA::HasUnalignedMemAcc && sreqLow) { 842 delete fst_data_pkt->req; 843 delete fst_data_pkt; 844 delete snd_data_pkt->req; 845 delete snd_data_pkt; 846 sreqLow = NULL; 847 sreqHigh = NULL; 848 } 849 850 req = NULL; 851 852 // If the access didn't succeed, tell the LSQ by setting 853 // the retry thread id. 854 lsq->setRetryTid(lsqID); 855 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 856 completedFirst = true; 857 858 // The first packet was sent without problems, so send this one 859 // too. If there is a problem with this packet then the whole 860 // load will be squashed, so indicate this to the state object. 861 // The first packet will return in completeDataAccess and be 862 // handled there. 863 ++usedPorts; 864 if (!dcachePort->sendTimingReq(snd_data_pkt)) { 865 866 // The main packet will be deleted in completeDataAccess. 867 delete snd_data_pkt->req; 868 delete snd_data_pkt; 869 870 state->complete(); 871 872 req = NULL; 873 sreqHigh = NULL; 874 875 lsq->setRetryTid(lsqID); 876 } 877 } 878 } 879 880 // If the cache was blocked, or has become blocked due to the access, 881 // handle it. 882 if (lsq->cacheBlocked()) { 883 if (req) 884 delete req; 885 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 886 delete sreqLow; 887 delete sreqHigh; 888 } 889 890 ++lsqCacheBlocked; 891 892 // If the first part of a split access succeeds, then let the LSQ 893 // handle the decrWb when completeDataAccess is called upon return 894 // of the requested first part of data 895 if (!completedFirst) 896 iewStage->decrWb(load_inst->seqNum); 897 898 // There's an older load that's already going to squash. 899 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 900 return NoFault; 901 902 // Record that the load was blocked due to memory. This 903 // load will squash all instructions after it, be 904 // refetched, and re-executed. 905 isLoadBlocked = true; 906 loadBlockedHandled = false; 907 blockedLoadSeqNum = load_inst->seqNum; 908 // No fault occurred, even though the interface is blocked. 909 return NoFault; 910 } 911 912 return NoFault; 913} 914 915template <class Impl> 916Fault 917LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 918 uint8_t *data, int store_idx) 919{ 920 assert(storeQueue[store_idx].inst); 921 922 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x" 923 " | storeHead:%i [sn:%i]\n", 924 store_idx, req->getPaddr(), storeHead, 925 storeQueue[store_idx].inst->seqNum); 926 927 storeQueue[store_idx].req = req; 928 storeQueue[store_idx].sreqLow = sreqLow; 929 storeQueue[store_idx].sreqHigh = sreqHigh; 930 unsigned size = req->getSize(); 931 storeQueue[store_idx].size = size; 932 storeQueue[store_idx].isAllZeros = req->getFlags() & Request::CACHE_BLOCK_ZERO; 933 assert(size <= sizeof(storeQueue[store_idx].data) || 934 (req->getFlags() & Request::CACHE_BLOCK_ZERO)); 935 936 // Split stores can only occur in ISAs with unaligned memory accesses. If 937 // a store request has been split, sreqLow and sreqHigh will be non-null. 938 if (TheISA::HasUnalignedMemAcc && sreqLow) { 939 storeQueue[store_idx].isSplit = true; 940 } 941 942 if (!(req->getFlags() & Request::CACHE_BLOCK_ZERO)) 943 memcpy(storeQueue[store_idx].data, data, size); 944 945 // This function only writes the data to the store queue, so no fault 946 // can happen here. 947 return NoFault; 948} 949 950#endif // __CPU_O3_LSQ_UNIT_HH__ 951