lsq_unit.hh revision 7511:bd104adbf04d
12036SN/A/* 22036SN/A * Copyright (c) 2004-2006 The Regents of The University of Michigan 32036SN/A * All rights reserved. 42036SN/A * 52036SN/A * Redistribution and use in source and binary forms, with or without 62036SN/A * modification, are permitted provided that the following conditions are 72036SN/A * met: redistributions of source code must retain the above copyright 82036SN/A * notice, this list of conditions and the following disclaimer; 92036SN/A * redistributions in binary form must reproduce the above copyright 102036SN/A * notice, this list of conditions and the following disclaimer in the 112036SN/A * documentation and/or other materials provided with the distribution; 122036SN/A * neither the name of the copyright holders nor the names of its 132036SN/A * contributors may be used to endorse or promote products derived from 142036SN/A * this software without specific prior written permission. 152036SN/A * 162036SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 172036SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 182036SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 192036SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 202036SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 212036SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 222036SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 232036SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 242036SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 252036SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 262036SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 272665Ssaidi@eecs.umich.edu * 282956Sgblack@eecs.umich.edu * Authors: Kevin Lim 292956Sgblack@eecs.umich.edu * Korey Sewell 302772Ssaidi@eecs.umich.edu */ 312036SN/A 322036SN/A#ifndef __CPU_O3_LSQ_UNIT_HH__ 332036SN/A#define __CPU_O3_LSQ_UNIT_HH__ 342036SN/A 352036SN/A#include <algorithm> 362036SN/A#include <cstring> 372036SN/A#include <map> 382036SN/A#include <queue> 392036SN/A 404176Sgblack@eecs.umich.edu#include "arch/faults.hh" 412779Sbinkertn@umich.edu#include "arch/locked_mem.hh" 426214Snate@binkert.org#include "config/full_system.hh" 432036SN/A#include "config/the_isa.hh" 442036SN/A#include "base/fast_alloc.hh" 452036SN/A#include "base/hashmap.hh" 462036SN/A#include "cpu/inst_seq.hh" 472565SN/A#include "mem/packet.hh" 482565SN/A#include "mem/port.hh" 492565SN/A 502565SN/Aclass DerivO3CPUParams; 513918Ssaidi@eecs.umich.edu 523483Ssaidi@eecs.umich.edu/** 532036SN/A * Class that implements the actual LQ and SQ for each specific 542036SN/A * thread. Both are circular queues; load entries are freed upon 552036SN/A * committing, while store entries are freed once they writeback. The 562036SN/A * LSQUnit tracks if there are memory ordering violations, and also 572778Ssaidi@eecs.umich.edu * detects partial load to store forwarding cases (a store only has 582778Ssaidi@eecs.umich.edu * part of a load's data) that requires the load to wait until the 592778Ssaidi@eecs.umich.edu * store writes back. In the former case it holds onto the instruction 602778Ssaidi@eecs.umich.edu * until the dependence unit looks at it, and in the latter it stalls 613799Sgblack@eecs.umich.edu * the LSQ until the store writes back. At that point the load is 623799Sgblack@eecs.umich.edu * replayed. 632036SN/A */ 642036SN/Atemplate <class Impl> 655549Snate@binkert.orgclass LSQUnit { 662036SN/A protected: 672036SN/A typedef TheISA::IntReg IntReg; 682565SN/A public: 692565SN/A typedef typename Impl::O3CPU O3CPU; 702778Ssaidi@eecs.umich.edu typedef typename Impl::DynInstPtr DynInstPtr; 712778Ssaidi@eecs.umich.edu typedef typename Impl::CPUPol::IEW IEW; 722565SN/A typedef typename Impl::CPUPol::LSQ LSQ; 732036SN/A typedef typename Impl::CPUPol::IssueStruct IssueStruct; 742036SN/A 752036SN/A public: 762036SN/A /** Constructs an LSQ unit. init() must be called prior to use. */ 772036SN/A LSQUnit(); 782036SN/A 792036SN/A /** Initializes the LSQ unit with the specified number of entries. */ 802036SN/A void init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params, 812565SN/A LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries, 822036SN/A unsigned id); 832036SN/A 845549Snate@binkert.org /** Returns the name of the LSQ unit. */ 852036SN/A std::string name() const; 862036SN/A 872565SN/A /** Registers statistics. */ 882565SN/A void regStats(); 892778Ssaidi@eecs.umich.edu 902778Ssaidi@eecs.umich.edu /** Sets the pointer to the dcache port. */ 912565SN/A void setDcachePort(Port *dcache_port); 922036SN/A 932036SN/A /** Switches out LSQ unit. */ 942036SN/A void switchOut(); 952565SN/A 962036SN/A /** Takes over from another CPU's thread. */ 972036SN/A void takeOverFrom(); 985549Snate@binkert.org 992036SN/A /** Returns if the LSQ is switched out. */ 1002036SN/A bool isSwitchedOut() { return switchedOut; } 1012565SN/A 1022565SN/A /** Ticks the LSQ unit, which in this case only resets the number of 1032778Ssaidi@eecs.umich.edu * used cache ports. 1042778Ssaidi@eecs.umich.edu * @todo: Move the number of used ports up to the LSQ level so it can 1052565SN/A * be shared by all LSQ units. 1062036SN/A */ 1072036SN/A void tick() { usedPorts = 0; } 1082565SN/A 1092036SN/A /** Inserts an instruction. */ 1102036SN/A void insert(DynInstPtr &inst); 1112764Sstever@eecs.umich.edu /** Inserts a load instruction. */ 1122764Sstever@eecs.umich.edu void insertLoad(DynInstPtr &load_inst); 1134176Sgblack@eecs.umich.edu /** Inserts a store instruction. */ 1142764Sstever@eecs.umich.edu void insertStore(DynInstPtr &store_inst); 1152764Sstever@eecs.umich.edu 1165549Snate@binkert.org /** Executes a load instruction. */ 1172764Sstever@eecs.umich.edu Fault executeLoad(DynInstPtr &inst); 1182764Sstever@eecs.umich.edu 1192764Sstever@eecs.umich.edu Fault executeLoad(int lq_idx) { panic("Not implemented"); return NoFault; } 1202764Sstever@eecs.umich.edu /** Executes a store instruction. */ 1212764Sstever@eecs.umich.edu Fault executeStore(DynInstPtr &inst); 1222764Sstever@eecs.umich.edu 1232764Sstever@eecs.umich.edu /** Commits the head load. */ 1242764Sstever@eecs.umich.edu void commitLoad(); 1252764Sstever@eecs.umich.edu /** Commits loads older than a specific sequence number. */ 1262764Sstever@eecs.umich.edu void commitLoads(InstSeqNum &youngest_inst); 1272764Sstever@eecs.umich.edu 1282036SN/A /** Commits stores older than a specific sequence number. */ 1294176Sgblack@eecs.umich.edu void commitStores(InstSeqNum &youngest_inst); 1305549Snate@binkert.org 1314176Sgblack@eecs.umich.edu /** Writes back stores. */ 1324176Sgblack@eecs.umich.edu void writebackStores(); 1334176Sgblack@eecs.umich.edu 1344176Sgblack@eecs.umich.edu /** Completes the data access that has been returned from the 1354176Sgblack@eecs.umich.edu * memory system. */ 1364176Sgblack@eecs.umich.edu void completeDataAccess(PacketPtr pkt); 1374176Sgblack@eecs.umich.edu 1385549Snate@binkert.org /** Clears all the entries in the LQ. */ 1394176Sgblack@eecs.umich.edu void clearLQ(); 1404176Sgblack@eecs.umich.edu 1414176Sgblack@eecs.umich.edu /** Clears all the entries in the SQ. */ 1424176Sgblack@eecs.umich.edu void clearSQ(); 1434176Sgblack@eecs.umich.edu 1444176Sgblack@eecs.umich.edu /** Resizes the LQ to a given size. */ 1452036SN/A void resizeLQ(unsigned size); 1462036SN/A 1475549Snate@binkert.org /** Resizes the SQ to a given size. */ 1485549Snate@binkert.org void resizeSQ(unsigned size); 1492036SN/A 1502036SN/A /** Squashes all instructions younger than a specific sequence number. */ 1512036SN/A void squash(const InstSeqNum &squashed_num); 1523927Ssaidi@eecs.umich.edu 1533799Sgblack@eecs.umich.edu /** Returns if there is a memory ordering violation. Value is reset upon 1545549Snate@binkert.org * call to getMemDepViolator(). 1555549Snate@binkert.org */ 1565549Snate@binkert.org bool violation() { return memDepViolator; } 1575549Snate@binkert.org 1583483Ssaidi@eecs.umich.edu /** Returns the memory ordering violator. */ 1593799Sgblack@eecs.umich.edu DynInstPtr getMemDepViolator(); 1605549Snate@binkert.org 1615549Snate@binkert.org /** Returns if a load became blocked due to the memory system. */ 1625549Snate@binkert.org bool loadBlocked() 1635549Snate@binkert.org { return isLoadBlocked; } 1642036SN/A 1652036SN/A /** Clears the signal that a load became blocked. */ 1662036SN/A void clearLoadBlocked() 1672036SN/A { isLoadBlocked = false; } 1682036SN/A 1692036SN/A /** Returns if the blocked load was handled. */ 1703799Sgblack@eecs.umich.edu bool isLoadBlockedHandled() 1713799Sgblack@eecs.umich.edu { return loadBlockedHandled; } 1725549Snate@binkert.org 1733799Sgblack@eecs.umich.edu /** Records the blocked load as being handled. */ 1745549Snate@binkert.org void setLoadBlockedHandled() 1753799Sgblack@eecs.umich.edu { loadBlockedHandled = true; } 1765549Snate@binkert.org 1773799Sgblack@eecs.umich.edu /** Returns the number of free entries (min of free LQ and SQ entries). */ 1785549Snate@binkert.org unsigned numFreeEntries(); 1793799Sgblack@eecs.umich.edu 1805549Snate@binkert.org /** Returns the number of loads ready to execute. */ 1813799Sgblack@eecs.umich.edu int numLoadsReady(); 1825549Snate@binkert.org 1832036SN/A /** Returns the number of loads in the LQ. */ 1842036SN/A int numLoads() { return loads; } 1852036SN/A 1862036SN/A /** Returns the number of stores in the SQ. */ 1873799Sgblack@eecs.umich.edu int numStores() { return stores; } 1883799Sgblack@eecs.umich.edu 1895549Snate@binkert.org /** Returns if either the LQ or SQ is full. */ 1903799Sgblack@eecs.umich.edu bool isFull() { return lqFull() || sqFull(); } 1915549Snate@binkert.org 1923799Sgblack@eecs.umich.edu /** Returns if the LQ is full. */ 1935549Snate@binkert.org bool lqFull() { return loads >= (LQEntries - 1); } 1943799Sgblack@eecs.umich.edu 1955549Snate@binkert.org /** Returns if the SQ is full. */ 1963799Sgblack@eecs.umich.edu bool sqFull() { return stores >= (SQEntries - 1); } 1975549Snate@binkert.org 1983799Sgblack@eecs.umich.edu /** Returns the number of instructions in the LSQ. */ 1995549Snate@binkert.org unsigned getCount() { return loads + stores; } 2002036SN/A 2012036SN/A /** Returns if there are any stores to writeback. */ 202 bool hasStoresToWB() { return storesToWB; } 203 204 /** Returns the number of stores to writeback. */ 205 int numStoresToWB() { return storesToWB; } 206 207 /** Returns if the LSQ unit will writeback on this cycle. */ 208 bool willWB() { return storeQueue[storeWBIdx].canWB && 209 !storeQueue[storeWBIdx].completed && 210 !isStoreBlocked; } 211 212 /** Handles doing the retry. */ 213 void recvRetry(); 214 215 private: 216 /** Writes back the instruction, sending it to IEW. */ 217 void writeback(DynInstPtr &inst, PacketPtr pkt); 218 219 /** Writes back a store that couldn't be completed the previous cycle. */ 220 void writebackPendingStore(); 221 222 /** Handles completing the send of a store to memory. */ 223 void storePostSend(PacketPtr pkt); 224 225 /** Completes the store at the specified index. */ 226 void completeStore(int store_idx); 227 228 /** Attempts to send a store to the cache. */ 229 bool sendStore(PacketPtr data_pkt); 230 231 /** Increments the given store index (circular queue). */ 232 inline void incrStIdx(int &store_idx); 233 /** Decrements the given store index (circular queue). */ 234 inline void decrStIdx(int &store_idx); 235 /** Increments the given load index (circular queue). */ 236 inline void incrLdIdx(int &load_idx); 237 /** Decrements the given load index (circular queue). */ 238 inline void decrLdIdx(int &load_idx); 239 240 public: 241 /** Debugging function to dump instructions in the LSQ. */ 242 void dumpInsts(); 243 244 private: 245 /** Pointer to the CPU. */ 246 O3CPU *cpu; 247 248 /** Pointer to the IEW stage. */ 249 IEW *iewStage; 250 251 /** Pointer to the LSQ. */ 252 LSQ *lsq; 253 254 /** Pointer to the dcache port. Used only for sending. */ 255 Port *dcachePort; 256 257 /** Derived class to hold any sender state the LSQ needs. */ 258 class LSQSenderState : public Packet::SenderState, public FastAlloc 259 { 260 public: 261 /** Default constructor. */ 262 LSQSenderState() 263 : noWB(false), isSplit(false), pktToSend(false), outstanding(1), 264 mainPkt(NULL), pendingPacket(NULL) 265 { } 266 267 /** Instruction who initiated the access to memory. */ 268 DynInstPtr inst; 269 /** Whether or not it is a load. */ 270 bool isLoad; 271 /** The LQ/SQ index of the instruction. */ 272 int idx; 273 /** Whether or not the instruction will need to writeback. */ 274 bool noWB; 275 /** Whether or not this access is split in two. */ 276 bool isSplit; 277 /** Whether or not there is a packet that needs sending. */ 278 bool pktToSend; 279 /** Number of outstanding packets to complete. */ 280 int outstanding; 281 /** The main packet from a split load, used during writeback. */ 282 PacketPtr mainPkt; 283 /** A second packet from a split store that needs sending. */ 284 PacketPtr pendingPacket; 285 286 /** Completes a packet and returns whether the access is finished. */ 287 inline bool complete() { return --outstanding == 0; } 288 }; 289 290 /** Writeback event, specifically for when stores forward data to loads. */ 291 class WritebackEvent : public Event { 292 public: 293 /** Constructs a writeback event. */ 294 WritebackEvent(DynInstPtr &_inst, PacketPtr pkt, LSQUnit *lsq_ptr); 295 296 /** Processes the writeback event. */ 297 void process(); 298 299 /** Returns the description of this event. */ 300 const char *description() const; 301 302 private: 303 /** Instruction whose results are being written back. */ 304 DynInstPtr inst; 305 306 /** The packet that would have been sent to memory. */ 307 PacketPtr pkt; 308 309 /** The pointer to the LSQ unit that issued the store. */ 310 LSQUnit<Impl> *lsqPtr; 311 }; 312 313 public: 314 struct SQEntry { 315 /** Constructs an empty store queue entry. */ 316 SQEntry() 317 : inst(NULL), req(NULL), size(0), 318 canWB(0), committed(0), completed(0) 319 { 320 std::memset(data, 0, sizeof(data)); 321 } 322 323 /** Constructs a store queue entry for a given instruction. */ 324 SQEntry(DynInstPtr &_inst) 325 : inst(_inst), req(NULL), sreqLow(NULL), sreqHigh(NULL), size(0), 326 isSplit(0), canWB(0), committed(0), completed(0) 327 { 328 std::memset(data, 0, sizeof(data)); 329 } 330 331 /** The store instruction. */ 332 DynInstPtr inst; 333 /** The request for the store. */ 334 RequestPtr req; 335 /** The split requests for the store. */ 336 RequestPtr sreqLow; 337 RequestPtr sreqHigh; 338 /** The size of the store. */ 339 int size; 340 /** The store data. */ 341 char data[sizeof(IntReg)]; 342 /** Whether or not the store is split into two requests. */ 343 bool isSplit; 344 /** Whether or not the store can writeback. */ 345 bool canWB; 346 /** Whether or not the store is committed. */ 347 bool committed; 348 /** Whether or not the store is completed. */ 349 bool completed; 350 }; 351 352 private: 353 /** The LSQUnit thread id. */ 354 ThreadID lsqID; 355 356 /** The store queue. */ 357 std::vector<SQEntry> storeQueue; 358 359 /** The load queue. */ 360 std::vector<DynInstPtr> loadQueue; 361 362 /** The number of LQ entries, plus a sentinel entry (circular queue). 363 * @todo: Consider having var that records the true number of LQ entries. 364 */ 365 unsigned LQEntries; 366 /** The number of SQ entries, plus a sentinel entry (circular queue). 367 * @todo: Consider having var that records the true number of SQ entries. 368 */ 369 unsigned SQEntries; 370 371 /** The number of load instructions in the LQ. */ 372 int loads; 373 /** The number of store instructions in the SQ. */ 374 int stores; 375 /** The number of store instructions in the SQ waiting to writeback. */ 376 int storesToWB; 377 378 /** The index of the head instruction in the LQ. */ 379 int loadHead; 380 /** The index of the tail instruction in the LQ. */ 381 int loadTail; 382 383 /** The index of the head instruction in the SQ. */ 384 int storeHead; 385 /** The index of the first instruction that may be ready to be 386 * written back, and has not yet been written back. 387 */ 388 int storeWBIdx; 389 /** The index of the tail instruction in the SQ. */ 390 int storeTail; 391 392 /// @todo Consider moving to a more advanced model with write vs read ports 393 /** The number of cache ports available each cycle. */ 394 int cachePorts; 395 396 /** The number of used cache ports in this cycle. */ 397 int usedPorts; 398 399 /** Is the LSQ switched out. */ 400 bool switchedOut; 401 402 //list<InstSeqNum> mshrSeqNums; 403 404 /** Wire to read information from the issue stage time queue. */ 405 typename TimeBuffer<IssueStruct>::wire fromIssue; 406 407 /** Whether or not the LSQ is stalled. */ 408 bool stalled; 409 /** The store that causes the stall due to partial store to load 410 * forwarding. 411 */ 412 InstSeqNum stallingStoreIsn; 413 /** The index of the above store. */ 414 int stallingLoadIdx; 415 416 /** The packet that needs to be retried. */ 417 PacketPtr retryPkt; 418 419 /** Whehter or not a store is blocked due to the memory system. */ 420 bool isStoreBlocked; 421 422 /** Whether or not a load is blocked due to the memory system. */ 423 bool isLoadBlocked; 424 425 /** Has the blocked load been handled. */ 426 bool loadBlockedHandled; 427 428 /** The sequence number of the blocked load. */ 429 InstSeqNum blockedLoadSeqNum; 430 431 /** The oldest load that caused a memory ordering violation. */ 432 DynInstPtr memDepViolator; 433 434 /** Whether or not there is a packet that couldn't be sent because of 435 * a lack of cache ports. */ 436 bool hasPendingPkt; 437 438 /** The packet that is pending free cache ports. */ 439 PacketPtr pendingPkt; 440 441 // Will also need how many read/write ports the Dcache has. Or keep track 442 // of that in stage that is one level up, and only call executeLoad/Store 443 // the appropriate number of times. 444 /** Total number of loads forwaded from LSQ stores. */ 445 Stats::Scalar lsqForwLoads; 446 447 /** Total number of loads ignored due to invalid addresses. */ 448 Stats::Scalar invAddrLoads; 449 450 /** Total number of squashed loads. */ 451 Stats::Scalar lsqSquashedLoads; 452 453 /** Total number of responses from the memory system that are 454 * ignored due to the instruction already being squashed. */ 455 Stats::Scalar lsqIgnoredResponses; 456 457 /** Tota number of memory ordering violations. */ 458 Stats::Scalar lsqMemOrderViolation; 459 460 /** Total number of squashed stores. */ 461 Stats::Scalar lsqSquashedStores; 462 463 /** Total number of software prefetches ignored due to invalid addresses. */ 464 Stats::Scalar invAddrSwpfs; 465 466 /** Ready loads blocked due to partial store-forwarding. */ 467 Stats::Scalar lsqBlockedLoads; 468 469 /** Number of loads that were rescheduled. */ 470 Stats::Scalar lsqRescheduledLoads; 471 472 /** Number of times the LSQ is blocked due to the cache. */ 473 Stats::Scalar lsqCacheBlocked; 474 475 public: 476 /** Executes the load at the given index. */ 477 template <class T> 478 Fault read(Request *req, Request *sreqLow, Request *sreqHigh, T &data, 479 int load_idx); 480 481 /** Executes the store at the given index. */ 482 template <class T> 483 Fault write(Request *req, Request *sreqLow, Request *sreqHigh, T &data, 484 int store_idx); 485 486 /** Returns the index of the head load instruction. */ 487 int getLoadHead() { return loadHead; } 488 /** Returns the sequence number of the head load instruction. */ 489 InstSeqNum getLoadHeadSeqNum() 490 { 491 if (loadQueue[loadHead]) { 492 return loadQueue[loadHead]->seqNum; 493 } else { 494 return 0; 495 } 496 497 } 498 499 /** Returns the index of the head store instruction. */ 500 int getStoreHead() { return storeHead; } 501 /** Returns the sequence number of the head store instruction. */ 502 InstSeqNum getStoreHeadSeqNum() 503 { 504 if (storeQueue[storeHead].inst) { 505 return storeQueue[storeHead].inst->seqNum; 506 } else { 507 return 0; 508 } 509 510 } 511 512 /** Returns whether or not the LSQ unit is stalled. */ 513 bool isStalled() { return stalled; } 514}; 515 516template <class Impl> 517template <class T> 518Fault 519LSQUnit<Impl>::read(Request *req, Request *sreqLow, Request *sreqHigh, 520 T &data, int load_idx) 521{ 522 DynInstPtr load_inst = loadQueue[load_idx]; 523 524 assert(load_inst); 525 526 assert(!load_inst->isExecuted()); 527 528 // Make sure this isn't an uncacheable access 529 // A bit of a hackish way to get uncached accesses to work only if they're 530 // at the head of the LSQ and are ready to commit (at the head of the ROB 531 // too). 532 if (req->isUncacheable() && 533 (load_idx != loadHead || !load_inst->isAtCommit())) { 534 iewStage->rescheduleMemInst(load_inst); 535 ++lsqRescheduledLoads; 536 537 // Must delete request now that it wasn't handed off to 538 // memory. This is quite ugly. @todo: Figure out the proper 539 // place to really handle request deletes. 540 delete req; 541 if (TheISA::HasUnalignedMemAcc && sreqLow) { 542 delete sreqLow; 543 delete sreqHigh; 544 } 545 return TheISA::genMachineCheckFault(); 546 } 547 548 // Check the SQ for any previous stores that might lead to forwarding 549 int store_idx = load_inst->sqIdx; 550 551 int store_size = 0; 552 553 DPRINTF(LSQUnit, "Read called, load idx: %i, store idx: %i, " 554 "storeHead: %i addr: %#x%s\n", 555 load_idx, store_idx, storeHead, req->getPaddr(), 556 sreqLow ? " split" : ""); 557 558 if (req->isLLSC()) { 559 assert(!sreqLow); 560 // Disable recording the result temporarily. Writing to misc 561 // regs normally updates the result, but this is not the 562 // desired behavior when handling store conditionals. 563 load_inst->recordResult = false; 564 TheISA::handleLockedRead(load_inst.get(), req); 565 load_inst->recordResult = true; 566 } 567 568 while (store_idx != -1) { 569 // End once we've reached the top of the LSQ 570 if (store_idx == storeWBIdx) { 571 break; 572 } 573 574 // Move the index to one younger 575 if (--store_idx < 0) 576 store_idx += SQEntries; 577 578 assert(storeQueue[store_idx].inst); 579 580 store_size = storeQueue[store_idx].size; 581 582 if (store_size == 0) 583 continue; 584 else if (storeQueue[store_idx].inst->uncacheable()) 585 continue; 586 587 assert(storeQueue[store_idx].inst->effAddrValid); 588 589 // Check if the store data is within the lower and upper bounds of 590 // addresses that the request needs. 591 bool store_has_lower_limit = 592 req->getVaddr() >= storeQueue[store_idx].inst->effAddr; 593 bool store_has_upper_limit = 594 (req->getVaddr() + req->getSize()) <= 595 (storeQueue[store_idx].inst->effAddr + store_size); 596 bool lower_load_has_store_part = 597 req->getVaddr() < (storeQueue[store_idx].inst->effAddr + 598 store_size); 599 bool upper_load_has_store_part = 600 (req->getVaddr() + req->getSize()) > 601 storeQueue[store_idx].inst->effAddr; 602 603 // If the store's data has all of the data needed, we can forward. 604 if ((store_has_lower_limit && store_has_upper_limit)) { 605 // Get shift amount for offset into the store's data. 606 int shift_amt = req->getVaddr() & (store_size - 1); 607 608 memcpy(&data, storeQueue[store_idx].data + shift_amt, sizeof(T)); 609 610 assert(!load_inst->memData); 611 load_inst->memData = new uint8_t[64]; 612 613 memcpy(load_inst->memData, 614 storeQueue[store_idx].data + shift_amt, req->getSize()); 615 616 DPRINTF(LSQUnit, "Forwarding from store idx %i to load to " 617 "addr %#x, data %#x\n", 618 store_idx, req->getVaddr(), data); 619 620 PacketPtr data_pkt = new Packet(req, MemCmd::ReadReq, 621 Packet::Broadcast); 622 data_pkt->dataStatic(load_inst->memData); 623 624 WritebackEvent *wb = new WritebackEvent(load_inst, data_pkt, this); 625 626 // We'll say this has a 1 cycle load-store forwarding latency 627 // for now. 628 // @todo: Need to make this a parameter. 629 cpu->schedule(wb, curTick); 630 631 // Don't need to do anything special for split loads. 632 if (TheISA::HasUnalignedMemAcc && sreqLow) { 633 delete sreqLow; 634 delete sreqHigh; 635 } 636 637 ++lsqForwLoads; 638 return NoFault; 639 } else if ((store_has_lower_limit && lower_load_has_store_part) || 640 (store_has_upper_limit && upper_load_has_store_part) || 641 (lower_load_has_store_part && upper_load_has_store_part)) { 642 // This is the partial store-load forwarding case where a store 643 // has only part of the load's data. 644 645 // If it's already been written back, then don't worry about 646 // stalling on it. 647 if (storeQueue[store_idx].completed) { 648 panic("Should not check one of these"); 649 continue; 650 } 651 652 // Must stall load and force it to retry, so long as it's the oldest 653 // load that needs to do so. 654 if (!stalled || 655 (stalled && 656 load_inst->seqNum < 657 loadQueue[stallingLoadIdx]->seqNum)) { 658 stalled = true; 659 stallingStoreIsn = storeQueue[store_idx].inst->seqNum; 660 stallingLoadIdx = load_idx; 661 } 662 663 // Tell IQ/mem dep unit that this instruction will need to be 664 // rescheduled eventually 665 iewStage->rescheduleMemInst(load_inst); 666 iewStage->decrWb(load_inst->seqNum); 667 load_inst->clearIssued(); 668 ++lsqRescheduledLoads; 669 670 // Do not generate a writeback event as this instruction is not 671 // complete. 672 DPRINTF(LSQUnit, "Load-store forwarding mis-match. " 673 "Store idx %i to load addr %#x\n", 674 store_idx, req->getVaddr()); 675 676 // Must delete request now that it wasn't handed off to 677 // memory. This is quite ugly. @todo: Figure out the 678 // proper place to really handle request deletes. 679 delete req; 680 if (TheISA::HasUnalignedMemAcc && sreqLow) { 681 delete sreqLow; 682 delete sreqHigh; 683 } 684 685 return NoFault; 686 } 687 } 688 689 // If there's no forwarding case, then go access memory 690 DPRINTF(LSQUnit, "Doing memory access for inst [sn:%lli] PC %#x\n", 691 load_inst->seqNum, load_inst->readPC()); 692 693 assert(!load_inst->memData); 694 load_inst->memData = new uint8_t[64]; 695 696 ++usedPorts; 697 698 // if we the cache is not blocked, do cache access 699 bool completedFirst = false; 700 if (!lsq->cacheBlocked()) { 701 MemCmd command = 702 req->isLLSC() ? MemCmd::LoadLockedReq : MemCmd::ReadReq; 703 PacketPtr data_pkt = new Packet(req, command, Packet::Broadcast); 704 PacketPtr fst_data_pkt = NULL; 705 PacketPtr snd_data_pkt = NULL; 706 707 data_pkt->dataStatic(load_inst->memData); 708 709 LSQSenderState *state = new LSQSenderState; 710 state->isLoad = true; 711 state->idx = load_idx; 712 state->inst = load_inst; 713 data_pkt->senderState = state; 714 715 if (!TheISA::HasUnalignedMemAcc || !sreqLow) { 716 717 // Point the first packet at the main data packet. 718 fst_data_pkt = data_pkt; 719 } else { 720 721 // Create the split packets. 722 fst_data_pkt = new Packet(sreqLow, command, Packet::Broadcast); 723 snd_data_pkt = new Packet(sreqHigh, command, Packet::Broadcast); 724 725 fst_data_pkt->dataStatic(load_inst->memData); 726 snd_data_pkt->dataStatic(load_inst->memData + sreqLow->getSize()); 727 728 fst_data_pkt->senderState = state; 729 snd_data_pkt->senderState = state; 730 731 state->isSplit = true; 732 state->outstanding = 2; 733 state->mainPkt = data_pkt; 734 } 735 736 if (!dcachePort->sendTiming(fst_data_pkt)) { 737 // Delete state and data packet because a load retry 738 // initiates a pipeline restart; it does not retry. 739 delete state; 740 delete data_pkt->req; 741 delete data_pkt; 742 if (TheISA::HasUnalignedMemAcc && sreqLow) { 743 delete fst_data_pkt->req; 744 delete fst_data_pkt; 745 delete snd_data_pkt->req; 746 delete snd_data_pkt; 747 sreqLow = NULL; 748 sreqHigh = NULL; 749 } 750 751 req = NULL; 752 753 // If the access didn't succeed, tell the LSQ by setting 754 // the retry thread id. 755 lsq->setRetryTid(lsqID); 756 } else if (TheISA::HasUnalignedMemAcc && sreqLow) { 757 completedFirst = true; 758 759 // The first packet was sent without problems, so send this one 760 // too. If there is a problem with this packet then the whole 761 // load will be squashed, so indicate this to the state object. 762 // The first packet will return in completeDataAccess and be 763 // handled there. 764 ++usedPorts; 765 if (!dcachePort->sendTiming(snd_data_pkt)) { 766 767 // The main packet will be deleted in completeDataAccess. 768 delete snd_data_pkt->req; 769 delete snd_data_pkt; 770 771 state->complete(); 772 773 req = NULL; 774 sreqHigh = NULL; 775 776 lsq->setRetryTid(lsqID); 777 } 778 } 779 } 780 781 // If the cache was blocked, or has become blocked due to the access, 782 // handle it. 783 if (lsq->cacheBlocked()) { 784 if (req) 785 delete req; 786 if (TheISA::HasUnalignedMemAcc && sreqLow && !completedFirst) { 787 delete sreqLow; 788 delete sreqHigh; 789 } 790 791 ++lsqCacheBlocked; 792 793 iewStage->decrWb(load_inst->seqNum); 794 // There's an older load that's already going to squash. 795 if (isLoadBlocked && blockedLoadSeqNum < load_inst->seqNum) 796 return NoFault; 797 798 // Record that the load was blocked due to memory. This 799 // load will squash all instructions after it, be 800 // refetched, and re-executed. 801 isLoadBlocked = true; 802 loadBlockedHandled = false; 803 blockedLoadSeqNum = load_inst->seqNum; 804 // No fault occurred, even though the interface is blocked. 805 return NoFault; 806 } 807 808 return NoFault; 809} 810 811template <class Impl> 812template <class T> 813Fault 814LSQUnit<Impl>::write(Request *req, Request *sreqLow, Request *sreqHigh, 815 T &data, int store_idx) 816{ 817 assert(storeQueue[store_idx].inst); 818 819 DPRINTF(LSQUnit, "Doing write to store idx %i, addr %#x data %#x" 820 " | storeHead:%i [sn:%i]\n", 821 store_idx, req->getPaddr(), data, storeHead, 822 storeQueue[store_idx].inst->seqNum); 823 824 storeQueue[store_idx].req = req; 825 storeQueue[store_idx].sreqLow = sreqLow; 826 storeQueue[store_idx].sreqHigh = sreqHigh; 827 storeQueue[store_idx].size = sizeof(T); 828 829 // Split stores can only occur in ISAs with unaligned memory accesses. If 830 // a store request has been split, sreqLow and sreqHigh will be non-null. 831 if (TheISA::HasUnalignedMemAcc && sreqLow) { 832 storeQueue[store_idx].isSplit = true; 833 } 834 assert(sizeof(T) <= sizeof(storeQueue[store_idx].data)); 835 836 T gData = htog(data); 837 memcpy(storeQueue[store_idx].data, &gData, sizeof(T)); 838 839 // This function only writes the data to the store queue, so no fault 840 // can happen here. 841 return NoFault; 842} 843 844#endif // __CPU_O3_LSQ_UNIT_HH__ 845