lsq.cc revision 11608
1/* 2 * Copyright (c) 2013-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Andrew Bardsley 38 */ 39 40#include <iomanip> 41#include <sstream> 42 43#include "arch/locked_mem.hh" 44#include "arch/mmapped_ipr.hh" 45#include "cpu/minor/cpu.hh" 46#include "cpu/minor/exec_context.hh" 47#include "cpu/minor/execute.hh" 48#include "cpu/minor/lsq.hh" 49#include "cpu/minor/pipeline.hh" 50#include "debug/Activity.hh" 51#include "debug/MinorMem.hh" 52 53namespace Minor 54{ 55 56/** Returns the offset of addr into an aligned a block of size block_size */ 57static Addr 58addrBlockOffset(Addr addr, unsigned int block_size) 59{ 60 return addr & (block_size - 1); 61} 62 63/** Returns true if the given [addr .. addr+size-1] transfer needs to be 64 * fragmented across a block size of block_size */ 65static bool 66transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size) 67{ 68 return (addrBlockOffset(addr, block_size) + size) > block_size; 69} 70 71LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_, 72 PacketDataPtr data_, uint64_t *res_) : 73 SenderState(), 74 port(port_), 75 inst(inst_), 76 isLoad(isLoad_), 77 data(data_), 78 packet(NULL), 79 request(), 80 fault(NoFault), 81 res(res_), 82 skipped(false), 83 issuedToMemory(false), 84 state(NotIssued) 85{ } 86 87LSQ::AddrRangeCoverage 88LSQ::LSQRequest::containsAddrRangeOf( 89 Addr req1_addr, unsigned int req1_size, 90 Addr req2_addr, unsigned int req2_size) 91{ 92 /* 'end' here means the address of the byte just past the request 93 * blocks */ 94 Addr req2_end_addr = req2_addr + req2_size; 95 Addr req1_end_addr = req1_addr + req1_size; 96 97 AddrRangeCoverage ret; 98 99 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr) 100 ret = NoAddrRangeCoverage; 101 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr) 102 ret = FullAddrRangeCoverage; 103 else 104 ret = PartialAddrRangeCoverage; 105 106 return ret; 107} 108 109LSQ::AddrRangeCoverage 110LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request) 111{ 112 return containsAddrRangeOf(request.getPaddr(), request.getSize(), 113 other_request->request.getPaddr(), other_request->request.getSize()); 114} 115 116bool 117LSQ::LSQRequest::isBarrier() 118{ 119 return inst->isInst() && inst->staticInst->isMemBarrier(); 120} 121 122bool 123LSQ::LSQRequest::needsToBeSentToStoreBuffer() 124{ 125 return state == StoreToStoreBuffer; 126} 127 128void 129LSQ::LSQRequest::setState(LSQRequestState new_state) 130{ 131 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:" 132 " %s\n", state, new_state, *inst); 133 state = new_state; 134} 135 136bool 137LSQ::LSQRequest::isComplete() const 138{ 139 /* @todo, There is currently only one 'completed' state. This 140 * may not be a good choice */ 141 return state == Complete; 142} 143 144void 145LSQ::LSQRequest::reportData(std::ostream &os) const 146{ 147 os << (isLoad ? 'R' : 'W') << ';'; 148 inst->reportData(os); 149 os << ';' << state; 150} 151 152std::ostream & 153operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage) 154{ 155 switch (coverage) { 156 case LSQ::PartialAddrRangeCoverage: 157 os << "PartialAddrRangeCoverage"; 158 break; 159 case LSQ::FullAddrRangeCoverage: 160 os << "FullAddrRangeCoverage"; 161 break; 162 case LSQ::NoAddrRangeCoverage: 163 os << "NoAddrRangeCoverage"; 164 break; 165 default: 166 os << "AddrRangeCoverage-" << static_cast<int>(coverage); 167 break; 168 } 169 return os; 170} 171 172std::ostream & 173operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state) 174{ 175 switch (state) { 176 case LSQ::LSQRequest::NotIssued: 177 os << "NotIssued"; 178 break; 179 case LSQ::LSQRequest::InTranslation: 180 os << "InTranslation"; 181 break; 182 case LSQ::LSQRequest::Translated: 183 os << "Translated"; 184 break; 185 case LSQ::LSQRequest::Failed: 186 os << "Failed"; 187 break; 188 case LSQ::LSQRequest::RequestIssuing: 189 os << "RequestIssuing"; 190 break; 191 case LSQ::LSQRequest::StoreToStoreBuffer: 192 os << "StoreToStoreBuffer"; 193 break; 194 case LSQ::LSQRequest::StoreInStoreBuffer: 195 os << "StoreInStoreBuffer"; 196 break; 197 case LSQ::LSQRequest::StoreBufferIssuing: 198 os << "StoreBufferIssuing"; 199 break; 200 case LSQ::LSQRequest::RequestNeedsRetry: 201 os << "RequestNeedsRetry"; 202 break; 203 case LSQ::LSQRequest::StoreBufferNeedsRetry: 204 os << "StoreBufferNeedsRetry"; 205 break; 206 case LSQ::LSQRequest::Complete: 207 os << "Complete"; 208 break; 209 default: 210 os << "LSQRequestState-" << static_cast<int>(state); 211 break; 212 } 213 return os; 214} 215 216void 217LSQ::clearMemBarrier(MinorDynInstPtr inst) 218{ 219 bool is_last_barrier = 220 inst->id.execSeqNum >= lastMemBarrier[inst->id.threadId]; 221 222 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n", 223 (is_last_barrier ? "last" : "a"), *inst); 224 225 if (is_last_barrier) 226 lastMemBarrier[inst->id.threadId] = 0; 227} 228 229void 230LSQ::SingleDataRequest::finish(const Fault &fault_, RequestPtr request_, 231 ThreadContext *tc, BaseTLB::Mode mode) 232{ 233 fault = fault_; 234 235 port.numAccessesInDTLB--; 236 237 DPRINTFS(MinorMem, (&port), "Received translation response for" 238 " request: %s\n", *inst); 239 240 makePacket(); 241 242 setState(Translated); 243 port.tryToSendToTransfers(this); 244 245 /* Let's try and wake up the processor for the next cycle */ 246 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 247} 248 249void 250LSQ::SingleDataRequest::startAddrTranslation() 251{ 252 ThreadContext *thread = port.cpu.getContext( 253 inst->id.threadId); 254 255 port.numAccessesInDTLB++; 256 257 setState(LSQ::LSQRequest::InTranslation); 258 259 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n"); 260 /* Submit the translation request. The response will come through 261 * finish/markDelayed on the LSQRequest as it bears the Translation 262 * interface */ 263 thread->getDTBPtr()->translateTiming( 264 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write)); 265} 266 267void 268LSQ::SingleDataRequest::retireResponse(PacketPtr packet_) 269{ 270 DPRINTFS(MinorMem, (&port), "Retiring packet\n"); 271 packet = packet_; 272 packetInFlight = false; 273 setState(Complete); 274} 275 276void 277LSQ::SplitDataRequest::finish(const Fault &fault_, RequestPtr request_, 278 ThreadContext *tc, BaseTLB::Mode mode) 279{ 280 fault = fault_; 281 282 port.numAccessesInDTLB--; 283 284 unsigned int M5_VAR_USED expected_fragment_index = 285 numTranslatedFragments; 286 287 numInTranslationFragments--; 288 numTranslatedFragments++; 289 290 DPRINTFS(MinorMem, (&port), "Received translation response for fragment" 291 " %d of request: %s\n", expected_fragment_index, *inst); 292 293 assert(request_ == fragmentRequests[expected_fragment_index]); 294 295 /* Wake up next cycle to get things going again in case the 296 * tryToSendToTransfers does take */ 297 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 298 299 if (fault != NoFault) { 300 /* tryToSendToTransfers will handle the fault */ 301 302 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:" 303 " %d of request: %s\n", 304 expected_fragment_index, *inst); 305 306 setState(Translated); 307 port.tryToSendToTransfers(this); 308 } else if (numTranslatedFragments == numFragments) { 309 makeFragmentPackets(); 310 311 setState(Translated); 312 port.tryToSendToTransfers(this); 313 } else { 314 /* Avoid calling translateTiming from within ::finish */ 315 assert(!translationEvent.scheduled()); 316 port.cpu.schedule(translationEvent, curTick()); 317 } 318} 319 320LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_, 321 bool isLoad_, PacketDataPtr data_, uint64_t *res_) : 322 LSQRequest(port_, inst_, isLoad_, data_, res_), 323 translationEvent(*this), 324 numFragments(0), 325 numInTranslationFragments(0), 326 numTranslatedFragments(0), 327 numIssuedFragments(0), 328 numRetiredFragments(0), 329 fragmentRequests(), 330 fragmentPackets() 331{ 332 /* Don't know how many elements are needed until the request is 333 * populated by the caller. */ 334} 335 336LSQ::SplitDataRequest::~SplitDataRequest() 337{ 338 for (auto i = fragmentRequests.begin(); 339 i != fragmentRequests.end(); i++) 340 { 341 delete *i; 342 } 343 344 for (auto i = fragmentPackets.begin(); 345 i != fragmentPackets.end(); i++) 346 { 347 delete *i; 348 } 349} 350 351void 352LSQ::SplitDataRequest::makeFragmentRequests() 353{ 354 Addr base_addr = request.getVaddr(); 355 unsigned int whole_size = request.getSize(); 356 unsigned int line_width = port.lineWidth; 357 358 unsigned int fragment_size; 359 Addr fragment_addr; 360 361 /* Assume that this transfer is across potentially many block snap 362 * boundaries: 363 * 364 * | _|________|________|________|___ | 365 * | |0| 1 | 2 | 3 | 4 | | 366 * | |_|________|________|________|___| | 367 * | | | | | | 368 * 369 * The first transfer (0) can be up to lineWidth in size. 370 * All the middle transfers (1-3) are lineWidth in size 371 * The last transfer (4) can be from zero to lineWidth - 1 in size 372 */ 373 unsigned int first_fragment_offset = 374 addrBlockOffset(base_addr, line_width); 375 unsigned int last_fragment_size = 376 addrBlockOffset(base_addr + whole_size, line_width); 377 unsigned int first_fragment_size = 378 line_width - first_fragment_offset; 379 380 unsigned int middle_fragments_total_size = 381 whole_size - (first_fragment_size + last_fragment_size); 382 383 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0); 384 385 unsigned int middle_fragment_count = 386 middle_fragments_total_size / line_width; 387 388 numFragments = 1 /* first */ + middle_fragment_count + 389 (last_fragment_size == 0 ? 0 : 1); 390 391 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests." 392 " First fragment size: %d Last fragment size: %d\n", 393 numFragments, first_fragment_size, 394 (last_fragment_size == 0 ? line_width : last_fragment_size)); 395 396 assert(((middle_fragment_count * line_width) + 397 first_fragment_size + last_fragment_size) == whole_size); 398 399 fragment_addr = base_addr; 400 fragment_size = first_fragment_size; 401 402 /* Just past the last address in the request */ 403 Addr end_addr = base_addr + whole_size; 404 405 for (unsigned int fragment_index = 0; fragment_index < numFragments; 406 fragment_index++) 407 { 408 bool M5_VAR_USED is_last_fragment = false; 409 410 if (fragment_addr == base_addr) { 411 /* First fragment */ 412 fragment_size = first_fragment_size; 413 } else { 414 if ((fragment_addr + line_width) > end_addr) { 415 /* Adjust size of last fragment */ 416 fragment_size = end_addr - fragment_addr; 417 is_last_fragment = true; 418 } else { 419 /* Middle fragments */ 420 fragment_size = line_width; 421 } 422 } 423 424 Request *fragment = new Request(); 425 426 fragment->setContext(request.contextId()); 427 fragment->setVirt(0 /* asid */, 428 fragment_addr, fragment_size, request.getFlags(), 429 request.masterId(), 430 request.getPC()); 431 432 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d" 433 " (whole request addr: 0x%x size: %d) %s\n", 434 fragment_addr, fragment_size, base_addr, whole_size, 435 (is_last_fragment ? "last fragment" : "")); 436 437 fragment_addr += fragment_size; 438 439 fragmentRequests.push_back(fragment); 440 } 441} 442 443void 444LSQ::SplitDataRequest::makeFragmentPackets() 445{ 446 Addr base_addr = request.getVaddr(); 447 448 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst); 449 450 for (unsigned int fragment_index = 0; fragment_index < numFragments; 451 fragment_index++) 452 { 453 Request *fragment = fragmentRequests[fragment_index]; 454 455 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s" 456 " (%d, 0x%x)\n", 457 fragment_index, *inst, 458 (fragment->hasPaddr() ? "has paddr" : "no paddr"), 459 (fragment->hasPaddr() ? fragment->getPaddr() : 0)); 460 461 Addr fragment_addr = fragment->getVaddr(); 462 unsigned int fragment_size = fragment->getSize(); 463 464 uint8_t *request_data = NULL; 465 466 if (!isLoad) { 467 /* Split data for Packets. Will become the property of the 468 * outgoing Packets */ 469 request_data = new uint8_t[fragment_size]; 470 std::memcpy(request_data, data + (fragment_addr - base_addr), 471 fragment_size); 472 } 473 474 assert(fragment->hasPaddr()); 475 476 PacketPtr fragment_packet = 477 makePacketForRequest(*fragment, isLoad, this, request_data); 478 479 fragmentPackets.push_back(fragment_packet); 480 /* Accumulate flags in parent request */ 481 request.setFlags(fragment->getFlags()); 482 } 483 484 /* Might as well make the overall/response packet here */ 485 /* Get the physical address for the whole request/packet from the first 486 * fragment */ 487 request.setPaddr(fragmentRequests[0]->getPaddr()); 488 makePacket(); 489} 490 491void 492LSQ::SplitDataRequest::startAddrTranslation() 493{ 494 setState(LSQ::LSQRequest::InTranslation); 495 496 makeFragmentRequests(); 497 498 numInTranslationFragments = 0; 499 numTranslatedFragments = 0; 500 501 /* @todo, just do these in sequence for now with 502 * a loop of: 503 * do { 504 * sendNextFragmentToTranslation ; translateTiming ; finish 505 * } while (numTranslatedFragments != numFragments); 506 */ 507 508 /* Do first translation */ 509 sendNextFragmentToTranslation(); 510} 511 512PacketPtr 513LSQ::SplitDataRequest::getHeadPacket() 514{ 515 assert(numIssuedFragments < numFragments); 516 517 return fragmentPackets[numIssuedFragments]; 518} 519 520void 521LSQ::SplitDataRequest::stepToNextPacket() 522{ 523 assert(numIssuedFragments < numFragments); 524 525 numIssuedFragments++; 526} 527 528void 529LSQ::SplitDataRequest::retireResponse(PacketPtr response) 530{ 531 assert(numRetiredFragments < numFragments); 532 533 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d" 534 " offset: 0x%x (retired fragment num: %d) %s\n", 535 response->req->getVaddr(), response->req->getSize(), 536 request.getVaddr() - response->req->getVaddr(), 537 numRetiredFragments, 538 (fault == NoFault ? "" : fault->name())); 539 540 numRetiredFragments++; 541 542 if (skipped) { 543 /* Skip because we already knew the request had faulted or been 544 * skipped */ 545 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n"); 546 } else if (response->isError()) { 547 /* Mark up the error and leave to execute to handle it */ 548 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n"); 549 setSkipped(); 550 packet->copyError(response); 551 } else { 552 if (isLoad) { 553 if (!data) { 554 /* For a split transfer, a Packet must be constructed 555 * to contain all returning data. This is that packet's 556 * data */ 557 data = new uint8_t[request.getSize()]; 558 } 559 560 /* Populate the portion of the overall response data represented 561 * by the response fragment */ 562 std::memcpy( 563 data + (response->req->getVaddr() - request.getVaddr()), 564 response->getConstPtr<uint8_t>(), 565 response->req->getSize()); 566 } 567 } 568 569 /* Complete early if we're skipping are no more in-flight accesses */ 570 if (skipped && !hasPacketsInMemSystem()) { 571 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n"); 572 setState(Complete); 573 if (packet->needsResponse()) 574 packet->makeResponse(); 575 } 576 577 if (numRetiredFragments == numFragments) 578 setState(Complete); 579 580 if (!skipped && isComplete()) { 581 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL); 582 583 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d" 584 " needsResponse: %d packetSize: %s requestSize: %s responseSize:" 585 " %s\n", packet->isRead(), packet->isWrite(), 586 packet->needsResponse(), packet->getSize(), request.getSize(), 587 response->getSize()); 588 589 /* A request can become complete by several paths, this is a sanity 590 * check to make sure the packet's data is created */ 591 if (!data) { 592 data = new uint8_t[request.getSize()]; 593 } 594 595 if (isLoad) { 596 DPRINTFS(MinorMem, (&port), "Copying read data\n"); 597 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize()); 598 } 599 packet->makeResponse(); 600 } 601 602 /* Packets are all deallocated together in ~SplitLSQRequest */ 603} 604 605void 606LSQ::SplitDataRequest::sendNextFragmentToTranslation() 607{ 608 unsigned int fragment_index = numTranslatedFragments; 609 610 ThreadContext *thread = port.cpu.getContext( 611 inst->id.threadId); 612 613 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n", 614 fragment_index); 615 616 port.numAccessesInDTLB++; 617 numInTranslationFragments++; 618 619 thread->getDTBPtr()->translateTiming( 620 fragmentRequests[fragment_index], thread, this, (isLoad ? 621 BaseTLB::Read : BaseTLB::Write)); 622} 623 624bool 625LSQ::StoreBuffer::canInsert() const 626{ 627 /* @todo, support store amalgamation */ 628 return slots.size() < numSlots; 629} 630 631void 632LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request) 633{ 634 auto found = std::find(slots.begin(), slots.end(), request); 635 636 if (found != slots.end()) { 637 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n", 638 request, *found, *(request->inst)); 639 slots.erase(found); 640 641 delete request; 642 } 643} 644 645void 646LSQ::StoreBuffer::insert(LSQRequestPtr request) 647{ 648 if (!canInsert()) { 649 warn("%s: store buffer insertion without space to insert from" 650 " inst: %s\n", name(), *(request->inst)); 651 } 652 653 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request); 654 655 numUnissuedAccesses++; 656 657 if (request->state != LSQRequest::Complete) 658 request->setState(LSQRequest::StoreInStoreBuffer); 659 660 slots.push_back(request); 661 662 /* Let's try and wake up the processor for the next cycle to step 663 * the store buffer */ 664 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 665} 666 667LSQ::AddrRangeCoverage 668LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request, 669 unsigned int &found_slot) 670{ 671 unsigned int slot_index = slots.size() - 1; 672 auto i = slots.rbegin(); 673 AddrRangeCoverage ret = NoAddrRangeCoverage; 674 675 /* Traverse the store buffer in reverse order (most to least recent) 676 * and try to find a slot whose address range overlaps this request */ 677 while (ret == NoAddrRangeCoverage && i != slots.rend()) { 678 LSQRequestPtr slot = *i; 679 680 if (slot->packet && 681 slot->inst->id.threadId == request->inst->id.threadId) { 682 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request); 683 684 if (coverage != NoAddrRangeCoverage) { 685 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:" 686 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n", 687 slot_index, coverage, 688 request->request.getPaddr(), request->request.getSize(), 689 slot->request.getPaddr(), slot->request.getSize()); 690 691 found_slot = slot_index; 692 ret = coverage; 693 } 694 } 695 696 i++; 697 slot_index--; 698 } 699 700 return ret; 701} 702 703/** Fill the given packet with appropriate date from slot slot_number */ 704void 705LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load, 706 unsigned int slot_number) 707{ 708 assert(slot_number < slots.size()); 709 assert(load->packet); 710 assert(load->isLoad); 711 712 LSQRequestPtr store = slots[slot_number]; 713 714 assert(store->packet); 715 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage); 716 717 Addr load_addr = load->request.getPaddr(); 718 Addr store_addr = store->request.getPaddr(); 719 Addr addr_offset = load_addr - store_addr; 720 721 unsigned int load_size = load->request.getSize(); 722 723 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer" 724 " slot: %d addr: 0x%x addressOffset: 0x%x\n", 725 load_size, load_addr, slot_number, 726 store_addr, addr_offset); 727 728 void *load_packet_data = load->packet->getPtr<void>(); 729 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset; 730 731 std::memcpy(load_packet_data, store_packet_data, load_size); 732} 733 734void 735LSQ::StoreBuffer::countIssuedStore(LSQRequestPtr request) 736{ 737 /* Barriers are accounted for as they are cleared from 738 * the queue, not after their transfers are complete */ 739 if (!request->isBarrier()) 740 numUnissuedAccesses--; 741} 742 743void 744LSQ::StoreBuffer::step() 745{ 746 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n", 747 numUnissuedAccesses); 748 749 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) { 750 /* Clear all the leading barriers */ 751 while (!slots.empty() && 752 slots.front()->isComplete() && slots.front()->isBarrier()) 753 { 754 LSQRequestPtr barrier = slots.front(); 755 756 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n", 757 *(barrier->inst)); 758 759 numUnissuedAccesses--; 760 lsq.clearMemBarrier(barrier->inst); 761 slots.pop_front(); 762 763 delete barrier; 764 } 765 766 auto i = slots.begin(); 767 bool issued = true; 768 unsigned int issue_count = 0; 769 770 /* Skip trying if the memory system is busy */ 771 if (lsq.state == LSQ::MemoryNeedsRetry) 772 issued = false; 773 774 /* Try to issue all stores in order starting from the head 775 * of the queue. Responses are allowed to be retired 776 * out of order */ 777 while (issued && 778 issue_count < storeLimitPerCycle && 779 lsq.canSendToMemorySystem() && 780 i != slots.end()) 781 { 782 LSQRequestPtr request = *i; 783 784 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d" 785 " state: %s\n", 786 *(request->inst), request->sentAllPackets(), 787 request->state); 788 789 if (request->isBarrier() && request->isComplete()) { 790 /* Give up at barriers */ 791 issued = false; 792 } else if (!(request->state == LSQRequest::StoreBufferIssuing && 793 request->sentAllPackets())) 794 { 795 DPRINTF(MinorMem, "Trying to send request: %s to memory" 796 " system\n", *(request->inst)); 797 798 if (lsq.tryToSend(request)) { 799 countIssuedStore(request); 800 issue_count++; 801 } else { 802 /* Don't step on to the next store buffer entry if this 803 * one hasn't issued all its packets as the store 804 * buffer must still enforce ordering */ 805 issued = false; 806 } 807 } 808 i++; 809 } 810 } 811} 812 813void 814LSQ::completeMemBarrierInst(MinorDynInstPtr inst, 815 bool committed) 816{ 817 if (committed) { 818 /* Not already sent to the store buffer as a store request? */ 819 if (!inst->inStoreBuffer) { 820 /* Insert an entry into the store buffer to tick off barriers 821 * until there are none in flight */ 822 storeBuffer.insert(new BarrierDataRequest(*this, inst)); 823 } 824 } else { 825 /* Clear the barrier anyway if it wasn't actually committed */ 826 clearMemBarrier(inst); 827 } 828} 829 830void 831LSQ::StoreBuffer::minorTrace() const 832{ 833 unsigned int size = slots.size(); 834 unsigned int i = 0; 835 std::ostringstream os; 836 837 while (i < size) { 838 LSQRequestPtr request = slots[i]; 839 840 request->reportData(os); 841 842 i++; 843 if (i < numSlots) 844 os << ','; 845 } 846 847 while (i < numSlots) { 848 os << '-'; 849 850 i++; 851 if (i < numSlots) 852 os << ','; 853 } 854 855 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(), 856 numUnissuedAccesses); 857} 858 859void 860LSQ::tryToSendToTransfers(LSQRequestPtr request) 861{ 862 if (state == MemoryNeedsRetry) { 863 DPRINTF(MinorMem, "Request needs retry, not issuing to" 864 " memory until retry arrives\n"); 865 return; 866 } 867 868 if (request->state == LSQRequest::InTranslation) { 869 DPRINTF(MinorMem, "Request still in translation, not issuing to" 870 " memory\n"); 871 return; 872 } 873 874 assert(request->state == LSQRequest::Translated || 875 request->state == LSQRequest::RequestIssuing || 876 request->state == LSQRequest::Failed || 877 request->state == LSQRequest::Complete); 878 879 if (requests.empty() || requests.front() != request) { 880 DPRINTF(MinorMem, "Request not at front of requests queue, can't" 881 " issue to memory\n"); 882 return; 883 } 884 885 if (transfers.unreservedRemainingSpace() == 0) { 886 DPRINTF(MinorMem, "No space to insert request into transfers" 887 " queue\n"); 888 return; 889 } 890 891 if (request->isComplete() || request->state == LSQRequest::Failed) { 892 DPRINTF(MinorMem, "Passing a %s transfer on to transfers" 893 " queue\n", (request->isComplete() ? "completed" : "failed")); 894 request->setState(LSQRequest::Complete); 895 request->setSkipped(); 896 moveFromRequestsToTransfers(request); 897 return; 898 } 899 900 if (!execute.instIsRightStream(request->inst)) { 901 /* Wrong stream, try to abort the transfer but only do so if 902 * there are no packets in flight */ 903 if (request->hasPacketsInMemSystem()) { 904 DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 905 " waiting for responses before aborting request\n"); 906 } else { 907 DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 908 " aborting request\n"); 909 request->setState(LSQRequest::Complete); 910 request->setSkipped(); 911 moveFromRequestsToTransfers(request); 912 } 913 return; 914 } 915 916 if (request->fault != NoFault) { 917 if (request->inst->staticInst->isPrefetch()) { 918 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n"); 919 } 920 DPRINTF(MinorMem, "Moving faulting request into the transfers" 921 " queue\n"); 922 request->setState(LSQRequest::Complete); 923 request->setSkipped(); 924 moveFromRequestsToTransfers(request); 925 return; 926 } 927 928 bool is_load = request->isLoad; 929 bool is_llsc = request->request.isLLSC(); 930 bool is_swap = request->request.isSwap(); 931 bool bufferable = !(request->request.isStrictlyOrdered() || 932 is_llsc || is_swap); 933 934 if (is_load) { 935 if (numStoresInTransfers != 0) { 936 DPRINTF(MinorMem, "Load request with stores still in transfers" 937 " queue, stalling\n"); 938 return; 939 } 940 } else { 941 /* Store. Can it be sent to the store buffer? */ 942 if (bufferable && !request->request.isMmappedIpr()) { 943 request->setState(LSQRequest::StoreToStoreBuffer); 944 moveFromRequestsToTransfers(request); 945 DPRINTF(MinorMem, "Moving store into transfers queue\n"); 946 return; 947 } 948 } 949 950 /* Check if this is the head instruction (and so must be executable as 951 * its stream sequence number was checked above) for loads which must 952 * not be speculatively issued and stores which must be issued here */ 953 if (!bufferable) { 954 if (!execute.instIsHeadInst(request->inst)) { 955 DPRINTF(MinorMem, "Memory access not the head inst., can't be" 956 " sure it can be performed, not issuing\n"); 957 return; 958 } 959 960 unsigned int forwarding_slot = 0; 961 962 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) != 963 NoAddrRangeCoverage) 964 { 965 DPRINTF(MinorMem, "Memory access can receive forwarded data" 966 " from the store buffer, need to wait for store buffer to" 967 " drain\n"); 968 return; 969 } 970 } 971 972 /* True: submit this packet to the transfers queue to be sent to the 973 * memory system. 974 * False: skip the memory and push a packet for this request onto 975 * requests */ 976 bool do_access = true; 977 978 if (!is_llsc) { 979 /* Check for match in the store buffer */ 980 if (is_load) { 981 unsigned int forwarding_slot = 0; 982 AddrRangeCoverage forwarding_result = 983 storeBuffer.canForwardDataToLoad(request, 984 forwarding_slot); 985 986 switch (forwarding_result) { 987 case FullAddrRangeCoverage: 988 /* Forward data from the store buffer into this request and 989 * repurpose this request's packet into a response packet */ 990 storeBuffer.forwardStoreData(request, forwarding_slot); 991 request->packet->makeResponse(); 992 993 /* Just move between queues, no access */ 994 do_access = false; 995 break; 996 case PartialAddrRangeCoverage: 997 DPRINTF(MinorMem, "Load partly satisfied by store buffer" 998 " data. Must wait for the store to complete\n"); 999 return; 1000 break; 1001 case NoAddrRangeCoverage: 1002 DPRINTF(MinorMem, "No forwardable data from store buffer\n"); 1003 /* Fall through to try access */ 1004 break; 1005 } 1006 } 1007 } else { 1008 if (!canSendToMemorySystem()) { 1009 DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 1010 return; 1011 } 1012 1013 SimpleThread &thread = *cpu.threads[request->inst->id.threadId]; 1014 1015 TheISA::PCState old_pc = thread.pcState(); 1016 ExecContext context(cpu, thread, execute, request->inst); 1017 1018 /* Handle LLSC requests and tests */ 1019 if (is_load) { 1020 TheISA::handleLockedRead(&context, &request->request); 1021 } else { 1022 do_access = TheISA::handleLockedWrite(&context, 1023 &request->request, cacheBlockMask); 1024 1025 if (!do_access) { 1026 DPRINTF(MinorMem, "Not perfoming a memory " 1027 "access for store conditional\n"); 1028 } 1029 } 1030 thread.pcState(old_pc); 1031 } 1032 1033 /* See the do_access comment above */ 1034 if (do_access) { 1035 if (!canSendToMemorySystem()) { 1036 DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 1037 return; 1038 } 1039 1040 /* Remember if this is an access which can't be idly 1041 * discarded by an interrupt */ 1042 if (!bufferable && !request->issuedToMemory) { 1043 numAccessesIssuedToMemory++; 1044 request->issuedToMemory = true; 1045 } 1046 1047 if (tryToSend(request)) { 1048 moveFromRequestsToTransfers(request); 1049 } 1050 } else { 1051 request->setState(LSQRequest::Complete); 1052 moveFromRequestsToTransfers(request); 1053 } 1054} 1055 1056bool 1057LSQ::tryToSend(LSQRequestPtr request) 1058{ 1059 bool ret = false; 1060 1061 if (!canSendToMemorySystem()) { 1062 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n", 1063 *(request->inst)); 1064 } else { 1065 PacketPtr packet = request->getHeadPacket(); 1066 1067 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n", 1068 *(request->inst), packet->req->getVaddr()); 1069 1070 /* The sender state of the packet *must* be an LSQRequest 1071 * so the response can be correctly handled */ 1072 assert(packet->findNextSenderState<LSQRequest>()); 1073 1074 if (request->request.isMmappedIpr()) { 1075 ThreadContext *thread = 1076 cpu.getContext(cpu.contextToThread( 1077 request->request.contextId())); 1078 1079 if (request->isLoad) { 1080 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst)); 1081 TheISA::handleIprRead(thread, packet); 1082 } else { 1083 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst)); 1084 TheISA::handleIprWrite(thread, packet); 1085 } 1086 1087 request->stepToNextPacket(); 1088 ret = request->sentAllPackets(); 1089 1090 if (!ret) { 1091 DPRINTF(MinorMem, "IPR access has another packet: %s\n", 1092 *(request->inst)); 1093 } 1094 1095 if (ret) 1096 request->setState(LSQRequest::Complete); 1097 else 1098 request->setState(LSQRequest::RequestIssuing); 1099 } else if (dcachePort.sendTimingReq(packet)) { 1100 DPRINTF(MinorMem, "Sent data memory request\n"); 1101 1102 numAccessesInMemorySystem++; 1103 1104 request->stepToNextPacket(); 1105 1106 ret = request->sentAllPackets(); 1107 1108 switch (request->state) { 1109 case LSQRequest::Translated: 1110 case LSQRequest::RequestIssuing: 1111 /* Fully or partially issued a request in the transfers 1112 * queue */ 1113 request->setState(LSQRequest::RequestIssuing); 1114 break; 1115 case LSQRequest::StoreInStoreBuffer: 1116 case LSQRequest::StoreBufferIssuing: 1117 /* Fully or partially issued a request in the store 1118 * buffer */ 1119 request->setState(LSQRequest::StoreBufferIssuing); 1120 break; 1121 default: 1122 assert(false); 1123 break; 1124 } 1125 1126 state = MemoryRunning; 1127 } else { 1128 DPRINTF(MinorMem, 1129 "Sending data memory request - needs retry\n"); 1130 1131 /* Needs to be resent, wait for that */ 1132 state = MemoryNeedsRetry; 1133 retryRequest = request; 1134 1135 switch (request->state) { 1136 case LSQRequest::Translated: 1137 case LSQRequest::RequestIssuing: 1138 request->setState(LSQRequest::RequestNeedsRetry); 1139 break; 1140 case LSQRequest::StoreInStoreBuffer: 1141 case LSQRequest::StoreBufferIssuing: 1142 request->setState(LSQRequest::StoreBufferNeedsRetry); 1143 break; 1144 default: 1145 assert(false); 1146 break; 1147 } 1148 } 1149 } 1150 1151 if (ret) 1152 threadSnoop(request); 1153 1154 return ret; 1155} 1156 1157void 1158LSQ::moveFromRequestsToTransfers(LSQRequestPtr request) 1159{ 1160 assert(!requests.empty() && requests.front() == request); 1161 assert(transfers.unreservedRemainingSpace() != 0); 1162 1163 /* Need to count the number of stores in the transfers 1164 * queue so that loads know when their store buffer forwarding 1165 * results will be correct (only when all those stores 1166 * have reached the store buffer) */ 1167 if (!request->isLoad) 1168 numStoresInTransfers++; 1169 1170 requests.pop(); 1171 transfers.push(request); 1172} 1173 1174bool 1175LSQ::canSendToMemorySystem() 1176{ 1177 return state == MemoryRunning && 1178 numAccessesInMemorySystem < inMemorySystemLimit; 1179} 1180 1181bool 1182LSQ::recvTimingResp(PacketPtr response) 1183{ 1184 LSQRequestPtr request = 1185 safe_cast<LSQRequestPtr>(response->popSenderState()); 1186 1187 DPRINTF(MinorMem, "Received response packet inst: %s" 1188 " addr: 0x%x cmd: %s\n", 1189 *(request->inst), response->getAddr(), 1190 response->cmd.toString()); 1191 1192 numAccessesInMemorySystem--; 1193 1194 if (response->isError()) { 1195 DPRINTF(MinorMem, "Received error response packet: %s\n", 1196 *request->inst); 1197 } 1198 1199 switch (request->state) { 1200 case LSQRequest::RequestIssuing: 1201 case LSQRequest::RequestNeedsRetry: 1202 /* Response to a request from the transfers queue */ 1203 request->retireResponse(response); 1204 1205 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n", 1206 request->hasPacketsInMemSystem(), request->isComplete()); 1207 1208 break; 1209 case LSQRequest::StoreBufferIssuing: 1210 case LSQRequest::StoreBufferNeedsRetry: 1211 /* Response to a request from the store buffer */ 1212 request->retireResponse(response); 1213 1214 /* Remove completed requests unless they are barriers (which will 1215 * need to be removed in order */ 1216 if (request->isComplete()) { 1217 if (!request->isBarrier()) { 1218 storeBuffer.deleteRequest(request); 1219 } else { 1220 DPRINTF(MinorMem, "Completed transfer for barrier: %s" 1221 " leaving the request as it is also a barrier\n", 1222 *(request->inst)); 1223 } 1224 } 1225 break; 1226 default: 1227 /* Shouldn't be allowed to receive a response from another 1228 * state */ 1229 assert(false); 1230 break; 1231 } 1232 1233 /* We go to idle even if there are more things in the requests queue 1234 * as it's the job of step to actually step us on to the next 1235 * transaction */ 1236 1237 /* Let's try and wake up the processor for the next cycle */ 1238 cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 1239 1240 /* Never busy */ 1241 return true; 1242} 1243 1244void 1245LSQ::recvReqRetry() 1246{ 1247 DPRINTF(MinorMem, "Received retry request\n"); 1248 1249 assert(state == MemoryNeedsRetry); 1250 1251 switch (retryRequest->state) { 1252 case LSQRequest::RequestNeedsRetry: 1253 /* Retry in the requests queue */ 1254 retryRequest->setState(LSQRequest::Translated); 1255 break; 1256 case LSQRequest::StoreBufferNeedsRetry: 1257 /* Retry in the store buffer */ 1258 retryRequest->setState(LSQRequest::StoreInStoreBuffer); 1259 break; 1260 default: 1261 assert(false); 1262 } 1263 1264 /* Set state back to MemoryRunning so that the following 1265 * tryToSend can actually send. Note that this won't 1266 * allow another transfer in as tryToSend should 1267 * issue a memory request and either succeed for this 1268 * request or return the LSQ back to MemoryNeedsRetry */ 1269 state = MemoryRunning; 1270 1271 /* Try to resend the request */ 1272 if (tryToSend(retryRequest)) { 1273 /* Successfully sent, need to move the request */ 1274 switch (retryRequest->state) { 1275 case LSQRequest::RequestIssuing: 1276 /* In the requests queue */ 1277 moveFromRequestsToTransfers(retryRequest); 1278 break; 1279 case LSQRequest::StoreBufferIssuing: 1280 /* In the store buffer */ 1281 storeBuffer.countIssuedStore(retryRequest); 1282 break; 1283 default: 1284 assert(false); 1285 break; 1286 } 1287 1288 retryRequest = NULL; 1289 } 1290} 1291 1292LSQ::LSQ(std::string name_, std::string dcache_port_name_, 1293 MinorCPU &cpu_, Execute &execute_, 1294 unsigned int in_memory_system_limit, unsigned int line_width, 1295 unsigned int requests_queue_size, unsigned int transfers_queue_size, 1296 unsigned int store_buffer_size, 1297 unsigned int store_buffer_cycle_store_limit) : 1298 Named(name_), 1299 cpu(cpu_), 1300 execute(execute_), 1301 dcachePort(dcache_port_name_, *this, cpu_), 1302 lastMemBarrier(cpu.numThreads, 0), 1303 state(MemoryRunning), 1304 inMemorySystemLimit(in_memory_system_limit), 1305 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)), 1306 requests(name_ + ".requests", "addr", requests_queue_size), 1307 transfers(name_ + ".transfers", "addr", transfers_queue_size), 1308 storeBuffer(name_ + ".storeBuffer", 1309 *this, store_buffer_size, store_buffer_cycle_store_limit), 1310 numAccessesInMemorySystem(0), 1311 numAccessesInDTLB(0), 1312 numStoresInTransfers(0), 1313 numAccessesIssuedToMemory(0), 1314 retryRequest(NULL), 1315 cacheBlockMask(~(cpu_.cacheLineSize() - 1)) 1316{ 1317 if (in_memory_system_limit < 1) { 1318 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_, 1319 in_memory_system_limit); 1320 } 1321 1322 if (store_buffer_cycle_store_limit < 1) { 1323 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be" 1324 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit); 1325 } 1326 1327 if (requests_queue_size < 1) { 1328 fatal("%s: executeLSQRequestsQueueSize must be" 1329 " >= 1 (%d)\n", name_, requests_queue_size); 1330 } 1331 1332 if (transfers_queue_size < 1) { 1333 fatal("%s: executeLSQTransfersQueueSize must be" 1334 " >= 1 (%d)\n", name_, transfers_queue_size); 1335 } 1336 1337 if (store_buffer_size < 1) { 1338 fatal("%s: executeLSQStoreBufferSize must be" 1339 " >= 1 (%d)\n", name_, store_buffer_size); 1340 } 1341 1342 if ((lineWidth & (lineWidth - 1)) != 0) { 1343 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth); 1344 } 1345} 1346 1347LSQ::~LSQ() 1348{ } 1349 1350LSQ::LSQRequest::~LSQRequest() 1351{ 1352 if (packet) 1353 delete packet; 1354 if (data) 1355 delete [] data; 1356} 1357 1358/** 1359 * Step the memory access mechanism on to its next state. In reality, most 1360 * of the stepping is done by the callbacks on the LSQ but this 1361 * function is responsible for issuing memory requests lodged in the 1362 * requests queue. 1363 */ 1364void 1365LSQ::step() 1366{ 1367 /* Try to move address-translated requests between queues and issue 1368 * them */ 1369 if (!requests.empty()) 1370 tryToSendToTransfers(requests.front()); 1371 1372 storeBuffer.step(); 1373} 1374 1375LSQ::LSQRequestPtr 1376LSQ::findResponse(MinorDynInstPtr inst) 1377{ 1378 LSQ::LSQRequestPtr ret = NULL; 1379 1380 if (!transfers.empty()) { 1381 LSQRequestPtr request = transfers.front(); 1382 1383 /* Same instruction and complete access or a store that's 1384 * capable of being moved to the store buffer */ 1385 if (request->inst->id == inst->id) { 1386 bool complete = request->isComplete(); 1387 bool can_store = storeBuffer.canInsert(); 1388 bool to_store_buffer = request->state == 1389 LSQRequest::StoreToStoreBuffer; 1390 1391 if ((complete && !(request->isBarrier() && !can_store)) || 1392 (to_store_buffer && can_store)) 1393 { 1394 ret = request; 1395 } 1396 } 1397 } 1398 1399 if (ret) { 1400 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n", 1401 *inst); 1402 } else { 1403 DPRINTF(MinorMem, "No matching memory response for inst: %s\n", 1404 *inst); 1405 } 1406 1407 return ret; 1408} 1409 1410void 1411LSQ::popResponse(LSQ::LSQRequestPtr response) 1412{ 1413 assert(!transfers.empty() && transfers.front() == response); 1414 1415 transfers.pop(); 1416 1417 if (!response->isLoad) 1418 numStoresInTransfers--; 1419 1420 if (response->issuedToMemory) 1421 numAccessesIssuedToMemory--; 1422 1423 if (response->state != LSQRequest::StoreInStoreBuffer) { 1424 DPRINTF(MinorMem, "Deleting %s request: %s\n", 1425 (response->isLoad ? "load" : "store"), 1426 *(response->inst)); 1427 1428 delete response; 1429 } 1430} 1431 1432void 1433LSQ::sendStoreToStoreBuffer(LSQRequestPtr request) 1434{ 1435 assert(request->state == LSQRequest::StoreToStoreBuffer); 1436 1437 DPRINTF(MinorMem, "Sending store: %s to store buffer\n", 1438 *(request->inst)); 1439 1440 request->inst->inStoreBuffer = true; 1441 1442 storeBuffer.insert(request); 1443} 1444 1445bool 1446LSQ::isDrained() 1447{ 1448 return requests.empty() && transfers.empty() && 1449 storeBuffer.isDrained(); 1450} 1451 1452bool 1453LSQ::needsToTick() 1454{ 1455 bool ret = false; 1456 1457 if (canSendToMemorySystem()) { 1458 bool have_translated_requests = !requests.empty() && 1459 requests.front()->state != LSQRequest::InTranslation && 1460 transfers.unreservedRemainingSpace() != 0; 1461 1462 ret = have_translated_requests || 1463 storeBuffer.numUnissuedStores() != 0; 1464 } 1465 1466 if (ret) 1467 DPRINTF(Activity, "Need to tick\n"); 1468 1469 return ret; 1470} 1471 1472void 1473LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data, 1474 unsigned int size, Addr addr, Request::Flags flags, 1475 uint64_t *res) 1476{ 1477 bool needs_burst = transferNeedsBurst(addr, size, lineWidth); 1478 LSQRequestPtr request; 1479 1480 /* Copy given data into the request. The request will pass this to the 1481 * packet and then it will own the data */ 1482 uint8_t *request_data = NULL; 1483 1484 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:" 1485 " 0x%x%s lineWidth : 0x%x\n", 1486 (isLoad ? "load" : "store"), addr, size, flags, 1487 (needs_burst ? " (needs burst)" : ""), lineWidth); 1488 1489 if (!isLoad) { 1490 /* request_data becomes the property of a ...DataRequest (see below) 1491 * and destroyed by its destructor */ 1492 request_data = new uint8_t[size]; 1493 if (flags & Request::CACHE_BLOCK_ZERO) { 1494 /* For cache zeroing, just use zeroed data */ 1495 std::memset(request_data, 0, size); 1496 } else { 1497 std::memcpy(request_data, data, size); 1498 } 1499 } 1500 1501 if (needs_burst) { 1502 request = new SplitDataRequest( 1503 *this, inst, isLoad, request_data, res); 1504 } else { 1505 request = new SingleDataRequest( 1506 *this, inst, isLoad, request_data, res); 1507 } 1508 1509 if (inst->traceData) 1510 inst->traceData->setMem(addr, size, flags); 1511 1512 int cid = cpu.threads[inst->id.threadId]->getTC()->contextId(); 1513 request->request.setContext(cid); 1514 request->request.setVirt(0 /* asid */, 1515 addr, size, flags, cpu.dataMasterId(), 1516 /* I've no idea why we need the PC, but give it */ 1517 inst->pc.instAddr()); 1518 1519 requests.push(request); 1520 request->startAddrTranslation(); 1521} 1522 1523void 1524LSQ::pushFailedRequest(MinorDynInstPtr inst) 1525{ 1526 LSQRequestPtr request = new FailedDataRequest(*this, inst); 1527 requests.push(request); 1528} 1529 1530void 1531LSQ::minorTrace() const 1532{ 1533 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d" 1534 " lastMemBarrier=%d\n", 1535 state, numAccessesInDTLB, numAccessesInMemorySystem, 1536 numStoresInTransfers, lastMemBarrier[0]); 1537 requests.minorTrace(); 1538 transfers.minorTrace(); 1539 storeBuffer.minorTrace(); 1540} 1541 1542LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_, 1543 unsigned int store_buffer_size, 1544 unsigned int store_limit_per_cycle) : 1545 Named(name_), lsq(lsq_), 1546 numSlots(store_buffer_size), 1547 storeLimitPerCycle(store_limit_per_cycle), 1548 slots(), 1549 numUnissuedAccesses(0) 1550{ 1551} 1552 1553PacketPtr 1554makePacketForRequest(Request &request, bool isLoad, 1555 Packet::SenderState *sender_state, PacketDataPtr data) 1556{ 1557 PacketPtr ret = isLoad ? Packet::createRead(&request) 1558 : Packet::createWrite(&request); 1559 1560 if (sender_state) 1561 ret->pushSenderState(sender_state); 1562 1563 if (isLoad) 1564 ret->allocate(); 1565 else 1566 ret->dataDynamic(data); 1567 1568 return ret; 1569} 1570 1571void 1572LSQ::issuedMemBarrierInst(MinorDynInstPtr inst) 1573{ 1574 assert(inst->isInst() && inst->staticInst->isMemBarrier()); 1575 assert(inst->id.execSeqNum > lastMemBarrier[inst->id.threadId]); 1576 1577 /* Remember the barrier. We only have a notion of one 1578 * barrier so this may result in some mem refs being 1579 * delayed if they are between barriers */ 1580 lastMemBarrier[inst->id.threadId] = inst->id.execSeqNum; 1581} 1582 1583void 1584LSQ::LSQRequest::makePacket() 1585{ 1586 /* Make the function idempotent */ 1587 if (packet) 1588 return; 1589 1590 // if the translation faulted, do not create a packet 1591 if (fault != NoFault) { 1592 assert(packet == NULL); 1593 return; 1594 } 1595 1596 packet = makePacketForRequest(request, isLoad, this, data); 1597 /* Null the ret data so we know not to deallocate it when the 1598 * ret is destroyed. The data now belongs to the ret and 1599 * the ret is responsible for its destruction */ 1600 data = NULL; 1601} 1602 1603std::ostream & 1604operator <<(std::ostream &os, LSQ::MemoryState state) 1605{ 1606 switch (state) { 1607 case LSQ::MemoryRunning: 1608 os << "MemoryRunning"; 1609 break; 1610 case LSQ::MemoryNeedsRetry: 1611 os << "MemoryNeedsRetry"; 1612 break; 1613 default: 1614 os << "MemoryState-" << static_cast<int>(state); 1615 break; 1616 } 1617 return os; 1618} 1619 1620void 1621LSQ::recvTimingSnoopReq(PacketPtr pkt) 1622{ 1623 /* LLSC operations in Minor can't be speculative and are executed from 1624 * the head of the requests queue. We shouldn't need to do more than 1625 * this action on snoops. */ 1626 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 1627 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) { 1628 cpu.wakeup(tid); 1629 } 1630 } 1631 1632 if (pkt->isInvalidate() || pkt->isWrite()) { 1633 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 1634 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt, 1635 cacheBlockMask); 1636 } 1637 } 1638} 1639 1640void 1641LSQ::threadSnoop(LSQRequestPtr request) 1642{ 1643 /* LLSC operations in Minor can't be speculative and are executed from 1644 * the head of the requests queue. We shouldn't need to do more than 1645 * this action on snoops. */ 1646 ThreadID req_tid = request->inst->id.threadId; 1647 PacketPtr pkt = request->packet; 1648 1649 for (ThreadID tid = 0; tid < cpu.numThreads; tid++) { 1650 if (tid != req_tid) { 1651 if (cpu.getCpuAddrMonitor(tid)->doMonitor(pkt)) { 1652 cpu.wakeup(tid); 1653 } 1654 1655 if (pkt->isInvalidate() || pkt->isWrite()) { 1656 TheISA::handleLockedSnoop(cpu.getContext(tid), pkt, 1657 cacheBlockMask); 1658 } 1659 } 1660 } 1661} 1662 1663} 1664