lsq.cc revision 10259
1/* 2 * Copyright (c) 2013-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Andrew Bardsley 38 */ 39 40#include <iomanip> 41#include <sstream> 42 43#include "arch/locked_mem.hh" 44#include "arch/mmapped_ipr.hh" 45#include "cpu/minor/cpu.hh" 46#include "cpu/minor/exec_context.hh" 47#include "cpu/minor/execute.hh" 48#include "cpu/minor/lsq.hh" 49#include "cpu/minor/pipeline.hh" 50#include "debug/Activity.hh" 51#include "debug/MinorMem.hh" 52 53namespace Minor 54{ 55 56/** Returns the offset of addr into an aligned a block of size block_size */ 57static Addr 58addrBlockOffset(Addr addr, unsigned int block_size) 59{ 60 return addr & (block_size - 1); 61} 62 63/** Returns true if the given [addr .. addr+size-1] transfer needs to be 64 * fragmented across a block size of block_size */ 65static bool 66transferNeedsBurst(Addr addr, unsigned int size, unsigned int block_size) 67{ 68 return (addrBlockOffset(addr, block_size) + size) > block_size; 69} 70 71LSQ::LSQRequest::LSQRequest(LSQ &port_, MinorDynInstPtr inst_, bool isLoad_, 72 PacketDataPtr data_, uint64_t *res_) : 73 SenderState(), 74 port(port_), 75 inst(inst_), 76 isLoad(isLoad_), 77 data(data_), 78 packet(NULL), 79 request(), 80 fault(NoFault), 81 res(res_), 82 skipped(false), 83 issuedToMemory(false), 84 state(NotIssued) 85{ } 86 87LSQ::AddrRangeCoverage 88LSQ::LSQRequest::containsAddrRangeOf( 89 Addr req1_addr, unsigned int req1_size, 90 Addr req2_addr, unsigned int req2_size) 91{ 92 /* 'end' here means the address of the byte just past the request 93 * blocks */ 94 Addr req2_end_addr = req2_addr + req2_size; 95 Addr req1_end_addr = req1_addr + req1_size; 96 97 AddrRangeCoverage ret; 98 99 if (req1_addr > req2_end_addr || req1_end_addr < req2_addr) 100 ret = NoAddrRangeCoverage; 101 else if (req1_addr <= req2_addr && req1_end_addr >= req2_end_addr) 102 ret = FullAddrRangeCoverage; 103 else 104 ret = PartialAddrRangeCoverage; 105 106 return ret; 107} 108 109LSQ::AddrRangeCoverage 110LSQ::LSQRequest::containsAddrRangeOf(LSQRequestPtr other_request) 111{ 112 return containsAddrRangeOf(request.getPaddr(), request.getSize(), 113 other_request->request.getPaddr(), other_request->request.getSize()); 114} 115 116bool 117LSQ::LSQRequest::isBarrier() 118{ 119 return inst->isInst() && inst->staticInst->isMemBarrier(); 120} 121 122bool 123LSQ::LSQRequest::needsToBeSentToStoreBuffer() 124{ 125 return state == StoreToStoreBuffer; 126} 127 128void 129LSQ::LSQRequest::setState(LSQRequestState new_state) 130{ 131 DPRINTFS(MinorMem, (&port), "Setting state from %d to %d for request:" 132 " %s\n", state, new_state, *inst); 133 state = new_state; 134} 135 136bool 137LSQ::LSQRequest::isComplete() const 138{ 139 /* @todo, There is currently only one 'completed' state. This 140 * may not be a good choice */ 141 return state == Complete; 142} 143 144void 145LSQ::LSQRequest::reportData(std::ostream &os) const 146{ 147 os << (isLoad ? 'R' : 'W') << ';'; 148 inst->reportData(os); 149 os << ';' << state; 150} 151 152std::ostream & 153operator <<(std::ostream &os, LSQ::AddrRangeCoverage coverage) 154{ 155 switch (coverage) { 156 case LSQ::PartialAddrRangeCoverage: 157 os << "PartialAddrRangeCoverage"; 158 break; 159 case LSQ::FullAddrRangeCoverage: 160 os << "FullAddrRangeCoverage"; 161 break; 162 case LSQ::NoAddrRangeCoverage: 163 os << "NoAddrRangeCoverage"; 164 break; 165 default: 166 os << "AddrRangeCoverage-" << static_cast<int>(coverage); 167 break; 168 } 169 return os; 170} 171 172std::ostream & 173operator <<(std::ostream &os, LSQ::LSQRequest::LSQRequestState state) 174{ 175 switch (state) { 176 case LSQ::LSQRequest::NotIssued: 177 os << "NotIssued"; 178 break; 179 case LSQ::LSQRequest::InTranslation: 180 os << "InTranslation"; 181 break; 182 case LSQ::LSQRequest::Translated: 183 os << "Translated"; 184 break; 185 case LSQ::LSQRequest::Failed: 186 os << "Failed"; 187 break; 188 case LSQ::LSQRequest::RequestIssuing: 189 os << "RequestIssuing"; 190 break; 191 case LSQ::LSQRequest::StoreToStoreBuffer: 192 os << "StoreToStoreBuffer"; 193 break; 194 case LSQ::LSQRequest::StoreInStoreBuffer: 195 os << "StoreInStoreBuffer"; 196 break; 197 case LSQ::LSQRequest::StoreBufferIssuing: 198 os << "StoreBufferIssuing"; 199 break; 200 case LSQ::LSQRequest::RequestNeedsRetry: 201 os << "RequestNeedsRetry"; 202 break; 203 case LSQ::LSQRequest::StoreBufferNeedsRetry: 204 os << "StoreBufferNeedsRetry"; 205 break; 206 case LSQ::LSQRequest::Complete: 207 os << "Complete"; 208 break; 209 default: 210 os << "LSQRequestState-" << static_cast<int>(state); 211 break; 212 } 213 return os; 214} 215 216void 217LSQ::clearMemBarrier(MinorDynInstPtr inst) 218{ 219 bool is_last_barrier = inst->id.execSeqNum >= lastMemBarrier; 220 221 DPRINTF(MinorMem, "Moving %s barrier out of store buffer inst: %s\n", 222 (is_last_barrier ? "last" : "a"), *inst); 223 224 if (is_last_barrier) 225 lastMemBarrier = 0; 226} 227 228void 229LSQ::SingleDataRequest::finish(Fault fault_, RequestPtr request_, 230 ThreadContext *tc, BaseTLB::Mode mode) 231{ 232 fault = fault_; 233 234 port.numAccessesInDTLB--; 235 236 DPRINTFS(MinorMem, (&port), "Received translation response for" 237 " request: %s\n", *inst); 238 239 makePacket(); 240 241 setState(Translated); 242 port.tryToSendToTransfers(this); 243 244 /* Let's try and wake up the processor for the next cycle */ 245 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 246} 247 248void 249LSQ::SingleDataRequest::startAddrTranslation() 250{ 251 ThreadContext *thread = port.cpu.getContext( 252 inst->id.threadId); 253 254 port.numAccessesInDTLB++; 255 256 setState(LSQ::LSQRequest::InTranslation); 257 258 DPRINTFS(MinorMem, (&port), "Submitting DTLB request\n"); 259 /* Submit the translation request. The response will come through 260 * finish/markDelayed on the LSQRequest as it bears the Translation 261 * interface */ 262 thread->getDTBPtr()->translateTiming( 263 &request, thread, this, (isLoad ? BaseTLB::Read : BaseTLB::Write)); 264} 265 266void 267LSQ::SingleDataRequest::retireResponse(PacketPtr packet_) 268{ 269 DPRINTFS(MinorMem, (&port), "Retiring packet\n"); 270 packet = packet_; 271 packetInFlight = false; 272 setState(Complete); 273} 274 275void 276LSQ::SplitDataRequest::finish(Fault fault_, RequestPtr request_, 277 ThreadContext *tc, BaseTLB::Mode mode) 278{ 279 fault = fault_; 280 281 port.numAccessesInDTLB--; 282 283 unsigned int M5_VAR_USED expected_fragment_index = 284 numTranslatedFragments; 285 286 numInTranslationFragments--; 287 numTranslatedFragments++; 288 289 DPRINTFS(MinorMem, (&port), "Received translation response for fragment" 290 " %d of request: %s\n", expected_fragment_index, *inst); 291 292 assert(request_ == fragmentRequests[expected_fragment_index]); 293 294 /* Wake up next cycle to get things going again in case the 295 * tryToSendToTransfers does take */ 296 port.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 297 298 if (fault != NoFault) { 299 /* tryToSendToTransfers will handle the fault */ 300 301 DPRINTFS(MinorMem, (&port), "Faulting translation for fragment:" 302 " %d of request: %s\n", 303 expected_fragment_index, *inst); 304 305 setState(Translated); 306 port.tryToSendToTransfers(this); 307 } else if (numTranslatedFragments == numFragments) { 308 makeFragmentPackets(); 309 310 setState(Translated); 311 port.tryToSendToTransfers(this); 312 } else { 313 /* Avoid calling translateTiming from within ::finish */ 314 assert(!translationEvent.scheduled()); 315 port.cpu.schedule(translationEvent, curTick()); 316 } 317} 318 319LSQ::SplitDataRequest::SplitDataRequest(LSQ &port_, MinorDynInstPtr inst_, 320 bool isLoad_, PacketDataPtr data_, uint64_t *res_) : 321 LSQRequest(port_, inst_, isLoad_, data_, res_), 322 translationEvent(*this), 323 numFragments(0), 324 numInTranslationFragments(0), 325 numTranslatedFragments(0), 326 numIssuedFragments(0), 327 numRetiredFragments(0), 328 fragmentRequests(), 329 fragmentPackets() 330{ 331 /* Don't know how many elements are needed until the request is 332 * populated by the caller. */ 333} 334 335LSQ::SplitDataRequest::~SplitDataRequest() 336{ 337 for (auto i = fragmentRequests.begin(); 338 i != fragmentRequests.end(); i++) 339 { 340 delete *i; 341 } 342 343 for (auto i = fragmentPackets.begin(); 344 i != fragmentPackets.end(); i++) 345 { 346 delete *i; 347 } 348} 349 350void 351LSQ::SplitDataRequest::makeFragmentRequests() 352{ 353 Addr base_addr = request.getVaddr(); 354 unsigned int whole_size = request.getSize(); 355 unsigned int line_width = port.lineWidth; 356 357 unsigned int fragment_size; 358 Addr fragment_addr; 359 360 /* Assume that this transfer is across potentially many block snap 361 * boundaries: 362 * 363 * | _|________|________|________|___ | 364 * | |0| 1 | 2 | 3 | 4 | | 365 * | |_|________|________|________|___| | 366 * | | | | | | 367 * 368 * The first transfer (0) can be up to lineWidth in size. 369 * All the middle transfers (1-3) are lineWidth in size 370 * The last transfer (4) can be from zero to lineWidth - 1 in size 371 */ 372 unsigned int first_fragment_offset = 373 addrBlockOffset(base_addr, line_width); 374 unsigned int last_fragment_size = 375 addrBlockOffset(base_addr + whole_size, line_width); 376 unsigned int first_fragment_size = 377 line_width - first_fragment_offset; 378 379 unsigned int middle_fragments_total_size = 380 whole_size - (first_fragment_size + last_fragment_size); 381 382 assert(addrBlockOffset(middle_fragments_total_size, line_width) == 0); 383 384 unsigned int middle_fragment_count = 385 middle_fragments_total_size / line_width; 386 387 numFragments = 1 /* first */ + middle_fragment_count + 388 (last_fragment_size == 0 ? 0 : 1); 389 390 DPRINTFS(MinorMem, (&port), "Dividing transfer into %d fragmentRequests." 391 " First fragment size: %d Last fragment size: %d\n", 392 numFragments, first_fragment_size, 393 (last_fragment_size == 0 ? line_width : last_fragment_size)); 394 395 assert(((middle_fragment_count * line_width) + 396 first_fragment_size + last_fragment_size) == whole_size); 397 398 fragment_addr = base_addr; 399 fragment_size = first_fragment_size; 400 401 /* Just past the last address in the request */ 402 Addr end_addr = base_addr + whole_size; 403 404 for (unsigned int fragment_index = 0; fragment_index < numFragments; 405 fragment_index++) 406 { 407 bool M5_VAR_USED is_last_fragment = false; 408 409 if (fragment_addr == base_addr) { 410 /* First fragment */ 411 fragment_size = first_fragment_size; 412 } else { 413 if ((fragment_addr + line_width) > end_addr) { 414 /* Adjust size of last fragment */ 415 fragment_size = end_addr - fragment_addr; 416 is_last_fragment = true; 417 } else { 418 /* Middle fragments */ 419 fragment_size = line_width; 420 } 421 } 422 423 Request *fragment = new Request(); 424 425 fragment->setThreadContext(request.contextId(), /* thread id */ 0); 426 fragment->setVirt(0 /* asid */, 427 fragment_addr, fragment_size, request.getFlags(), 428 request.masterId(), 429 request.getPC()); 430 431 DPRINTFS(MinorMem, (&port), "Generating fragment addr: 0x%x size: %d" 432 " (whole request addr: 0x%x size: %d) %s\n", 433 fragment_addr, fragment_size, base_addr, whole_size, 434 (is_last_fragment ? "last fragment" : "")); 435 436 fragment_addr += fragment_size; 437 438 fragmentRequests.push_back(fragment); 439 } 440} 441 442void 443LSQ::SplitDataRequest::makeFragmentPackets() 444{ 445 Addr base_addr = request.getVaddr(); 446 447 DPRINTFS(MinorMem, (&port), "Making packets for request: %s\n", *inst); 448 449 for (unsigned int fragment_index = 0; fragment_index < numFragments; 450 fragment_index++) 451 { 452 Request *fragment = fragmentRequests[fragment_index]; 453 454 DPRINTFS(MinorMem, (&port), "Making packet %d for request: %s" 455 " (%d, 0x%x)\n", 456 fragment_index, *inst, 457 (fragment->hasPaddr() ? "has paddr" : "no paddr"), 458 (fragment->hasPaddr() ? fragment->getPaddr() : 0)); 459 460 Addr fragment_addr = fragment->getVaddr(); 461 unsigned int fragment_size = fragment->getSize(); 462 463 uint8_t *request_data = NULL; 464 465 if (!isLoad) { 466 /* Split data for Packets. Will become the property of the 467 * outgoing Packets */ 468 request_data = new uint8_t[fragment_size]; 469 std::memcpy(request_data, data + (fragment_addr - base_addr), 470 fragment_size); 471 } 472 473 assert(fragment->hasPaddr()); 474 475 PacketPtr fragment_packet = 476 makePacketForRequest(*fragment, isLoad, this, request_data); 477 478 fragmentPackets.push_back(fragment_packet); 479 } 480 481 /* Might as well make the overall/response packet here */ 482 /* Get the physical address for the whole request/packet from the first 483 * fragment */ 484 request.setPaddr(fragmentRequests[0]->getPaddr()); 485 makePacket(); 486} 487 488void 489LSQ::SplitDataRequest::startAddrTranslation() 490{ 491 setState(LSQ::LSQRequest::InTranslation); 492 493 makeFragmentRequests(); 494 495 numInTranslationFragments = 0; 496 numTranslatedFragments = 0; 497 498 /* @todo, just do these in sequence for now with 499 * a loop of: 500 * do { 501 * sendNextFragmentToTranslation ; translateTiming ; finish 502 * } while (numTranslatedFragments != numFragments); 503 */ 504 505 /* Do first translation */ 506 sendNextFragmentToTranslation(); 507} 508 509PacketPtr 510LSQ::SplitDataRequest::getHeadPacket() 511{ 512 assert(numIssuedFragments < numFragments); 513 514 return fragmentPackets[numIssuedFragments]; 515} 516 517void 518LSQ::SplitDataRequest::stepToNextPacket() 519{ 520 assert(numIssuedFragments < numFragments); 521 522 numIssuedFragments++; 523} 524 525void 526LSQ::SplitDataRequest::retireResponse(PacketPtr response) 527{ 528 assert(numRetiredFragments < numFragments); 529 530 DPRINTFS(MinorMem, (&port), "Retiring fragment addr: 0x%x size: %d" 531 " offset: 0x%x (retired fragment num: %d) %s\n", 532 response->req->getVaddr(), response->req->getSize(), 533 request.getVaddr() - response->req->getVaddr(), 534 numRetiredFragments, 535 (fault == NoFault ? "" : fault->name())); 536 537 numRetiredFragments++; 538 539 if (skipped) { 540 /* Skip because we already knew the request had faulted or been 541 * skipped */ 542 DPRINTFS(MinorMem, (&port), "Skipping this fragment\n"); 543 } else if (response->isError()) { 544 /* Mark up the error and leave to execute to handle it */ 545 DPRINTFS(MinorMem, (&port), "Fragment has an error, skipping\n"); 546 setSkipped(); 547 packet->copyError(response); 548 } else { 549 if (isLoad) { 550 if (!data) { 551 /* For a split transfer, a Packet must be constructed 552 * to contain all returning data. This is that packet's 553 * data */ 554 data = new uint8_t[request.getSize()]; 555 } 556 557 /* Populate the portion of the overall response data represented 558 * by the response fragment */ 559 std::memcpy( 560 data + (response->req->getVaddr() - request.getVaddr()), 561 response->getPtr<uint8_t>(), 562 response->req->getSize()); 563 } 564 } 565 566 /* Complete early if we're skipping are no more in-flight accesses */ 567 if (skipped && !hasPacketsInMemSystem()) { 568 DPRINTFS(MinorMem, (&port), "Completed skipped burst\n"); 569 setState(Complete); 570 if (packet->needsResponse()) 571 packet->makeResponse(); 572 } 573 574 if (numRetiredFragments == numFragments) 575 setState(Complete); 576 577 if (!skipped && isComplete()) { 578 DPRINTFS(MinorMem, (&port), "Completed burst %d\n", packet != NULL); 579 580 DPRINTFS(MinorMem, (&port), "Retired packet isRead: %d isWrite: %d" 581 " needsResponse: %d packetSize: %s requestSize: %s responseSize:" 582 " %s\n", packet->isRead(), packet->isWrite(), 583 packet->needsResponse(), packet->getSize(), request.getSize(), 584 response->getSize()); 585 586 /* A request can become complete by several paths, this is a sanity 587 * check to make sure the packet's data is created */ 588 if (!data) { 589 data = new uint8_t[request.getSize()]; 590 } 591 592 if (isLoad) { 593 DPRINTFS(MinorMem, (&port), "Copying read data\n"); 594 std::memcpy(packet->getPtr<uint8_t>(), data, request.getSize()); 595 } 596 packet->makeResponse(); 597 } 598 599 /* Packets are all deallocated together in ~SplitLSQRequest */ 600} 601 602void 603LSQ::SplitDataRequest::sendNextFragmentToTranslation() 604{ 605 unsigned int fragment_index = numTranslatedFragments; 606 607 ThreadContext *thread = port.cpu.getContext( 608 inst->id.threadId); 609 610 DPRINTFS(MinorMem, (&port), "Submitting DTLB request for fragment: %d\n", 611 fragment_index); 612 613 port.numAccessesInDTLB++; 614 numInTranslationFragments++; 615 616 thread->getDTBPtr()->translateTiming( 617 fragmentRequests[fragment_index], thread, this, (isLoad ? 618 BaseTLB::Read : BaseTLB::Write)); 619} 620 621bool 622LSQ::StoreBuffer::canInsert() const 623{ 624 /* @todo, support store amalgamation */ 625 return slots.size() < numSlots; 626} 627 628void 629LSQ::StoreBuffer::deleteRequest(LSQRequestPtr request) 630{ 631 auto found = std::find(slots.begin(), slots.end(), request); 632 633 if (found != slots.end()) { 634 DPRINTF(MinorMem, "Deleting request: %s %s %s from StoreBuffer\n", 635 request, *found, *(request->inst)); 636 slots.erase(found); 637 638 delete request; 639 } 640} 641 642void 643LSQ::StoreBuffer::insert(LSQRequestPtr request) 644{ 645 if (!canInsert()) { 646 warn("%s: store buffer insertion without space to insert from" 647 " inst: %s\n", name(), *(request->inst)); 648 } 649 650 DPRINTF(MinorMem, "Pushing store: %s into store buffer\n", request); 651 652 numUnissuedAccesses++; 653 654 if (request->state != LSQRequest::Complete) 655 request->setState(LSQRequest::StoreInStoreBuffer); 656 657 slots.push_back(request); 658 659 /* Let's try and wake up the processor for the next cycle to step 660 * the store buffer */ 661 lsq.cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 662} 663 664LSQ::AddrRangeCoverage 665LSQ::StoreBuffer::canForwardDataToLoad(LSQRequestPtr request, 666 unsigned int &found_slot) 667{ 668 unsigned int slot_index = slots.size() - 1; 669 auto i = slots.rbegin(); 670 AddrRangeCoverage ret = NoAddrRangeCoverage; 671 672 /* Traverse the store buffer in reverse order (most to least recent) 673 * and try to find a slot whose address range overlaps this request */ 674 while (ret == NoAddrRangeCoverage && i != slots.rend()) { 675 LSQRequestPtr slot = *i; 676 677 if (slot->packet) { 678 AddrRangeCoverage coverage = slot->containsAddrRangeOf(request); 679 680 if (coverage != NoAddrRangeCoverage) { 681 DPRINTF(MinorMem, "Forwarding: slot: %d result: %s thisAddr:" 682 " 0x%x thisSize: %d slotAddr: 0x%x slotSize: %d\n", 683 slot_index, coverage, 684 request->request.getPaddr(), request->request.getSize(), 685 slot->request.getPaddr(), slot->request.getSize()); 686 687 found_slot = slot_index; 688 ret = coverage; 689 } 690 } 691 692 i++; 693 slot_index--; 694 } 695 696 return ret; 697} 698 699/** Fill the given packet with appropriate date from slot slot_number */ 700void 701LSQ::StoreBuffer::forwardStoreData(LSQRequestPtr load, 702 unsigned int slot_number) 703{ 704 assert(slot_number < slots.size()); 705 assert(load->packet); 706 assert(load->isLoad); 707 708 LSQRequestPtr store = slots[slot_number]; 709 710 assert(store->packet); 711 assert(store->containsAddrRangeOf(load) == FullAddrRangeCoverage); 712 713 Addr load_addr = load->request.getPaddr(); 714 Addr store_addr = store->request.getPaddr(); 715 Addr addr_offset = load_addr - store_addr; 716 717 unsigned int load_size = load->request.getSize(); 718 719 DPRINTF(MinorMem, "Forwarding %d bytes for addr: 0x%x from store buffer" 720 " slot: %d addr: 0x%x addressOffset: 0x%x\n", 721 load_size, load_addr, slot_number, 722 store_addr, addr_offset); 723 724 void *load_packet_data = load->packet->getPtr<void>(); 725 void *store_packet_data = store->packet->getPtr<uint8_t>() + addr_offset; 726 727 std::memcpy(load_packet_data, store_packet_data, load_size); 728} 729 730void 731LSQ::StoreBuffer::step() 732{ 733 DPRINTF(MinorMem, "StoreBuffer step numUnissuedAccesses: %d\n", 734 numUnissuedAccesses); 735 736 if (numUnissuedAccesses != 0 && lsq.state == LSQ::MemoryRunning) { 737 /* Clear all the leading barriers */ 738 while (!slots.empty() && 739 slots.front()->isComplete() && slots.front()->isBarrier()) 740 { 741 LSQRequestPtr barrier = slots.front(); 742 743 DPRINTF(MinorMem, "Clearing barrier for inst: %s\n", 744 *(barrier->inst)); 745 746 numUnissuedAccesses--; 747 lsq.clearMemBarrier(barrier->inst); 748 slots.pop_front(); 749 750 delete barrier; 751 } 752 753 auto i = slots.begin(); 754 bool issued = true; 755 unsigned int issue_count = 0; 756 757 /* Skip trying if the memory system is busy */ 758 if (lsq.state == LSQ::MemoryNeedsRetry) 759 issued = false; 760 761 /* Try to issue all stores in order starting from the head 762 * of the queue. Responses are allowed to be retired 763 * out of order */ 764 while (issued && 765 issue_count < storeLimitPerCycle && 766 lsq.canSendToMemorySystem() && 767 i != slots.end()) 768 { 769 LSQRequestPtr request = *i; 770 771 DPRINTF(MinorMem, "Considering request: %s, sentAllPackets: %d" 772 " state: %s\n", 773 *(request->inst), request->sentAllPackets(), 774 request->state); 775 776 if (request->isBarrier() && request->isComplete()) { 777 /* Give up at barriers */ 778 issued = false; 779 } else if (!(request->state == LSQRequest::StoreBufferIssuing && 780 request->sentAllPackets())) 781 { 782 DPRINTF(MinorMem, "Trying to send request: %s to memory" 783 " system\n", *(request->inst)); 784 785 if (lsq.tryToSend(request)) { 786 /* Barrier are accounted for as they are cleared from 787 * the queue, not after their transfers are complete */ 788 if (!request->isBarrier()) 789 numUnissuedAccesses--; 790 issue_count++; 791 } else { 792 /* Don't step on to the next store buffer entry if this 793 * one hasn't issued all its packets as the store 794 * buffer must still enforce ordering */ 795 issued = false; 796 } 797 } 798 i++; 799 } 800 } 801} 802 803void 804LSQ::completeMemBarrierInst(MinorDynInstPtr inst, 805 bool committed) 806{ 807 if (committed) { 808 /* Not already sent to the store buffer as a store request? */ 809 if (!inst->inStoreBuffer) { 810 /* Insert an entry into the store buffer to tick off barriers 811 * until there are none in flight */ 812 storeBuffer.insert(new BarrierDataRequest(*this, inst)); 813 } 814 } else { 815 /* Clear the barrier anyway if it wasn't actually committed */ 816 clearMemBarrier(inst); 817 } 818} 819 820void 821LSQ::StoreBuffer::minorTrace() const 822{ 823 unsigned int size = slots.size(); 824 unsigned int i = 0; 825 std::ostringstream os; 826 827 while (i < size) { 828 LSQRequestPtr request = slots[i]; 829 830 request->reportData(os); 831 832 i++; 833 if (i < numSlots) 834 os << ','; 835 } 836 837 while (i < numSlots) { 838 os << '-'; 839 840 i++; 841 if (i < numSlots) 842 os << ','; 843 } 844 845 MINORTRACE("addr=%s num_unissued_stores=%d\n", os.str(), 846 numUnissuedAccesses); 847} 848 849void 850LSQ::tryToSendToTransfers(LSQRequestPtr request) 851{ 852 if (state == MemoryNeedsRetry) { 853 DPRINTF(MinorMem, "Request needs retry, not issuing to" 854 " memory until retry arrives\n"); 855 return; 856 } 857 858 if (request->state == LSQRequest::InTranslation) { 859 DPRINTF(MinorMem, "Request still in translation, not issuing to" 860 " memory\n"); 861 return; 862 } 863 864 assert(request->state == LSQRequest::Translated || 865 request->state == LSQRequest::RequestIssuing || 866 request->state == LSQRequest::Failed || 867 request->state == LSQRequest::Complete); 868 869 if (requests.empty() || requests.front() != request) { 870 DPRINTF(MinorMem, "Request not at front of requests queue, can't" 871 " issue to memory\n"); 872 return; 873 } 874 875 if (transfers.unreservedRemainingSpace() == 0) { 876 DPRINTF(MinorMem, "No space to insert request into transfers" 877 " queue\n"); 878 return; 879 } 880 881 if (request->isComplete() || request->state == LSQRequest::Failed) { 882 DPRINTF(MinorMem, "Passing a %s transfer on to transfers" 883 " queue\n", (request->isComplete() ? "completed" : "failed")); 884 request->setState(LSQRequest::Complete); 885 request->setSkipped(); 886 moveFromRequestsToTransfers(request); 887 return; 888 } 889 890 if (!execute.instIsRightStream(request->inst)) { 891 /* Wrong stream, try to abort the transfer but only do so if 892 * there are no packets in flight */ 893 if (request->hasPacketsInMemSystem()) { 894 DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 895 " waiting for responses before aborting request\n"); 896 } else { 897 DPRINTF(MinorMem, "Request's inst. is from the wrong stream," 898 " aborting request\n"); 899 request->setState(LSQRequest::Complete); 900 request->setSkipped(); 901 moveFromRequestsToTransfers(request); 902 } 903 return; 904 } 905 906 if (request->fault != NoFault) { 907 if (request->inst->staticInst->isPrefetch()) { 908 DPRINTF(MinorMem, "Not signalling fault for faulting prefetch\n"); 909 } 910 DPRINTF(MinorMem, "Moving faulting request into the transfers" 911 " queue\n"); 912 request->setState(LSQRequest::Complete); 913 request->setSkipped(); 914 moveFromRequestsToTransfers(request); 915 return; 916 } 917 918 bool is_load = request->isLoad; 919 bool is_llsc = request->request.isLLSC(); 920 bool is_swap = request->request.isSwap(); 921 bool bufferable = !(request->request.isUncacheable() || 922 is_llsc || is_swap); 923 924 if (is_load) { 925 if (numStoresInTransfers != 0) { 926 DPRINTF(MinorMem, "Load request with stores still in transfers" 927 " queue, stalling\n"); 928 return; 929 } 930 } else { 931 /* Store. Can it be sent to the store buffer? */ 932 if (bufferable && !request->request.isMmappedIpr()) { 933 request->setState(LSQRequest::StoreToStoreBuffer); 934 moveFromRequestsToTransfers(request); 935 DPRINTF(MinorMem, "Moving store into transfers queue\n"); 936 return; 937 } 938 } 939 940 /* Check if this is the head instruction (and so must be executable as 941 * its stream sequence number was checked above) for loads which must 942 * not be speculatively issued and stores which must be issued here */ 943 if (!bufferable) { 944 if (!execute.instIsHeadInst(request->inst)) { 945 DPRINTF(MinorMem, "Memory access not the head inst., can't be" 946 " sure it can be performed, not issuing\n"); 947 return; 948 } 949 950 unsigned int forwarding_slot = 0; 951 952 if (storeBuffer.canForwardDataToLoad(request, forwarding_slot) != 953 NoAddrRangeCoverage) 954 { 955 DPRINTF(MinorMem, "Memory access can receive forwarded data" 956 " from the store buffer, need to wait for store buffer to" 957 " drain\n"); 958 return; 959 } 960 } 961 962 /* True: submit this packet to the transfers queue to be sent to the 963 * memory system. 964 * False: skip the memory and push a packet for this request onto 965 * requests */ 966 bool do_access = true; 967 968 if (!is_llsc) { 969 /* Check for match in the store buffer */ 970 if (is_load) { 971 unsigned int forwarding_slot = 0; 972 AddrRangeCoverage forwarding_result = 973 storeBuffer.canForwardDataToLoad(request, 974 forwarding_slot); 975 976 switch (forwarding_result) { 977 case FullAddrRangeCoverage: 978 /* Forward data from the store buffer into this request and 979 * repurpose this request's packet into a response packet */ 980 storeBuffer.forwardStoreData(request, forwarding_slot); 981 request->packet->makeResponse(); 982 983 /* Just move between queues, no access */ 984 do_access = false; 985 break; 986 case PartialAddrRangeCoverage: 987 DPRINTF(MinorMem, "Load partly satisfied by store buffer" 988 " data. Must wait for the store to complete\n"); 989 return; 990 break; 991 case NoAddrRangeCoverage: 992 DPRINTF(MinorMem, "No forwardable data from store buffer\n"); 993 /* Fall through to try access */ 994 break; 995 } 996 } 997 } else { 998 if (!canSendToMemorySystem()) { 999 DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 1000 return; 1001 } 1002 1003 SimpleThread &thread = *cpu.threads[request->inst->id.threadId]; 1004 1005 TheISA::PCState old_pc = thread.pcState(); 1006 ExecContext context(cpu, thread, execute, request->inst); 1007 1008 /* Handle LLSC requests and tests */ 1009 if (is_load) { 1010 TheISA::handleLockedRead(&context, &request->request); 1011 } else { 1012 do_access = TheISA::handleLockedWrite(&context, 1013 &request->request, cacheBlockMask); 1014 1015 if (!do_access) { 1016 DPRINTF(MinorMem, "Not perfoming a memory " 1017 "access for store conditional\n"); 1018 } 1019 } 1020 thread.pcState(old_pc); 1021 } 1022 1023 /* See the do_access comment above */ 1024 if (do_access) { 1025 if (!canSendToMemorySystem()) { 1026 DPRINTF(MinorMem, "Can't send request to memory system yet\n"); 1027 return; 1028 } 1029 1030 /* Remember if this is an access which can't be idly 1031 * discarded by an interrupt */ 1032 if (!bufferable) { 1033 numAccessesIssuedToMemory++; 1034 request->issuedToMemory = true; 1035 } 1036 1037 if (tryToSend(request)) 1038 moveFromRequestsToTransfers(request); 1039 } else { 1040 request->setState(LSQRequest::Complete); 1041 moveFromRequestsToTransfers(request); 1042 } 1043} 1044 1045bool 1046LSQ::tryToSend(LSQRequestPtr request) 1047{ 1048 bool ret = false; 1049 1050 if (!canSendToMemorySystem()) { 1051 DPRINTF(MinorMem, "Can't send request: %s yet, no space in memory\n", 1052 *(request->inst)); 1053 } else { 1054 PacketPtr packet = request->getHeadPacket(); 1055 1056 DPRINTF(MinorMem, "Trying to send request: %s addr: 0x%x\n", 1057 *(request->inst), packet->req->getVaddr()); 1058 1059 /* The sender state of the packet *must* be an LSQRequest 1060 * so the response can be correctly handled */ 1061 assert(packet->findNextSenderState<LSQRequest>()); 1062 1063 if (request->request.isMmappedIpr()) { 1064 ThreadContext *thread = 1065 cpu.getContext(request->request.threadId()); 1066 1067 if (request->isLoad) { 1068 DPRINTF(MinorMem, "IPR read inst: %s\n", *(request->inst)); 1069 TheISA::handleIprRead(thread, packet); 1070 } else { 1071 DPRINTF(MinorMem, "IPR write inst: %s\n", *(request->inst)); 1072 TheISA::handleIprWrite(thread, packet); 1073 } 1074 1075 request->stepToNextPacket(); 1076 ret = request->sentAllPackets(); 1077 1078 if (!ret) { 1079 DPRINTF(MinorMem, "IPR access has another packet: %s\n", 1080 *(request->inst)); 1081 } 1082 1083 if (ret) 1084 request->setState(LSQRequest::Complete); 1085 else 1086 request->setState(LSQRequest::RequestIssuing); 1087 } else if (dcachePort.sendTimingReq(packet)) { 1088 DPRINTF(MinorMem, "Sent data memory request\n"); 1089 1090 numAccessesInMemorySystem++; 1091 1092 request->stepToNextPacket(); 1093 1094 ret = request->sentAllPackets(); 1095 1096 switch (request->state) { 1097 case LSQRequest::Translated: 1098 case LSQRequest::RequestIssuing: 1099 /* Fully or partially issued a request in the transfers 1100 * queue */ 1101 request->setState(LSQRequest::RequestIssuing); 1102 break; 1103 case LSQRequest::StoreInStoreBuffer: 1104 case LSQRequest::StoreBufferIssuing: 1105 /* Fully or partially issued a request in the store 1106 * buffer */ 1107 request->setState(LSQRequest::StoreBufferIssuing); 1108 break; 1109 default: 1110 assert(false); 1111 break; 1112 } 1113 1114 state = MemoryRunning; 1115 } else { 1116 DPRINTF(MinorMem, 1117 "Sending data memory request - needs retry\n"); 1118 1119 /* Needs to be resent, wait for that */ 1120 state = MemoryNeedsRetry; 1121 retryRequest = request; 1122 1123 switch (request->state) { 1124 case LSQRequest::Translated: 1125 case LSQRequest::RequestIssuing: 1126 request->setState(LSQRequest::RequestNeedsRetry); 1127 break; 1128 case LSQRequest::StoreInStoreBuffer: 1129 case LSQRequest::StoreBufferIssuing: 1130 request->setState(LSQRequest::StoreBufferNeedsRetry); 1131 break; 1132 default: 1133 assert(false); 1134 break; 1135 } 1136 } 1137 } 1138 1139 return ret; 1140} 1141 1142void 1143LSQ::moveFromRequestsToTransfers(LSQRequestPtr request) 1144{ 1145 assert(!requests.empty() && requests.front() == request); 1146 assert(transfers.unreservedRemainingSpace() != 0); 1147 1148 /* Need to count the number of stores in the transfers 1149 * queue so that loads know when their store buffer forwarding 1150 * results will be correct (only when all those stores 1151 * have reached the store buffer) */ 1152 if (!request->isLoad) 1153 numStoresInTransfers++; 1154 1155 requests.pop(); 1156 transfers.push(request); 1157} 1158 1159bool 1160LSQ::canSendToMemorySystem() 1161{ 1162 return state == MemoryRunning && 1163 numAccessesInMemorySystem < inMemorySystemLimit; 1164} 1165 1166bool 1167LSQ::recvTimingResp(PacketPtr response) 1168{ 1169 LSQRequestPtr request = 1170 safe_cast<LSQRequestPtr>(response->popSenderState()); 1171 1172 DPRINTF(MinorMem, "Received response packet inst: %s" 1173 " addr: 0x%x cmd: %s\n", 1174 *(request->inst), response->getAddr(), 1175 response->cmd.toString()); 1176 1177 numAccessesInMemorySystem--; 1178 1179 if (response->isError()) { 1180 DPRINTF(MinorMem, "Received error response packet: %s\n", 1181 *request->inst); 1182 } 1183 1184 switch (request->state) { 1185 case LSQRequest::RequestIssuing: 1186 case LSQRequest::RequestNeedsRetry: 1187 /* Response to a request from the transfers queue */ 1188 request->retireResponse(response); 1189 1190 DPRINTF(MinorMem, "Has outstanding packets?: %d %d\n", 1191 request->hasPacketsInMemSystem(), request->isComplete()); 1192 1193 break; 1194 case LSQRequest::StoreBufferIssuing: 1195 case LSQRequest::StoreBufferNeedsRetry: 1196 /* Response to a request from the store buffer */ 1197 request->retireResponse(response); 1198 1199 /* Remove completed requests unless they are barrier (which will 1200 * need to be removed in order */ 1201 if (request->isComplete()) { 1202 if (!request->isBarrier()) { 1203 storeBuffer.deleteRequest(request); 1204 } else { 1205 DPRINTF(MinorMem, "Completed transfer for barrier: %s" 1206 " leaving the request as it is also a barrier\n", 1207 *(request->inst)); 1208 } 1209 } 1210 break; 1211 default: 1212 /* Shouldn't be allowed to receive a response from another 1213 * state */ 1214 assert(false); 1215 break; 1216 } 1217 1218 /* We go to idle even if there are more things in the requests queue 1219 * as it's the job of step to actually step us on to the next 1220 * transaction */ 1221 1222 /* Let's try and wake up the processor for the next cycle */ 1223 cpu.wakeupOnEvent(Pipeline::ExecuteStageId); 1224 1225 /* Never busy */ 1226 return true; 1227} 1228 1229void 1230LSQ::recvRetry() 1231{ 1232 DPRINTF(MinorMem, "Received retry request\n"); 1233 1234 assert(state == MemoryNeedsRetry); 1235 1236 switch (retryRequest->state) { 1237 case LSQRequest::RequestNeedsRetry: 1238 /* Retry in the requests queue */ 1239 retryRequest->setState(LSQRequest::Translated); 1240 break; 1241 case LSQRequest::StoreBufferNeedsRetry: 1242 /* Retry in the store buffer */ 1243 retryRequest->setState(LSQRequest::StoreInStoreBuffer); 1244 break; 1245 default: 1246 assert(false); 1247 } 1248 1249 /* Set state back to MemoryRunning so that the following 1250 * tryToSend can actually send. Note that this won't 1251 * allow another transfer in as tryToSend should 1252 * issue a memory request and either succeed for this 1253 * request or return the LSQ back to MemoryNeedsRetry */ 1254 state = MemoryRunning; 1255 1256 /* Try to resend the request */ 1257 if (tryToSend(retryRequest)) { 1258 /* Successfully sent, need to move the request */ 1259 switch (retryRequest->state) { 1260 case LSQRequest::RequestIssuing: 1261 /* In the requests queue */ 1262 moveFromRequestsToTransfers(retryRequest); 1263 break; 1264 case LSQRequest::StoreBufferIssuing: 1265 /* In the store buffer */ 1266 storeBuffer.numUnissuedAccesses--; 1267 break; 1268 default: 1269 assert(false); 1270 break; 1271 } 1272 } 1273 1274 retryRequest = NULL; 1275} 1276 1277LSQ::LSQ(std::string name_, std::string dcache_port_name_, 1278 MinorCPU &cpu_, Execute &execute_, 1279 unsigned int in_memory_system_limit, unsigned int line_width, 1280 unsigned int requests_queue_size, unsigned int transfers_queue_size, 1281 unsigned int store_buffer_size, 1282 unsigned int store_buffer_cycle_store_limit) : 1283 Named(name_), 1284 cpu(cpu_), 1285 execute(execute_), 1286 dcachePort(dcache_port_name_, *this, cpu_), 1287 lastMemBarrier(0), 1288 state(MemoryRunning), 1289 inMemorySystemLimit(in_memory_system_limit), 1290 lineWidth((line_width == 0 ? cpu.cacheLineSize() : line_width)), 1291 requests(name_ + ".requests", "addr", requests_queue_size), 1292 transfers(name_ + ".transfers", "addr", transfers_queue_size), 1293 storeBuffer(name_ + ".storeBuffer", 1294 *this, store_buffer_size, store_buffer_cycle_store_limit), 1295 numAccessesInMemorySystem(0), 1296 numAccessesInDTLB(0), 1297 numStoresInTransfers(0), 1298 numAccessesIssuedToMemory(0), 1299 retryRequest(NULL), 1300 cacheBlockMask(~(cpu_.cacheLineSize() - 1)) 1301{ 1302 if (in_memory_system_limit < 1) { 1303 fatal("%s: executeMaxAccessesInMemory must be >= 1 (%d)\n", name_, 1304 in_memory_system_limit); 1305 } 1306 1307 if (store_buffer_cycle_store_limit < 1) { 1308 fatal("%s: executeLSQMaxStoreBufferStoresPerCycle must be" 1309 " >= 1 (%d)\n", name_, store_buffer_cycle_store_limit); 1310 } 1311 1312 if (requests_queue_size < 1) { 1313 fatal("%s: executeLSQRequestsQueueSize must be" 1314 " >= 1 (%d)\n", name_, requests_queue_size); 1315 } 1316 1317 if (transfers_queue_size < 1) { 1318 fatal("%s: executeLSQTransfersQueueSize must be" 1319 " >= 1 (%d)\n", name_, transfers_queue_size); 1320 } 1321 1322 if (store_buffer_size < 1) { 1323 fatal("%s: executeLSQStoreBufferSize must be" 1324 " >= 1 (%d)\n", name_, store_buffer_size); 1325 } 1326 1327 if ((lineWidth & (lineWidth - 1)) != 0) { 1328 fatal("%s: lineWidth: %d must be a power of 2\n", name(), lineWidth); 1329 } 1330} 1331 1332LSQ::~LSQ() 1333{ } 1334 1335LSQ::LSQRequest::~LSQRequest() 1336{ 1337 if (packet) 1338 delete packet; 1339 if (data) 1340 delete [] data; 1341} 1342 1343/** 1344 * Step the memory access mechanism on to its next state. In reality, most 1345 * of the stepping is done by the callbacks on the LSQ but this 1346 * function is responsible for issuing memory requests lodged in the 1347 * requests queue. 1348 */ 1349void 1350LSQ::step() 1351{ 1352 /* Try to move address-translated requests between queues and issue 1353 * them */ 1354 if (!requests.empty()) 1355 tryToSendToTransfers(requests.front()); 1356 1357 storeBuffer.step(); 1358} 1359 1360LSQ::LSQRequestPtr 1361LSQ::findResponse(MinorDynInstPtr inst) 1362{ 1363 LSQ::LSQRequestPtr ret = NULL; 1364 1365 if (!transfers.empty()) { 1366 LSQRequestPtr request = transfers.front(); 1367 1368 /* Same instruction and complete access or a store that's 1369 * capable of being moved to the store buffer */ 1370 if (request->inst->id == inst->id) { 1371 if (request->isComplete() || 1372 (request->state == LSQRequest::StoreToStoreBuffer && 1373 storeBuffer.canInsert())) 1374 { 1375 ret = request; 1376 } 1377 } 1378 } 1379 1380 if (ret) { 1381 DPRINTF(MinorMem, "Found matching memory response for inst: %s\n", 1382 *inst); 1383 } else { 1384 DPRINTF(MinorMem, "No matching memory response for inst: %s\n", 1385 *inst); 1386 } 1387 1388 return ret; 1389} 1390 1391void 1392LSQ::popResponse(LSQ::LSQRequestPtr response) 1393{ 1394 assert(!transfers.empty() && transfers.front() == response); 1395 1396 transfers.pop(); 1397 1398 if (!response->isLoad) 1399 numStoresInTransfers--; 1400 1401 if (response->issuedToMemory) 1402 numAccessesIssuedToMemory--; 1403 1404 if (response->state != LSQRequest::StoreInStoreBuffer) { 1405 DPRINTF(MinorMem, "Deleting %s request: %s\n", 1406 (response->isLoad ? "load" : "store"), 1407 *(response->inst)); 1408 1409 delete response; 1410 } 1411} 1412 1413void 1414LSQ::sendStoreToStoreBuffer(LSQRequestPtr request) 1415{ 1416 assert(request->state == LSQRequest::StoreToStoreBuffer); 1417 1418 DPRINTF(MinorMem, "Sending store: %s to store buffer\n", 1419 *(request->inst)); 1420 1421 request->inst->inStoreBuffer = true; 1422 1423 storeBuffer.insert(request); 1424} 1425 1426bool 1427LSQ::isDrained() 1428{ 1429 return requests.empty() && transfers.empty() && 1430 storeBuffer.isDrained(); 1431} 1432 1433bool 1434LSQ::needsToTick() 1435{ 1436 bool ret = false; 1437 1438 if (canSendToMemorySystem()) { 1439 bool have_translated_requests = !requests.empty() && 1440 requests.front()->state != LSQRequest::InTranslation && 1441 transfers.unreservedRemainingSpace() != 0; 1442 1443 ret = have_translated_requests || 1444 storeBuffer.numUnissuedStores() != 0; 1445 } 1446 1447 if (ret) 1448 DPRINTF(Activity, "Need to tick\n"); 1449 1450 return ret; 1451} 1452 1453void 1454LSQ::pushRequest(MinorDynInstPtr inst, bool isLoad, uint8_t *data, 1455 unsigned int size, Addr addr, unsigned int flags, uint64_t *res) 1456{ 1457 bool needs_burst = transferNeedsBurst(addr, size, lineWidth); 1458 LSQRequestPtr request; 1459 1460 /* Copy given data into the request. The request will pass this to the 1461 * packet and then it will own the data */ 1462 uint8_t *request_data = NULL; 1463 1464 DPRINTF(MinorMem, "Pushing request (%s) addr: 0x%x size: %d flags:" 1465 " 0x%x%s lineWidth : 0x%x\n", 1466 (isLoad ? "load" : "store"), addr, size, flags, 1467 (needs_burst ? " (needs burst)" : ""), lineWidth); 1468 1469 if (!isLoad) { 1470 /* request_data becomes the property of a ...DataRequest (see below) 1471 * and destroyed by its destructor */ 1472 request_data = new uint8_t[size]; 1473 if (flags & Request::CACHE_BLOCK_ZERO) { 1474 /* For cache zeroing, just use zeroed data */ 1475 std::memset(request_data, 0, size); 1476 } else { 1477 std::memcpy(request_data, data, size); 1478 } 1479 } 1480 1481 if (needs_burst) { 1482 request = new SplitDataRequest( 1483 *this, inst, isLoad, request_data, res); 1484 } else { 1485 request = new SingleDataRequest( 1486 *this, inst, isLoad, request_data, res); 1487 } 1488 1489 if (inst->traceData) 1490 inst->traceData->setAddr(addr); 1491 1492 request->request.setThreadContext(cpu.cpuId(), /* thread id */ 0); 1493 request->request.setVirt(0 /* asid */, 1494 addr, size, flags, cpu.instMasterId(), 1495 /* I've no idea why we need the PC, but give it */ 1496 inst->pc.instAddr()); 1497 1498 requests.push(request); 1499 request->startAddrTranslation(); 1500} 1501 1502void 1503LSQ::pushFailedRequest(MinorDynInstPtr inst) 1504{ 1505 LSQRequestPtr request = new FailedDataRequest(*this, inst); 1506 requests.push(request); 1507} 1508 1509void 1510LSQ::minorTrace() const 1511{ 1512 MINORTRACE("state=%s in_tlb_mem=%d/%d stores_in_transfers=%d" 1513 " lastMemBarrier=%d\n", 1514 state, numAccessesInDTLB, numAccessesInMemorySystem, 1515 numStoresInTransfers, lastMemBarrier); 1516 requests.minorTrace(); 1517 transfers.minorTrace(); 1518 storeBuffer.minorTrace(); 1519} 1520 1521LSQ::StoreBuffer::StoreBuffer(std::string name_, LSQ &lsq_, 1522 unsigned int store_buffer_size, 1523 unsigned int store_limit_per_cycle) : 1524 Named(name_), lsq(lsq_), 1525 numSlots(store_buffer_size), 1526 storeLimitPerCycle(store_limit_per_cycle), 1527 slots(), 1528 numUnissuedAccesses(0) 1529{ 1530} 1531 1532PacketPtr 1533makePacketForRequest(Request &request, bool isLoad, 1534 Packet::SenderState *sender_state, PacketDataPtr data) 1535{ 1536 MemCmd command; 1537 1538 /* Make a ret with the right command type to match the request */ 1539 if (request.isLLSC()) { 1540 command = (isLoad ? MemCmd::LoadLockedReq : MemCmd::StoreCondReq); 1541 } else if (request.isSwap()) { 1542 command = MemCmd::SwapReq; 1543 } else { 1544 command = (isLoad ? MemCmd::ReadReq : MemCmd::WriteReq); 1545 } 1546 1547 PacketPtr ret = new Packet(&request, command); 1548 1549 if (sender_state) 1550 ret->pushSenderState(sender_state); 1551 1552 if (isLoad) 1553 ret->allocate(); 1554 else 1555 ret->dataDynamicArray(data); 1556 1557 return ret; 1558} 1559 1560void 1561LSQ::issuedMemBarrierInst(MinorDynInstPtr inst) 1562{ 1563 assert(inst->isInst() && inst->staticInst->isMemBarrier()); 1564 assert(inst->id.execSeqNum > lastMemBarrier); 1565 1566 /* Remember the barrier. We only have a notion of one 1567 * barrier so this may result in some mem refs being 1568 * delayed if they are between barriers */ 1569 lastMemBarrier = inst->id.execSeqNum; 1570} 1571 1572void 1573LSQ::LSQRequest::makePacket() 1574{ 1575 /* Make the function idempotent */ 1576 if (packet) 1577 return; 1578 1579 packet = makePacketForRequest(request, isLoad, this, data); 1580 /* Null the ret data so we know not to deallocate it when the 1581 * ret is destroyed. The data now belongs to the ret and 1582 * the ret is responsible for its destruction */ 1583 data = NULL; 1584} 1585 1586std::ostream & 1587operator <<(std::ostream &os, LSQ::MemoryState state) 1588{ 1589 switch (state) { 1590 case LSQ::MemoryRunning: 1591 os << "MemoryRunning"; 1592 break; 1593 case LSQ::MemoryNeedsRetry: 1594 os << "MemoryNeedsRetry"; 1595 break; 1596 default: 1597 os << "MemoryState-" << static_cast<int>(state); 1598 break; 1599 } 1600 return os; 1601} 1602 1603void 1604LSQ::recvTimingSnoopReq(PacketPtr pkt) 1605{ 1606 /* LLSC operations in Minor can't be speculative and are executed from 1607 * the head of the requests queue. We shouldn't need to do more than 1608 * this action on snoops. */ 1609 1610 /* THREAD */ 1611 TheISA::handleLockedSnoop(cpu.getContext(0), pkt, cacheBlockMask); 1612} 1613 1614} 1615