44#include "sim/system.hh" 45 46using namespace std; 47 48Sequencer * 49RubySequencerParams::create() 50{ 51 return new Sequencer(this); 52} 53 54Sequencer::Sequencer(const Params *p) 55 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this) 56{ 57 m_outstanding_count = 0; 58 59 m_instCache_ptr = p->icache; 60 m_dataCache_ptr = p->dcache; 61 m_data_cache_hit_latency = p->dcache_hit_latency; 62 m_inst_cache_hit_latency = p->icache_hit_latency; 63 m_max_outstanding_requests = p->max_outstanding_requests; 64 m_deadlock_threshold = p->deadlock_threshold; 65 66 m_coreId = p->coreid; // for tracking the two CorePair sequencers 67 assert(m_max_outstanding_requests > 0); 68 assert(m_deadlock_threshold > 0); 69 assert(m_instCache_ptr != NULL); 70 assert(m_dataCache_ptr != NULL); 71 assert(m_data_cache_hit_latency > 0); 72 assert(m_inst_cache_hit_latency > 0); 73 74 m_runningGarnetStandalone = p->garnet_standalone; 75} 76 77Sequencer::~Sequencer() 78{ 79} 80 81void 82Sequencer::wakeup() 83{ 84 assert(drainState() != DrainState::Draining); 85 86 // Check for deadlock of any of the requests 87 Cycles current_time = curCycle(); 88 89 // Check across all outstanding requests 90 int total_outstanding = 0; 91 92 RequestTable::iterator read = m_readRequestTable.begin(); 93 RequestTable::iterator read_end = m_readRequestTable.end(); 94 for (; read != read_end; ++read) { 95 SequencerRequest* request = read->second; 96 if (current_time - request->issue_time < m_deadlock_threshold) 97 continue; 98 99 panic("Possible Deadlock detected. Aborting!\n" 100 "version: %d request.paddr: 0x%x m_readRequestTable: %d " 101 "current time: %u issue_time: %d difference: %d\n", m_version, 102 request->pkt->getAddr(), m_readRequestTable.size(), 103 current_time * clockPeriod(), request->issue_time * clockPeriod(), 104 (current_time * clockPeriod()) - (request->issue_time * clockPeriod())); 105 } 106 107 RequestTable::iterator write = m_writeRequestTable.begin(); 108 RequestTable::iterator write_end = m_writeRequestTable.end(); 109 for (; write != write_end; ++write) { 110 SequencerRequest* request = write->second; 111 if (current_time - request->issue_time < m_deadlock_threshold) 112 continue; 113 114 panic("Possible Deadlock detected. Aborting!\n" 115 "version: %d request.paddr: 0x%x m_writeRequestTable: %d " 116 "current time: %u issue_time: %d difference: %d\n", m_version, 117 request->pkt->getAddr(), m_writeRequestTable.size(), 118 current_time * clockPeriod(), request->issue_time * clockPeriod(), 119 (current_time * clockPeriod()) - (request->issue_time * clockPeriod())); 120 } 121 122 total_outstanding += m_writeRequestTable.size(); 123 total_outstanding += m_readRequestTable.size(); 124 125 assert(m_outstanding_count == total_outstanding); 126 127 if (m_outstanding_count > 0) { 128 // If there are still outstanding requests, keep checking 129 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold)); 130 } 131} 132 133void Sequencer::resetStats() 134{ 135 m_latencyHist.reset(); 136 m_hitLatencyHist.reset(); 137 m_missLatencyHist.reset(); 138 for (int i = 0; i < RubyRequestType_NUM; i++) { 139 m_typeLatencyHist[i]->reset(); 140 m_hitTypeLatencyHist[i]->reset(); 141 m_missTypeLatencyHist[i]->reset(); 142 for (int j = 0; j < MachineType_NUM; j++) { 143 m_hitTypeMachLatencyHist[i][j]->reset(); 144 m_missTypeMachLatencyHist[i][j]->reset(); 145 } 146 } 147 148 for (int i = 0; i < MachineType_NUM; i++) { 149 m_missMachLatencyHist[i]->reset(); 150 m_hitMachLatencyHist[i]->reset(); 151 152 m_IssueToInitialDelayHist[i]->reset(); 153 m_InitialToForwardDelayHist[i]->reset(); 154 m_ForwardToFirstResponseDelayHist[i]->reset(); 155 m_FirstResponseToCompletionDelayHist[i]->reset(); 156 157 m_IncompleteTimes[i] = 0; 158 } 159} 160 161// Insert the request on the correct request table. Return true if 162// the entry was already present. 163RequestStatus 164Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type) 165{ 166 assert(m_outstanding_count == 167 (m_writeRequestTable.size() + m_readRequestTable.size())); 168 169 // See if we should schedule a deadlock check 170 if (!deadlockCheckEvent.scheduled() && 171 drainState() != DrainState::Draining) { 172 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold)); 173 } 174 175 Addr line_addr = makeLineAddress(pkt->getAddr()); 176 177 // Check if the line is blocked for a Locked_RMW 178 if (m_controller->isBlocked(line_addr) && 179 (request_type != RubyRequestType_Locked_RMW_Write)) { 180 // Return that this request's cache line address aliases with 181 // a prior request that locked the cache line. The request cannot 182 // proceed until the cache line is unlocked by a Locked_RMW_Write 183 return RequestStatus_Aliased; 184 } 185 186 // Create a default entry, mapping the address to NULL, the cast is 187 // there to make gcc 4.4 happy 188 RequestTable::value_type default_entry(line_addr, 189 (SequencerRequest*) NULL); 190 191 if ((request_type == RubyRequestType_ST) || 192 (request_type == RubyRequestType_RMW_Read) || 193 (request_type == RubyRequestType_RMW_Write) || 194 (request_type == RubyRequestType_Load_Linked) || 195 (request_type == RubyRequestType_Store_Conditional) || 196 (request_type == RubyRequestType_Locked_RMW_Read) || 197 (request_type == RubyRequestType_Locked_RMW_Write) || 198 (request_type == RubyRequestType_FLUSH)) { 199 200 // Check if there is any outstanding read request for the same 201 // cache line. 202 if (m_readRequestTable.count(line_addr) > 0) { 203 m_store_waiting_on_load++; 204 return RequestStatus_Aliased; 205 } 206 207 pair<RequestTable::iterator, bool> r = 208 m_writeRequestTable.insert(default_entry); 209 if (r.second) { 210 RequestTable::iterator i = r.first; 211 i->second = new SequencerRequest(pkt, request_type, curCycle()); 212 m_outstanding_count++; 213 } else { 214 // There is an outstanding write request for the cache line 215 m_store_waiting_on_store++; 216 return RequestStatus_Aliased; 217 } 218 } else { 219 // Check if there is any outstanding write request for the same 220 // cache line. 221 if (m_writeRequestTable.count(line_addr) > 0) { 222 m_load_waiting_on_store++; 223 return RequestStatus_Aliased; 224 } 225 226 pair<RequestTable::iterator, bool> r = 227 m_readRequestTable.insert(default_entry); 228 229 if (r.second) { 230 RequestTable::iterator i = r.first; 231 i->second = new SequencerRequest(pkt, request_type, curCycle()); 232 m_outstanding_count++; 233 } else { 234 // There is an outstanding read request for the cache line 235 m_load_waiting_on_load++; 236 return RequestStatus_Aliased; 237 } 238 } 239 240 m_outstandReqHist.sample(m_outstanding_count); 241 assert(m_outstanding_count == 242 (m_writeRequestTable.size() + m_readRequestTable.size())); 243 244 return RequestStatus_Ready; 245} 246 247void 248Sequencer::markRemoved() 249{ 250 m_outstanding_count--; 251 assert(m_outstanding_count == 252 m_writeRequestTable.size() + m_readRequestTable.size()); 253} 254 255void 256Sequencer::invalidateSC(Addr address) 257{ 258 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address); 259 // The controller has lost the coherence permissions, hence the lock 260 // on the cache line maintained by the cache should be cleared. 261 if (e && e->isLocked(m_version)) { 262 e->clearLocked(); 263 } 264} 265 266bool 267Sequencer::handleLlsc(Addr address, SequencerRequest* request) 268{ 269 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address); 270 if (!e) 271 return true; 272 273 // The success flag indicates whether the LLSC operation was successful. 274 // LL ops will always succeed, but SC may fail if the cache line is no 275 // longer locked. 276 bool success = true; 277 if (request->m_type == RubyRequestType_Store_Conditional) { 278 if (!e->isLocked(m_version)) { 279 // 280 // For failed SC requests, indicate the failure to the cpu by 281 // setting the extra data to zero. 282 // 283 request->pkt->req->setExtraData(0); 284 success = false; 285 } else { 286 // 287 // For successful SC requests, indicate the success to the cpu by 288 // setting the extra data to one. 289 // 290 request->pkt->req->setExtraData(1); 291 } 292 // 293 // Independent of success, all SC operations must clear the lock 294 // 295 e->clearLocked(); 296 } else if (request->m_type == RubyRequestType_Load_Linked) { 297 // 298 // Note: To fully follow Alpha LLSC semantics, should the LL clear any 299 // previously locked cache lines? 300 // 301 e->setLocked(m_version); 302 } else if (e->isLocked(m_version)) { 303 // 304 // Normal writes should clear the locked address 305 // 306 e->clearLocked(); 307 } 308 return success; 309} 310 311void 312Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type, 313 const MachineType respondingMach, 314 bool isExternalHit, Cycles issuedTime, 315 Cycles initialRequestTime, 316 Cycles forwardRequestTime, 317 Cycles firstResponseTime, Cycles completionTime) 318{ 319 m_latencyHist.sample(cycles); 320 m_typeLatencyHist[type]->sample(cycles); 321 322 if (isExternalHit) { 323 m_missLatencyHist.sample(cycles); 324 m_missTypeLatencyHist[type]->sample(cycles); 325 326 if (respondingMach != MachineType_NUM) { 327 m_missMachLatencyHist[respondingMach]->sample(cycles); 328 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles); 329 330 if ((issuedTime <= initialRequestTime) && 331 (initialRequestTime <= forwardRequestTime) && 332 (forwardRequestTime <= firstResponseTime) && 333 (firstResponseTime <= completionTime)) { 334 335 m_IssueToInitialDelayHist[respondingMach]->sample( 336 initialRequestTime - issuedTime); 337 m_InitialToForwardDelayHist[respondingMach]->sample( 338 forwardRequestTime - initialRequestTime); 339 m_ForwardToFirstResponseDelayHist[respondingMach]->sample( 340 firstResponseTime - forwardRequestTime); 341 m_FirstResponseToCompletionDelayHist[respondingMach]->sample( 342 completionTime - firstResponseTime); 343 } else { 344 m_IncompleteTimes[respondingMach]++; 345 } 346 } 347 } else { 348 m_hitLatencyHist.sample(cycles); 349 m_hitTypeLatencyHist[type]->sample(cycles); 350 351 if (respondingMach != MachineType_NUM) { 352 m_hitMachLatencyHist[respondingMach]->sample(cycles); 353 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles); 354 } 355 } 356} 357 358void 359Sequencer::writeCallback(Addr address, DataBlock& data, 360 const bool externalHit, const MachineType mach, 361 const Cycles initialRequestTime, 362 const Cycles forwardRequestTime, 363 const Cycles firstResponseTime) 364{ 365 assert(address == makeLineAddress(address)); 366 assert(m_writeRequestTable.count(makeLineAddress(address))); 367 368 RequestTable::iterator i = m_writeRequestTable.find(address); 369 assert(i != m_writeRequestTable.end()); 370 SequencerRequest* request = i->second; 371 372 m_writeRequestTable.erase(i); 373 markRemoved(); 374 375 assert((request->m_type == RubyRequestType_ST) || 376 (request->m_type == RubyRequestType_ATOMIC) || 377 (request->m_type == RubyRequestType_RMW_Read) || 378 (request->m_type == RubyRequestType_RMW_Write) || 379 (request->m_type == RubyRequestType_Load_Linked) || 380 (request->m_type == RubyRequestType_Store_Conditional) || 381 (request->m_type == RubyRequestType_Locked_RMW_Read) || 382 (request->m_type == RubyRequestType_Locked_RMW_Write) || 383 (request->m_type == RubyRequestType_FLUSH)); 384 385 // 386 // For Alpha, properly handle LL, SC, and write requests with respect to 387 // locked cache blocks. 388 // 389 // Not valid for Garnet_standalone protocl 390 // 391 bool success = true; 392 if (!m_runningGarnetStandalone) 393 success = handleLlsc(address, request); 394 395 // Handle SLICC block_on behavior for Locked_RMW accesses. NOTE: the 396 // address variable here is assumed to be a line address, so when 397 // blocking buffers, must check line addresses. 398 if (request->m_type == RubyRequestType_Locked_RMW_Read) { 399 // blockOnQueue blocks all first-level cache controller queues 400 // waiting on memory accesses for the specified address that go to 401 // the specified queue. In this case, a Locked_RMW_Write must go to 402 // the mandatory_q before unblocking the first-level controller. 403 // This will block standard loads, stores, ifetches, etc. 404 m_controller->blockOnQueue(address, m_mandatory_q_ptr); 405 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) { 406 m_controller->unblock(address); 407 } 408 409 hitCallback(request, data, success, mach, externalHit, 410 initialRequestTime, forwardRequestTime, firstResponseTime); 411} 412 413void 414Sequencer::readCallback(Addr address, DataBlock& data, 415 bool externalHit, const MachineType mach, 416 Cycles initialRequestTime, 417 Cycles forwardRequestTime, 418 Cycles firstResponseTime) 419{ 420 assert(address == makeLineAddress(address)); 421 assert(m_readRequestTable.count(makeLineAddress(address))); 422 423 RequestTable::iterator i = m_readRequestTable.find(address); 424 assert(i != m_readRequestTable.end()); 425 SequencerRequest* request = i->second; 426 427 m_readRequestTable.erase(i); 428 markRemoved(); 429 430 assert((request->m_type == RubyRequestType_LD) || 431 (request->m_type == RubyRequestType_IFETCH)); 432 433 hitCallback(request, data, true, mach, externalHit, 434 initialRequestTime, forwardRequestTime, firstResponseTime); 435} 436 437void 438Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data, 439 bool llscSuccess, 440 const MachineType mach, const bool externalHit, 441 const Cycles initialRequestTime, 442 const Cycles forwardRequestTime, 443 const Cycles firstResponseTime) 444{ 445 warn_once("Replacement policy updates recently became the responsibility " 446 "of SLICC state machines. Make sure to setMRU() near callbacks " 447 "in .sm files!"); 448 449 PacketPtr pkt = srequest->pkt; 450 Addr request_address(pkt->getAddr()); 451 RubyRequestType type = srequest->m_type; 452 Cycles issued_time = srequest->issue_time; 453 454 assert(curCycle() >= issued_time); 455 Cycles total_latency = curCycle() - issued_time; 456 457 // Profile the latency for all demand accesses. 458 recordMissLatency(total_latency, type, mach, externalHit, issued_time, 459 initialRequestTime, forwardRequestTime, 460 firstResponseTime, curCycle()); 461 462 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n", 463 curTick(), m_version, "Seq", 464 llscSuccess ? "Done" : "SC_Failed", "", "", 465 printAddress(request_address), total_latency); 466 467 // update the data unless it is a non-data-carrying flush 468 if (RubySystem::getWarmupEnabled()) { 469 data.setData(pkt->getConstPtr<uint8_t>(), 470 getOffset(request_address), pkt->getSize()); 471 } else if (!pkt->isFlush()) { 472 if ((type == RubyRequestType_LD) || 473 (type == RubyRequestType_IFETCH) || 474 (type == RubyRequestType_RMW_Read) || 475 (type == RubyRequestType_Locked_RMW_Read) || 476 (type == RubyRequestType_Load_Linked)) { 477 memcpy(pkt->getPtr<uint8_t>(), 478 data.getData(getOffset(request_address), pkt->getSize()), 479 pkt->getSize()); 480 DPRINTF(RubySequencer, "read data %s\n", data); 481 } else if (pkt->req->isSwap()) { 482 std::vector<uint8_t> overwrite_val(pkt->getSize()); 483 memcpy(&overwrite_val[0], pkt->getConstPtr<uint8_t>(), 484 pkt->getSize()); 485 memcpy(pkt->getPtr<uint8_t>(), 486 data.getData(getOffset(request_address), pkt->getSize()), 487 pkt->getSize()); 488 data.setData(&overwrite_val[0], 489 getOffset(request_address), pkt->getSize()); 490 DPRINTF(RubySequencer, "swap data %s\n", data); 491 } else { 492 data.setData(pkt->getConstPtr<uint8_t>(), 493 getOffset(request_address), pkt->getSize()); 494 DPRINTF(RubySequencer, "set data %s\n", data); 495 } 496 } 497 498 // If using the RubyTester, update the RubyTester sender state's 499 // subBlock with the recieved data. The tester will later access 500 // this state. 501 if (m_usingRubyTester) { 502 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n", 503 pkt->cmdString(), pkt->getAddr()); 504 RubyTester::SenderState* testerSenderState = 505 pkt->findNextSenderState<RubyTester::SenderState>(); 506 assert(testerSenderState); 507 testerSenderState->subBlock.mergeFrom(data); 508 } 509 510 delete srequest; 511 512 RubySystem *rs = m_ruby_system; 513 if (RubySystem::getWarmupEnabled()) { 514 assert(pkt->req); 515 delete pkt->req; 516 delete pkt; 517 rs->m_cache_recorder->enqueueNextFetchRequest(); 518 } else if (RubySystem::getCooldownEnabled()) { 519 delete pkt; 520 rs->m_cache_recorder->enqueueNextFlushRequest(); 521 } else { 522 ruby_hit_callback(pkt); 523 testDrainComplete(); 524 } 525} 526 527bool 528Sequencer::empty() const 529{ 530 return m_writeRequestTable.empty() && m_readRequestTable.empty(); 531} 532 533RequestStatus 534Sequencer::makeRequest(PacketPtr pkt) 535{ 536 if (m_outstanding_count >= m_max_outstanding_requests) { 537 return RequestStatus_BufferFull; 538 } 539 540 RubyRequestType primary_type = RubyRequestType_NULL; 541 RubyRequestType secondary_type = RubyRequestType_NULL; 542 543 if (pkt->isLLSC()) { 544 // 545 // Alpha LL/SC instructions need to be handled carefully by the cache 546 // coherence protocol to ensure they follow the proper semantics. In 547 // particular, by identifying the operations as atomic, the protocol 548 // should understand that migratory sharing optimizations should not 549 // be performed (i.e. a load between the LL and SC should not steal 550 // away exclusive permission). 551 // 552 if (pkt->isWrite()) { 553 DPRINTF(RubySequencer, "Issuing SC\n"); 554 primary_type = RubyRequestType_Store_Conditional; 555 } else { 556 DPRINTF(RubySequencer, "Issuing LL\n"); 557 assert(pkt->isRead()); 558 primary_type = RubyRequestType_Load_Linked; 559 } 560 secondary_type = RubyRequestType_ATOMIC; 561 } else if (pkt->req->isLockedRMW()) { 562 // 563 // x86 locked instructions are translated to store cache coherence 564 // requests because these requests should always be treated as read 565 // exclusive operations and should leverage any migratory sharing 566 // optimization built into the protocol. 567 // 568 if (pkt->isWrite()) { 569 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n"); 570 primary_type = RubyRequestType_Locked_RMW_Write; 571 } else { 572 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n"); 573 assert(pkt->isRead()); 574 primary_type = RubyRequestType_Locked_RMW_Read; 575 } 576 secondary_type = RubyRequestType_ST; 577 } else { 578 // 579 // To support SwapReq, we need to check isWrite() first: a SwapReq 580 // should always be treated like a write, but since a SwapReq implies 581 // both isWrite() and isRead() are true, check isWrite() first here. 582 // 583 if (pkt->isWrite()) { 584 // 585 // Note: M5 packets do not differentiate ST from RMW_Write 586 // 587 primary_type = secondary_type = RubyRequestType_ST; 588 } else if (pkt->isRead()) { 589 if (pkt->req->isInstFetch()) { 590 primary_type = secondary_type = RubyRequestType_IFETCH; 591 } else { 592 bool storeCheck = false; 593 // only X86 need the store check 594 if (system->getArch() == Arch::X86ISA) { 595 uint32_t flags = pkt->req->getFlags(); 596 storeCheck = flags & 597 (X86ISA::StoreCheck << X86ISA::FlagShift); 598 } 599 if (storeCheck) { 600 primary_type = RubyRequestType_RMW_Read; 601 secondary_type = RubyRequestType_ST; 602 } else { 603 primary_type = secondary_type = RubyRequestType_LD; 604 } 605 } 606 } else if (pkt->isFlush()) { 607 primary_type = secondary_type = RubyRequestType_FLUSH; 608 } else { 609 panic("Unsupported ruby packet type\n"); 610 } 611 } 612 613 RequestStatus status = insertRequest(pkt, primary_type); 614 if (status != RequestStatus_Ready) 615 return status; 616 617 issueRequest(pkt, secondary_type); 618 619 // TODO: issue hardware prefetches here 620 return RequestStatus_Issued; 621} 622 623void 624Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type) 625{ 626 assert(pkt != NULL); 627 ContextID proc_id = pkt->req->hasContextId() ? 628 pkt->req->contextId() : InvalidContextID; 629 630 ContextID core_id = coreId(); 631 632 // If valid, copy the pc to the ruby request 633 Addr pc = 0; 634 if (pkt->req->hasPC()) { 635 pc = pkt->req->getPC(); 636 } 637 638 // check if the packet has data as for example prefetch and flush 639 // requests do not 640 std::shared_ptr<RubyRequest> msg = 641 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(), 642 pkt->isFlush() ? 643 nullptr : pkt->getPtr<uint8_t>(), 644 pkt->getSize(), pc, secondary_type, 645 RubyAccessMode_Supervisor, pkt, 646 PrefetchBit_No, proc_id, core_id); 647 648 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n", 649 curTick(), m_version, "Seq", "Begin", "", "", 650 printAddress(msg->getPhysicalAddress()), 651 RubyRequestType_to_string(secondary_type)); 652 653 // The Sequencer currently assesses instruction and data cache hit latency 654 // for the top-level caches at the beginning of a memory access. 655 // TODO: Eventually, this latency should be moved to represent the actual 656 // cache access latency portion of the memory access. This will require 657 // changing cache controller protocol files to assess the latency on the 658 // access response path. 659 Cycles latency(0); // Initialize to zero to catch misconfigured latency 660 if (secondary_type == RubyRequestType_IFETCH) 661 latency = m_inst_cache_hit_latency; 662 else 663 latency = m_data_cache_hit_latency; 664 665 // Send the message to the cache controller 666 assert(latency > 0); 667 668 assert(m_mandatory_q_ptr != NULL); 669 m_mandatory_q_ptr->enqueue(msg, clockEdge(), cyclesToTicks(latency)); 670} 671 672template <class KEY, class VALUE> 673std::ostream & 674operator<<(ostream &out, const std::unordered_map<KEY, VALUE> &map) 675{ 676 auto i = map.begin(); 677 auto end = map.end(); 678 679 out << "["; 680 for (; i != end; ++i) 681 out << " " << i->first << "=" << i->second; 682 out << " ]"; 683 684 return out; 685} 686 687void 688Sequencer::print(ostream& out) const 689{ 690 out << "[Sequencer: " << m_version 691 << ", outstanding requests: " << m_outstanding_count 692 << ", read request table: " << m_readRequestTable 693 << ", write request table: " << m_writeRequestTable 694 << "]"; 695} 696 697// this can be called from setState whenever coherence permissions are 698// upgraded when invoked, coherence violations will be checked for the 699// given block 700void 701Sequencer::checkCoherence(Addr addr) 702{ 703#ifdef CHECK_COHERENCE 704 m_ruby_system->checkGlobalCoherenceInvariant(addr); 705#endif 706} 707 708void 709Sequencer::recordRequestType(SequencerRequestType requestType) { 710 DPRINTF(RubyStats, "Recorded statistic: %s\n", 711 SequencerRequestType_to_string(requestType)); 712} 713 714 715void 716Sequencer::evictionCallback(Addr address) 717{ 718 ruby_eviction_callback(address); 719} 720 721void 722Sequencer::regStats() 723{ 724 RubyPort::regStats(); 725 726 m_store_waiting_on_load 727 .name(name() + ".store_waiting_on_load") 728 .desc("Number of times a store aliased with a pending load") 729 .flags(Stats::nozero); 730 m_store_waiting_on_store 731 .name(name() + ".store_waiting_on_store") 732 .desc("Number of times a store aliased with a pending store") 733 .flags(Stats::nozero); 734 m_load_waiting_on_load 735 .name(name() + ".load_waiting_on_load") 736 .desc("Number of times a load aliased with a pending load") 737 .flags(Stats::nozero); 738 m_load_waiting_on_store 739 .name(name() + ".load_waiting_on_store") 740 .desc("Number of times a load aliased with a pending store") 741 .flags(Stats::nozero); 742 743 // These statistical variables are not for display. 744 // The profiler will collate these across different 745 // sequencers and display those collated statistics. 746 m_outstandReqHist.init(10); 747 m_latencyHist.init(10); 748 m_hitLatencyHist.init(10); 749 m_missLatencyHist.init(10); 750 751 for (int i = 0; i < RubyRequestType_NUM; i++) { 752 m_typeLatencyHist.push_back(new Stats::Histogram()); 753 m_typeLatencyHist[i]->init(10); 754 755 m_hitTypeLatencyHist.push_back(new Stats::Histogram()); 756 m_hitTypeLatencyHist[i]->init(10); 757 758 m_missTypeLatencyHist.push_back(new Stats::Histogram()); 759 m_missTypeLatencyHist[i]->init(10); 760 } 761 762 for (int i = 0; i < MachineType_NUM; i++) { 763 m_hitMachLatencyHist.push_back(new Stats::Histogram()); 764 m_hitMachLatencyHist[i]->init(10); 765 766 m_missMachLatencyHist.push_back(new Stats::Histogram()); 767 m_missMachLatencyHist[i]->init(10); 768 769 m_IssueToInitialDelayHist.push_back(new Stats::Histogram()); 770 m_IssueToInitialDelayHist[i]->init(10); 771 772 m_InitialToForwardDelayHist.push_back(new Stats::Histogram()); 773 m_InitialToForwardDelayHist[i]->init(10); 774 775 m_ForwardToFirstResponseDelayHist.push_back(new Stats::Histogram()); 776 m_ForwardToFirstResponseDelayHist[i]->init(10); 777 778 m_FirstResponseToCompletionDelayHist.push_back(new Stats::Histogram()); 779 m_FirstResponseToCompletionDelayHist[i]->init(10); 780 } 781 782 for (int i = 0; i < RubyRequestType_NUM; i++) { 783 m_hitTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>()); 784 m_missTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>()); 785 786 for (int j = 0; j < MachineType_NUM; j++) { 787 m_hitTypeMachLatencyHist[i].push_back(new Stats::Histogram()); 788 m_hitTypeMachLatencyHist[i][j]->init(10); 789 790 m_missTypeMachLatencyHist[i].push_back(new Stats::Histogram()); 791 m_missTypeMachLatencyHist[i][j]->init(10); 792 } 793 } 794}
| 45#include "sim/system.hh" 46 47using namespace std; 48 49Sequencer * 50RubySequencerParams::create() 51{ 52 return new Sequencer(this); 53} 54 55Sequencer::Sequencer(const Params *p) 56 : RubyPort(p), m_IncompleteTimes(MachineType_NUM), deadlockCheckEvent(this) 57{ 58 m_outstanding_count = 0; 59 60 m_instCache_ptr = p->icache; 61 m_dataCache_ptr = p->dcache; 62 m_data_cache_hit_latency = p->dcache_hit_latency; 63 m_inst_cache_hit_latency = p->icache_hit_latency; 64 m_max_outstanding_requests = p->max_outstanding_requests; 65 m_deadlock_threshold = p->deadlock_threshold; 66 67 m_coreId = p->coreid; // for tracking the two CorePair sequencers 68 assert(m_max_outstanding_requests > 0); 69 assert(m_deadlock_threshold > 0); 70 assert(m_instCache_ptr != NULL); 71 assert(m_dataCache_ptr != NULL); 72 assert(m_data_cache_hit_latency > 0); 73 assert(m_inst_cache_hit_latency > 0); 74 75 m_runningGarnetStandalone = p->garnet_standalone; 76} 77 78Sequencer::~Sequencer() 79{ 80} 81 82void 83Sequencer::wakeup() 84{ 85 assert(drainState() != DrainState::Draining); 86 87 // Check for deadlock of any of the requests 88 Cycles current_time = curCycle(); 89 90 // Check across all outstanding requests 91 int total_outstanding = 0; 92 93 RequestTable::iterator read = m_readRequestTable.begin(); 94 RequestTable::iterator read_end = m_readRequestTable.end(); 95 for (; read != read_end; ++read) { 96 SequencerRequest* request = read->second; 97 if (current_time - request->issue_time < m_deadlock_threshold) 98 continue; 99 100 panic("Possible Deadlock detected. Aborting!\n" 101 "version: %d request.paddr: 0x%x m_readRequestTable: %d " 102 "current time: %u issue_time: %d difference: %d\n", m_version, 103 request->pkt->getAddr(), m_readRequestTable.size(), 104 current_time * clockPeriod(), request->issue_time * clockPeriod(), 105 (current_time * clockPeriod()) - (request->issue_time * clockPeriod())); 106 } 107 108 RequestTable::iterator write = m_writeRequestTable.begin(); 109 RequestTable::iterator write_end = m_writeRequestTable.end(); 110 for (; write != write_end; ++write) { 111 SequencerRequest* request = write->second; 112 if (current_time - request->issue_time < m_deadlock_threshold) 113 continue; 114 115 panic("Possible Deadlock detected. Aborting!\n" 116 "version: %d request.paddr: 0x%x m_writeRequestTable: %d " 117 "current time: %u issue_time: %d difference: %d\n", m_version, 118 request->pkt->getAddr(), m_writeRequestTable.size(), 119 current_time * clockPeriod(), request->issue_time * clockPeriod(), 120 (current_time * clockPeriod()) - (request->issue_time * clockPeriod())); 121 } 122 123 total_outstanding += m_writeRequestTable.size(); 124 total_outstanding += m_readRequestTable.size(); 125 126 assert(m_outstanding_count == total_outstanding); 127 128 if (m_outstanding_count > 0) { 129 // If there are still outstanding requests, keep checking 130 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold)); 131 } 132} 133 134void Sequencer::resetStats() 135{ 136 m_latencyHist.reset(); 137 m_hitLatencyHist.reset(); 138 m_missLatencyHist.reset(); 139 for (int i = 0; i < RubyRequestType_NUM; i++) { 140 m_typeLatencyHist[i]->reset(); 141 m_hitTypeLatencyHist[i]->reset(); 142 m_missTypeLatencyHist[i]->reset(); 143 for (int j = 0; j < MachineType_NUM; j++) { 144 m_hitTypeMachLatencyHist[i][j]->reset(); 145 m_missTypeMachLatencyHist[i][j]->reset(); 146 } 147 } 148 149 for (int i = 0; i < MachineType_NUM; i++) { 150 m_missMachLatencyHist[i]->reset(); 151 m_hitMachLatencyHist[i]->reset(); 152 153 m_IssueToInitialDelayHist[i]->reset(); 154 m_InitialToForwardDelayHist[i]->reset(); 155 m_ForwardToFirstResponseDelayHist[i]->reset(); 156 m_FirstResponseToCompletionDelayHist[i]->reset(); 157 158 m_IncompleteTimes[i] = 0; 159 } 160} 161 162// Insert the request on the correct request table. Return true if 163// the entry was already present. 164RequestStatus 165Sequencer::insertRequest(PacketPtr pkt, RubyRequestType request_type) 166{ 167 assert(m_outstanding_count == 168 (m_writeRequestTable.size() + m_readRequestTable.size())); 169 170 // See if we should schedule a deadlock check 171 if (!deadlockCheckEvent.scheduled() && 172 drainState() != DrainState::Draining) { 173 schedule(deadlockCheckEvent, clockEdge(m_deadlock_threshold)); 174 } 175 176 Addr line_addr = makeLineAddress(pkt->getAddr()); 177 178 // Check if the line is blocked for a Locked_RMW 179 if (m_controller->isBlocked(line_addr) && 180 (request_type != RubyRequestType_Locked_RMW_Write)) { 181 // Return that this request's cache line address aliases with 182 // a prior request that locked the cache line. The request cannot 183 // proceed until the cache line is unlocked by a Locked_RMW_Write 184 return RequestStatus_Aliased; 185 } 186 187 // Create a default entry, mapping the address to NULL, the cast is 188 // there to make gcc 4.4 happy 189 RequestTable::value_type default_entry(line_addr, 190 (SequencerRequest*) NULL); 191 192 if ((request_type == RubyRequestType_ST) || 193 (request_type == RubyRequestType_RMW_Read) || 194 (request_type == RubyRequestType_RMW_Write) || 195 (request_type == RubyRequestType_Load_Linked) || 196 (request_type == RubyRequestType_Store_Conditional) || 197 (request_type == RubyRequestType_Locked_RMW_Read) || 198 (request_type == RubyRequestType_Locked_RMW_Write) || 199 (request_type == RubyRequestType_FLUSH)) { 200 201 // Check if there is any outstanding read request for the same 202 // cache line. 203 if (m_readRequestTable.count(line_addr) > 0) { 204 m_store_waiting_on_load++; 205 return RequestStatus_Aliased; 206 } 207 208 pair<RequestTable::iterator, bool> r = 209 m_writeRequestTable.insert(default_entry); 210 if (r.second) { 211 RequestTable::iterator i = r.first; 212 i->second = new SequencerRequest(pkt, request_type, curCycle()); 213 m_outstanding_count++; 214 } else { 215 // There is an outstanding write request for the cache line 216 m_store_waiting_on_store++; 217 return RequestStatus_Aliased; 218 } 219 } else { 220 // Check if there is any outstanding write request for the same 221 // cache line. 222 if (m_writeRequestTable.count(line_addr) > 0) { 223 m_load_waiting_on_store++; 224 return RequestStatus_Aliased; 225 } 226 227 pair<RequestTable::iterator, bool> r = 228 m_readRequestTable.insert(default_entry); 229 230 if (r.second) { 231 RequestTable::iterator i = r.first; 232 i->second = new SequencerRequest(pkt, request_type, curCycle()); 233 m_outstanding_count++; 234 } else { 235 // There is an outstanding read request for the cache line 236 m_load_waiting_on_load++; 237 return RequestStatus_Aliased; 238 } 239 } 240 241 m_outstandReqHist.sample(m_outstanding_count); 242 assert(m_outstanding_count == 243 (m_writeRequestTable.size() + m_readRequestTable.size())); 244 245 return RequestStatus_Ready; 246} 247 248void 249Sequencer::markRemoved() 250{ 251 m_outstanding_count--; 252 assert(m_outstanding_count == 253 m_writeRequestTable.size() + m_readRequestTable.size()); 254} 255 256void 257Sequencer::invalidateSC(Addr address) 258{ 259 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address); 260 // The controller has lost the coherence permissions, hence the lock 261 // on the cache line maintained by the cache should be cleared. 262 if (e && e->isLocked(m_version)) { 263 e->clearLocked(); 264 } 265} 266 267bool 268Sequencer::handleLlsc(Addr address, SequencerRequest* request) 269{ 270 AbstractCacheEntry *e = m_dataCache_ptr->lookup(address); 271 if (!e) 272 return true; 273 274 // The success flag indicates whether the LLSC operation was successful. 275 // LL ops will always succeed, but SC may fail if the cache line is no 276 // longer locked. 277 bool success = true; 278 if (request->m_type == RubyRequestType_Store_Conditional) { 279 if (!e->isLocked(m_version)) { 280 // 281 // For failed SC requests, indicate the failure to the cpu by 282 // setting the extra data to zero. 283 // 284 request->pkt->req->setExtraData(0); 285 success = false; 286 } else { 287 // 288 // For successful SC requests, indicate the success to the cpu by 289 // setting the extra data to one. 290 // 291 request->pkt->req->setExtraData(1); 292 } 293 // 294 // Independent of success, all SC operations must clear the lock 295 // 296 e->clearLocked(); 297 } else if (request->m_type == RubyRequestType_Load_Linked) { 298 // 299 // Note: To fully follow Alpha LLSC semantics, should the LL clear any 300 // previously locked cache lines? 301 // 302 e->setLocked(m_version); 303 } else if (e->isLocked(m_version)) { 304 // 305 // Normal writes should clear the locked address 306 // 307 e->clearLocked(); 308 } 309 return success; 310} 311 312void 313Sequencer::recordMissLatency(const Cycles cycles, const RubyRequestType type, 314 const MachineType respondingMach, 315 bool isExternalHit, Cycles issuedTime, 316 Cycles initialRequestTime, 317 Cycles forwardRequestTime, 318 Cycles firstResponseTime, Cycles completionTime) 319{ 320 m_latencyHist.sample(cycles); 321 m_typeLatencyHist[type]->sample(cycles); 322 323 if (isExternalHit) { 324 m_missLatencyHist.sample(cycles); 325 m_missTypeLatencyHist[type]->sample(cycles); 326 327 if (respondingMach != MachineType_NUM) { 328 m_missMachLatencyHist[respondingMach]->sample(cycles); 329 m_missTypeMachLatencyHist[type][respondingMach]->sample(cycles); 330 331 if ((issuedTime <= initialRequestTime) && 332 (initialRequestTime <= forwardRequestTime) && 333 (forwardRequestTime <= firstResponseTime) && 334 (firstResponseTime <= completionTime)) { 335 336 m_IssueToInitialDelayHist[respondingMach]->sample( 337 initialRequestTime - issuedTime); 338 m_InitialToForwardDelayHist[respondingMach]->sample( 339 forwardRequestTime - initialRequestTime); 340 m_ForwardToFirstResponseDelayHist[respondingMach]->sample( 341 firstResponseTime - forwardRequestTime); 342 m_FirstResponseToCompletionDelayHist[respondingMach]->sample( 343 completionTime - firstResponseTime); 344 } else { 345 m_IncompleteTimes[respondingMach]++; 346 } 347 } 348 } else { 349 m_hitLatencyHist.sample(cycles); 350 m_hitTypeLatencyHist[type]->sample(cycles); 351 352 if (respondingMach != MachineType_NUM) { 353 m_hitMachLatencyHist[respondingMach]->sample(cycles); 354 m_hitTypeMachLatencyHist[type][respondingMach]->sample(cycles); 355 } 356 } 357} 358 359void 360Sequencer::writeCallback(Addr address, DataBlock& data, 361 const bool externalHit, const MachineType mach, 362 const Cycles initialRequestTime, 363 const Cycles forwardRequestTime, 364 const Cycles firstResponseTime) 365{ 366 assert(address == makeLineAddress(address)); 367 assert(m_writeRequestTable.count(makeLineAddress(address))); 368 369 RequestTable::iterator i = m_writeRequestTable.find(address); 370 assert(i != m_writeRequestTable.end()); 371 SequencerRequest* request = i->second; 372 373 m_writeRequestTable.erase(i); 374 markRemoved(); 375 376 assert((request->m_type == RubyRequestType_ST) || 377 (request->m_type == RubyRequestType_ATOMIC) || 378 (request->m_type == RubyRequestType_RMW_Read) || 379 (request->m_type == RubyRequestType_RMW_Write) || 380 (request->m_type == RubyRequestType_Load_Linked) || 381 (request->m_type == RubyRequestType_Store_Conditional) || 382 (request->m_type == RubyRequestType_Locked_RMW_Read) || 383 (request->m_type == RubyRequestType_Locked_RMW_Write) || 384 (request->m_type == RubyRequestType_FLUSH)); 385 386 // 387 // For Alpha, properly handle LL, SC, and write requests with respect to 388 // locked cache blocks. 389 // 390 // Not valid for Garnet_standalone protocl 391 // 392 bool success = true; 393 if (!m_runningGarnetStandalone) 394 success = handleLlsc(address, request); 395 396 // Handle SLICC block_on behavior for Locked_RMW accesses. NOTE: the 397 // address variable here is assumed to be a line address, so when 398 // blocking buffers, must check line addresses. 399 if (request->m_type == RubyRequestType_Locked_RMW_Read) { 400 // blockOnQueue blocks all first-level cache controller queues 401 // waiting on memory accesses for the specified address that go to 402 // the specified queue. In this case, a Locked_RMW_Write must go to 403 // the mandatory_q before unblocking the first-level controller. 404 // This will block standard loads, stores, ifetches, etc. 405 m_controller->blockOnQueue(address, m_mandatory_q_ptr); 406 } else if (request->m_type == RubyRequestType_Locked_RMW_Write) { 407 m_controller->unblock(address); 408 } 409 410 hitCallback(request, data, success, mach, externalHit, 411 initialRequestTime, forwardRequestTime, firstResponseTime); 412} 413 414void 415Sequencer::readCallback(Addr address, DataBlock& data, 416 bool externalHit, const MachineType mach, 417 Cycles initialRequestTime, 418 Cycles forwardRequestTime, 419 Cycles firstResponseTime) 420{ 421 assert(address == makeLineAddress(address)); 422 assert(m_readRequestTable.count(makeLineAddress(address))); 423 424 RequestTable::iterator i = m_readRequestTable.find(address); 425 assert(i != m_readRequestTable.end()); 426 SequencerRequest* request = i->second; 427 428 m_readRequestTable.erase(i); 429 markRemoved(); 430 431 assert((request->m_type == RubyRequestType_LD) || 432 (request->m_type == RubyRequestType_IFETCH)); 433 434 hitCallback(request, data, true, mach, externalHit, 435 initialRequestTime, forwardRequestTime, firstResponseTime); 436} 437 438void 439Sequencer::hitCallback(SequencerRequest* srequest, DataBlock& data, 440 bool llscSuccess, 441 const MachineType mach, const bool externalHit, 442 const Cycles initialRequestTime, 443 const Cycles forwardRequestTime, 444 const Cycles firstResponseTime) 445{ 446 warn_once("Replacement policy updates recently became the responsibility " 447 "of SLICC state machines. Make sure to setMRU() near callbacks " 448 "in .sm files!"); 449 450 PacketPtr pkt = srequest->pkt; 451 Addr request_address(pkt->getAddr()); 452 RubyRequestType type = srequest->m_type; 453 Cycles issued_time = srequest->issue_time; 454 455 assert(curCycle() >= issued_time); 456 Cycles total_latency = curCycle() - issued_time; 457 458 // Profile the latency for all demand accesses. 459 recordMissLatency(total_latency, type, mach, externalHit, issued_time, 460 initialRequestTime, forwardRequestTime, 461 firstResponseTime, curCycle()); 462 463 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %d cycles\n", 464 curTick(), m_version, "Seq", 465 llscSuccess ? "Done" : "SC_Failed", "", "", 466 printAddress(request_address), total_latency); 467 468 // update the data unless it is a non-data-carrying flush 469 if (RubySystem::getWarmupEnabled()) { 470 data.setData(pkt->getConstPtr<uint8_t>(), 471 getOffset(request_address), pkt->getSize()); 472 } else if (!pkt->isFlush()) { 473 if ((type == RubyRequestType_LD) || 474 (type == RubyRequestType_IFETCH) || 475 (type == RubyRequestType_RMW_Read) || 476 (type == RubyRequestType_Locked_RMW_Read) || 477 (type == RubyRequestType_Load_Linked)) { 478 memcpy(pkt->getPtr<uint8_t>(), 479 data.getData(getOffset(request_address), pkt->getSize()), 480 pkt->getSize()); 481 DPRINTF(RubySequencer, "read data %s\n", data); 482 } else if (pkt->req->isSwap()) { 483 std::vector<uint8_t> overwrite_val(pkt->getSize()); 484 memcpy(&overwrite_val[0], pkt->getConstPtr<uint8_t>(), 485 pkt->getSize()); 486 memcpy(pkt->getPtr<uint8_t>(), 487 data.getData(getOffset(request_address), pkt->getSize()), 488 pkt->getSize()); 489 data.setData(&overwrite_val[0], 490 getOffset(request_address), pkt->getSize()); 491 DPRINTF(RubySequencer, "swap data %s\n", data); 492 } else { 493 data.setData(pkt->getConstPtr<uint8_t>(), 494 getOffset(request_address), pkt->getSize()); 495 DPRINTF(RubySequencer, "set data %s\n", data); 496 } 497 } 498 499 // If using the RubyTester, update the RubyTester sender state's 500 // subBlock with the recieved data. The tester will later access 501 // this state. 502 if (m_usingRubyTester) { 503 DPRINTF(RubySequencer, "hitCallback %s 0x%x using RubyTester\n", 504 pkt->cmdString(), pkt->getAddr()); 505 RubyTester::SenderState* testerSenderState = 506 pkt->findNextSenderState<RubyTester::SenderState>(); 507 assert(testerSenderState); 508 testerSenderState->subBlock.mergeFrom(data); 509 } 510 511 delete srequest; 512 513 RubySystem *rs = m_ruby_system; 514 if (RubySystem::getWarmupEnabled()) { 515 assert(pkt->req); 516 delete pkt->req; 517 delete pkt; 518 rs->m_cache_recorder->enqueueNextFetchRequest(); 519 } else if (RubySystem::getCooldownEnabled()) { 520 delete pkt; 521 rs->m_cache_recorder->enqueueNextFlushRequest(); 522 } else { 523 ruby_hit_callback(pkt); 524 testDrainComplete(); 525 } 526} 527 528bool 529Sequencer::empty() const 530{ 531 return m_writeRequestTable.empty() && m_readRequestTable.empty(); 532} 533 534RequestStatus 535Sequencer::makeRequest(PacketPtr pkt) 536{ 537 if (m_outstanding_count >= m_max_outstanding_requests) { 538 return RequestStatus_BufferFull; 539 } 540 541 RubyRequestType primary_type = RubyRequestType_NULL; 542 RubyRequestType secondary_type = RubyRequestType_NULL; 543 544 if (pkt->isLLSC()) { 545 // 546 // Alpha LL/SC instructions need to be handled carefully by the cache 547 // coherence protocol to ensure they follow the proper semantics. In 548 // particular, by identifying the operations as atomic, the protocol 549 // should understand that migratory sharing optimizations should not 550 // be performed (i.e. a load between the LL and SC should not steal 551 // away exclusive permission). 552 // 553 if (pkt->isWrite()) { 554 DPRINTF(RubySequencer, "Issuing SC\n"); 555 primary_type = RubyRequestType_Store_Conditional; 556 } else { 557 DPRINTF(RubySequencer, "Issuing LL\n"); 558 assert(pkt->isRead()); 559 primary_type = RubyRequestType_Load_Linked; 560 } 561 secondary_type = RubyRequestType_ATOMIC; 562 } else if (pkt->req->isLockedRMW()) { 563 // 564 // x86 locked instructions are translated to store cache coherence 565 // requests because these requests should always be treated as read 566 // exclusive operations and should leverage any migratory sharing 567 // optimization built into the protocol. 568 // 569 if (pkt->isWrite()) { 570 DPRINTF(RubySequencer, "Issuing Locked RMW Write\n"); 571 primary_type = RubyRequestType_Locked_RMW_Write; 572 } else { 573 DPRINTF(RubySequencer, "Issuing Locked RMW Read\n"); 574 assert(pkt->isRead()); 575 primary_type = RubyRequestType_Locked_RMW_Read; 576 } 577 secondary_type = RubyRequestType_ST; 578 } else { 579 // 580 // To support SwapReq, we need to check isWrite() first: a SwapReq 581 // should always be treated like a write, but since a SwapReq implies 582 // both isWrite() and isRead() are true, check isWrite() first here. 583 // 584 if (pkt->isWrite()) { 585 // 586 // Note: M5 packets do not differentiate ST from RMW_Write 587 // 588 primary_type = secondary_type = RubyRequestType_ST; 589 } else if (pkt->isRead()) { 590 if (pkt->req->isInstFetch()) { 591 primary_type = secondary_type = RubyRequestType_IFETCH; 592 } else { 593 bool storeCheck = false; 594 // only X86 need the store check 595 if (system->getArch() == Arch::X86ISA) { 596 uint32_t flags = pkt->req->getFlags(); 597 storeCheck = flags & 598 (X86ISA::StoreCheck << X86ISA::FlagShift); 599 } 600 if (storeCheck) { 601 primary_type = RubyRequestType_RMW_Read; 602 secondary_type = RubyRequestType_ST; 603 } else { 604 primary_type = secondary_type = RubyRequestType_LD; 605 } 606 } 607 } else if (pkt->isFlush()) { 608 primary_type = secondary_type = RubyRequestType_FLUSH; 609 } else { 610 panic("Unsupported ruby packet type\n"); 611 } 612 } 613 614 RequestStatus status = insertRequest(pkt, primary_type); 615 if (status != RequestStatus_Ready) 616 return status; 617 618 issueRequest(pkt, secondary_type); 619 620 // TODO: issue hardware prefetches here 621 return RequestStatus_Issued; 622} 623 624void 625Sequencer::issueRequest(PacketPtr pkt, RubyRequestType secondary_type) 626{ 627 assert(pkt != NULL); 628 ContextID proc_id = pkt->req->hasContextId() ? 629 pkt->req->contextId() : InvalidContextID; 630 631 ContextID core_id = coreId(); 632 633 // If valid, copy the pc to the ruby request 634 Addr pc = 0; 635 if (pkt->req->hasPC()) { 636 pc = pkt->req->getPC(); 637 } 638 639 // check if the packet has data as for example prefetch and flush 640 // requests do not 641 std::shared_ptr<RubyRequest> msg = 642 std::make_shared<RubyRequest>(clockEdge(), pkt->getAddr(), 643 pkt->isFlush() ? 644 nullptr : pkt->getPtr<uint8_t>(), 645 pkt->getSize(), pc, secondary_type, 646 RubyAccessMode_Supervisor, pkt, 647 PrefetchBit_No, proc_id, core_id); 648 649 DPRINTFR(ProtocolTrace, "%15s %3s %10s%20s %6s>%-6s %#x %s\n", 650 curTick(), m_version, "Seq", "Begin", "", "", 651 printAddress(msg->getPhysicalAddress()), 652 RubyRequestType_to_string(secondary_type)); 653 654 // The Sequencer currently assesses instruction and data cache hit latency 655 // for the top-level caches at the beginning of a memory access. 656 // TODO: Eventually, this latency should be moved to represent the actual 657 // cache access latency portion of the memory access. This will require 658 // changing cache controller protocol files to assess the latency on the 659 // access response path. 660 Cycles latency(0); // Initialize to zero to catch misconfigured latency 661 if (secondary_type == RubyRequestType_IFETCH) 662 latency = m_inst_cache_hit_latency; 663 else 664 latency = m_data_cache_hit_latency; 665 666 // Send the message to the cache controller 667 assert(latency > 0); 668 669 assert(m_mandatory_q_ptr != NULL); 670 m_mandatory_q_ptr->enqueue(msg, clockEdge(), cyclesToTicks(latency)); 671} 672 673template <class KEY, class VALUE> 674std::ostream & 675operator<<(ostream &out, const std::unordered_map<KEY, VALUE> &map) 676{ 677 auto i = map.begin(); 678 auto end = map.end(); 679 680 out << "["; 681 for (; i != end; ++i) 682 out << " " << i->first << "=" << i->second; 683 out << " ]"; 684 685 return out; 686} 687 688void 689Sequencer::print(ostream& out) const 690{ 691 out << "[Sequencer: " << m_version 692 << ", outstanding requests: " << m_outstanding_count 693 << ", read request table: " << m_readRequestTable 694 << ", write request table: " << m_writeRequestTable 695 << "]"; 696} 697 698// this can be called from setState whenever coherence permissions are 699// upgraded when invoked, coherence violations will be checked for the 700// given block 701void 702Sequencer::checkCoherence(Addr addr) 703{ 704#ifdef CHECK_COHERENCE 705 m_ruby_system->checkGlobalCoherenceInvariant(addr); 706#endif 707} 708 709void 710Sequencer::recordRequestType(SequencerRequestType requestType) { 711 DPRINTF(RubyStats, "Recorded statistic: %s\n", 712 SequencerRequestType_to_string(requestType)); 713} 714 715 716void 717Sequencer::evictionCallback(Addr address) 718{ 719 ruby_eviction_callback(address); 720} 721 722void 723Sequencer::regStats() 724{ 725 RubyPort::regStats(); 726 727 m_store_waiting_on_load 728 .name(name() + ".store_waiting_on_load") 729 .desc("Number of times a store aliased with a pending load") 730 .flags(Stats::nozero); 731 m_store_waiting_on_store 732 .name(name() + ".store_waiting_on_store") 733 .desc("Number of times a store aliased with a pending store") 734 .flags(Stats::nozero); 735 m_load_waiting_on_load 736 .name(name() + ".load_waiting_on_load") 737 .desc("Number of times a load aliased with a pending load") 738 .flags(Stats::nozero); 739 m_load_waiting_on_store 740 .name(name() + ".load_waiting_on_store") 741 .desc("Number of times a load aliased with a pending store") 742 .flags(Stats::nozero); 743 744 // These statistical variables are not for display. 745 // The profiler will collate these across different 746 // sequencers and display those collated statistics. 747 m_outstandReqHist.init(10); 748 m_latencyHist.init(10); 749 m_hitLatencyHist.init(10); 750 m_missLatencyHist.init(10); 751 752 for (int i = 0; i < RubyRequestType_NUM; i++) { 753 m_typeLatencyHist.push_back(new Stats::Histogram()); 754 m_typeLatencyHist[i]->init(10); 755 756 m_hitTypeLatencyHist.push_back(new Stats::Histogram()); 757 m_hitTypeLatencyHist[i]->init(10); 758 759 m_missTypeLatencyHist.push_back(new Stats::Histogram()); 760 m_missTypeLatencyHist[i]->init(10); 761 } 762 763 for (int i = 0; i < MachineType_NUM; i++) { 764 m_hitMachLatencyHist.push_back(new Stats::Histogram()); 765 m_hitMachLatencyHist[i]->init(10); 766 767 m_missMachLatencyHist.push_back(new Stats::Histogram()); 768 m_missMachLatencyHist[i]->init(10); 769 770 m_IssueToInitialDelayHist.push_back(new Stats::Histogram()); 771 m_IssueToInitialDelayHist[i]->init(10); 772 773 m_InitialToForwardDelayHist.push_back(new Stats::Histogram()); 774 m_InitialToForwardDelayHist[i]->init(10); 775 776 m_ForwardToFirstResponseDelayHist.push_back(new Stats::Histogram()); 777 m_ForwardToFirstResponseDelayHist[i]->init(10); 778 779 m_FirstResponseToCompletionDelayHist.push_back(new Stats::Histogram()); 780 m_FirstResponseToCompletionDelayHist[i]->init(10); 781 } 782 783 for (int i = 0; i < RubyRequestType_NUM; i++) { 784 m_hitTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>()); 785 m_missTypeMachLatencyHist.push_back(std::vector<Stats::Histogram *>()); 786 787 for (int j = 0; j < MachineType_NUM; j++) { 788 m_hitTypeMachLatencyHist[i].push_back(new Stats::Histogram()); 789 m_hitTypeMachLatencyHist[i][j]->init(10); 790 791 m_missTypeMachLatencyHist[i].push_back(new Stats::Histogram()); 792 m_missTypeMachLatencyHist[i][j]->init(10); 793 } 794 } 795}
|