65MSHR::MSHR() : downstreamPending(false), 66 pendingModified(false), 67 postInvalidate(false), postDowngrade(false), 68 isForward(false) 69{ 70} 71 72MSHR::TargetList::TargetList() 73 : needsWritable(false), hasUpgrade(false), allocOnFill(false) 74{} 75 76 77void 78MSHR::TargetList::updateFlags(PacketPtr pkt, Target::Source source, 79 bool alloc_on_fill) 80{ 81 if (source != Target::FromSnoop) { 82 if (pkt->needsWritable()) { 83 needsWritable = true; 84 } 85 86 // StoreCondReq is effectively an upgrade if it's in an MSHR 87 // since it would have been failed already if we didn't have a 88 // read-only copy 89 if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 90 hasUpgrade = true; 91 } 92 93 // potentially re-evaluate whether we should allocate on a fill or 94 // not 95 allocOnFill = allocOnFill || alloc_on_fill; 96 } 97} 98 99void 100MSHR::TargetList::populateFlags() 101{ 102 resetFlags(); 103 for (auto& t: *this) { 104 updateFlags(t.pkt, t.source, t.allocOnFill); 105 } 106} 107 108inline void 109MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 110 Counter order, Target::Source source, bool markPending, 111 bool alloc_on_fill) 112{ 113 updateFlags(pkt, source, alloc_on_fill); 114 if (markPending) { 115 // Iterate over the SenderState stack and see if we find 116 // an MSHR entry. If we do, set the downstreamPending 117 // flag. Otherwise, do nothing. 118 MSHR *mshr = pkt->findNextSenderState<MSHR>(); 119 if (mshr != nullptr) { 120 assert(!mshr->downstreamPending); 121 mshr->downstreamPending = true; 122 } else { 123 // No need to clear downstreamPending later 124 markPending = false; 125 } 126 } 127 128 emplace_back(pkt, readyTime, order, source, markPending, alloc_on_fill); 129} 130 131 132static void 133replaceUpgrade(PacketPtr pkt) 134{ 135 // remember if the current packet has data allocated 136 bool has_data = pkt->hasData() || pkt->hasRespData(); 137 138 if (pkt->cmd == MemCmd::UpgradeReq) { 139 pkt->cmd = MemCmd::ReadExReq; 140 DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); 141 } else if (pkt->cmd == MemCmd::SCUpgradeReq) { 142 pkt->cmd = MemCmd::SCUpgradeFailReq; 143 DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); 144 } else if (pkt->cmd == MemCmd::StoreCondReq) { 145 pkt->cmd = MemCmd::StoreCondFailReq; 146 DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n"); 147 } 148 149 if (!has_data) { 150 // there is no sensible way of setting the data field if the 151 // new command actually would carry data 152 assert(!pkt->hasData()); 153 154 if (pkt->hasRespData()) { 155 // we went from a packet that had no data (neither request, 156 // nor response), to one that does, and therefore we need to 157 // actually allocate space for the data payload 158 pkt->allocate(); 159 } 160 } 161} 162 163 164void 165MSHR::TargetList::replaceUpgrades() 166{ 167 if (!hasUpgrade) 168 return; 169 170 for (auto& t : *this) { 171 replaceUpgrade(t.pkt); 172 } 173 174 hasUpgrade = false; 175} 176 177 178void 179MSHR::TargetList::clearDownstreamPending() 180{ 181 for (auto& t : *this) { 182 if (t.markedPending) { 183 // Iterate over the SenderState stack and see if we find 184 // an MSHR entry. If we find one, clear the 185 // downstreamPending flag by calling 186 // clearDownstreamPending(). This recursively clears the 187 // downstreamPending flag in all caches this packet has 188 // passed through. 189 MSHR *mshr = t.pkt->findNextSenderState<MSHR>(); 190 if (mshr != nullptr) { 191 mshr->clearDownstreamPending(); 192 } 193 t.markedPending = false; 194 } 195 } 196} 197 198 199bool 200MSHR::TargetList::checkFunctional(PacketPtr pkt) 201{ 202 for (auto& t : *this) { 203 if (pkt->checkFunctional(t.pkt)) { 204 return true; 205 } 206 } 207 208 return false; 209} 210 211 212void 213MSHR::TargetList::print(std::ostream &os, int verbosity, 214 const std::string &prefix) const 215{ 216 for (auto& t : *this) { 217 const char *s; 218 switch (t.source) { 219 case Target::FromCPU: 220 s = "FromCPU"; 221 break; 222 case Target::FromSnoop: 223 s = "FromSnoop"; 224 break; 225 case Target::FromPrefetcher: 226 s = "FromPrefetcher"; 227 break; 228 default: 229 s = ""; 230 break; 231 } 232 ccprintf(os, "%s%s: ", prefix, s); 233 t.pkt->print(os, verbosity, ""); 234 ccprintf(os, "\n"); 235 } 236} 237 238 239void 240MSHR::allocate(Addr blk_addr, unsigned blk_size, PacketPtr target, 241 Tick when_ready, Counter _order, bool alloc_on_fill) 242{ 243 blkAddr = blk_addr; 244 blkSize = blk_size; 245 isSecure = target->isSecure(); 246 readyTime = when_ready; 247 order = _order; 248 assert(target); 249 isForward = false; 250 _isUncacheable = target->req->isUncacheable(); 251 inService = false; 252 downstreamPending = false; 253 assert(targets.isReset()); 254 // Don't know of a case where we would allocate a new MSHR for a 255 // snoop (mem-side request), so set source according to request here 256 Target::Source source = (target->cmd == MemCmd::HardPFReq) ? 257 Target::FromPrefetcher : Target::FromCPU; 258 targets.add(target, when_ready, _order, source, true, alloc_on_fill); 259 assert(deferredTargets.isReset()); 260} 261 262 263void 264MSHR::clearDownstreamPending() 265{ 266 assert(downstreamPending); 267 downstreamPending = false; 268 // recursively clear flag on any MSHRs we will be forwarding 269 // responses to 270 targets.clearDownstreamPending(); 271} 272 273void 274MSHR::markInService(bool pending_modified_resp) 275{ 276 assert(!inService); 277 278 inService = true; 279 pendingModified = targets.needsWritable || pending_modified_resp; 280 postInvalidate = postDowngrade = false; 281 282 if (!downstreamPending) { 283 // let upstream caches know that the request has made it to a 284 // level where it's going to get a response 285 targets.clearDownstreamPending(); 286 } 287} 288 289 290void 291MSHR::deallocate() 292{ 293 assert(targets.empty()); 294 targets.resetFlags(); 295 assert(deferredTargets.isReset()); 296 inService = false; 297} 298 299/* 300 * Adds a target to an MSHR 301 */ 302void 303MSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order, 304 bool alloc_on_fill) 305{ 306 // assume we'd never issue a prefetch when we've got an 307 // outstanding miss 308 assert(pkt->cmd != MemCmd::HardPFReq); 309 310 // uncacheable accesses always allocate a new MSHR, and cacheable 311 // accesses ignore any uncacheable MSHRs, thus we should never 312 // have targets addded if originally allocated uncacheable 313 assert(!_isUncacheable); 314 315 // if there's a request already in service for this MSHR, we will 316 // have to defer the new target until after the response if any of 317 // the following are true: 318 // - there are other targets already deferred 319 // - there's a pending invalidate to be applied after the response 320 // comes back (but before this target is processed) 321 // - the MSHR's first (and only) non-deferred target is a cache 322 // maintenance packet 323 // - the new target is a cache maintenance packet (this is probably 324 // overly conservative but certainly safe) 325 // - this target requires a writable block and either we're not 326 // getting a writable block back or we have already snooped 327 // another read request that will downgrade our writable block 328 // to non-writable (Shared or Owned) 329 PacketPtr tgt_pkt = targets.front().pkt; 330 if (pkt->req->isCacheMaintenance() || 331 tgt_pkt->req->isCacheMaintenance() || 332 !deferredTargets.empty() || 333 (inService && 334 (hasPostInvalidate() || 335 (pkt->needsWritable() && 336 (!isPendingModified() || hasPostDowngrade() || isForward))))) { 337 // need to put on deferred list 338 if (inService && hasPostInvalidate()) 339 replaceUpgrade(pkt); 340 deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true, 341 alloc_on_fill); 342 } else { 343 // No request outstanding, or still OK to append to 344 // outstanding request: append to regular target list. Only 345 // mark pending if current request hasn't been issued yet 346 // (isn't in service). 347 targets.add(pkt, whenReady, _order, Target::FromCPU, !inService, 348 alloc_on_fill); 349 } 350} 351 352bool 353MSHR::handleSnoop(PacketPtr pkt, Counter _order) 354{ 355 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print()); 356 357 // when we snoop packets the needsWritable and isInvalidate flags 358 // should always be the same, however, this assumes that we never 359 // snoop writes as they are currently not marked as invalidations 360 panic_if((pkt->needsWritable() != pkt->isInvalidate()) && 361 !pkt->req->isCacheMaintenance(), 362 "%s got snoop %s where needsWritable, " 363 "does not match isInvalidate", name(), pkt->print()); 364 365 if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 366 // Request has not been issued yet, or it's been issued 367 // locally but is buffered unissued at some downstream cache 368 // which is forwarding us this snoop. Either way, the packet 369 // we're snooping logically precedes this MSHR's request, so 370 // the snoop has no impact on the MSHR, but must be processed 371 // in the standard way by the cache. The only exception is 372 // that if we're an L2+ cache buffering an UpgradeReq from a 373 // higher-level cache, and the snoop is invalidating, then our 374 // buffered upgrades must be converted to read exclusives, 375 // since the upper-level cache no longer has a valid copy. 376 // That is, even though the upper-level cache got out on its 377 // local bus first, some other invalidating transaction 378 // reached the global bus before the upgrade did. 379 if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 380 targets.replaceUpgrades(); 381 deferredTargets.replaceUpgrades(); 382 } 383 384 return false; 385 } 386 387 // From here on down, the request issued by this MSHR logically 388 // precedes the request we're snooping. 389 if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 390 // snooped request still precedes the re-request we'll have to 391 // issue for deferred targets, if any... 392 deferredTargets.replaceUpgrades(); 393 } 394 395 PacketPtr tgt_pkt = targets.front().pkt; 396 if (hasPostInvalidate() || tgt_pkt->req->isCacheInvalidate()) { 397 // a prior snoop has already appended an invalidation or a 398 // cache invalidation operation is in progress, so logically 399 // we don't have the block anymore; no need for further 400 // snooping. 401 return true; 402 } 403 404 if (isPendingModified() || pkt->isInvalidate()) { 405 // We need to save and replay the packet in two cases: 406 // 1. We're awaiting a writable copy (Modified or Exclusive), 407 // so this MSHR is the orgering point, and we need to respond 408 // after we receive data. 409 // 2. It's an invalidation (e.g., UpgradeReq), and we need 410 // to forward the snoop up the hierarchy after the current 411 // transaction completes. 412 413 // Start by determining if we will eventually respond or not, 414 // matching the conditions checked in Cache::handleSnoop 415 bool will_respond = isPendingModified() && pkt->needsResponse() && 416 !pkt->isClean(); 417 418 // The packet we are snooping may be deleted by the time we 419 // actually process the target, and we consequently need to 420 // save a copy here. Clear flags and also allocate new data as 421 // the original packet data storage may have been deleted by 422 // the time we get to process this packet. In the cases where 423 // we are not responding after handling the snoop we also need 424 // to create a copy of the request to be on the safe side. In 425 // the latter case the cache is responsible for deleting both 426 // the packet and the request as part of handling the deferred 427 // snoop. 428 PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : 429 new Packet(new Request(*pkt->req), pkt->cmd, blkSize, pkt->id); 430 431 if (will_respond) { 432 // we are the ordering point, and will consequently 433 // respond, and depending on whether the packet 434 // needsWritable or not we either pass a Shared line or a 435 // Modified line 436 pkt->setCacheResponding(); 437 438 // inform the cache hierarchy that this cache had the line 439 // in the Modified state, even if the response is passed 440 // as Shared (and thus non-writable) 441 pkt->setResponderHadWritable(); 442 443 // in the case of an uncacheable request there is no need 444 // to set the responderHadWritable flag, but since the 445 // recipient does not care there is no harm in doing so 446 } 447 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, 448 downstreamPending && targets.needsWritable, false); 449 450 if (pkt->needsWritable() || pkt->isInvalidate()) { 451 // This transaction will take away our pending copy 452 postInvalidate = true; 453 } 454 455 if (isPendingModified() && pkt->isClean()) { 456 pkt->setSatisfied(); 457 } 458 } 459 460 if (!pkt->needsWritable() && !pkt->req->isUncacheable()) { 461 // This transaction will get a read-shared copy, downgrading 462 // our copy if we had a writable one 463 postDowngrade = true; 464 // make sure that any downstream cache does not respond with a 465 // writable (and dirty) copy even if it has one, unless it was 466 // explicitly asked for one 467 pkt->setHasSharers(); 468 } 469 470 return true; 471} 472 473MSHR::TargetList 474MSHR::extractServiceableTargets(PacketPtr pkt) 475{ 476 TargetList ready_targets; 477 // If the downstream MSHR got an invalidation request then we only 478 // service the first of the FromCPU targets and any other 479 // non-FromCPU target. This way the remaining FromCPU targets 480 // issue a new request and get a fresh copy of the block and we 481 // avoid memory consistency violations. 482 if (pkt->cmd == MemCmd::ReadRespWithInvalidate) { 483 auto it = targets.begin(); 484 assert((it->source == Target::FromCPU) || 485 (it->source == Target::FromPrefetcher)); 486 ready_targets.push_back(*it); 487 it = targets.erase(it); 488 while (it != targets.end()) { 489 if (it->source == Target::FromCPU) { 490 it++; 491 } else { 492 assert(it->source == Target::FromSnoop); 493 ready_targets.push_back(*it); 494 it = targets.erase(it); 495 } 496 } 497 ready_targets.populateFlags(); 498 } else { 499 std::swap(ready_targets, targets); 500 } 501 targets.populateFlags(); 502 503 return ready_targets; 504} 505 506bool 507MSHR::promoteDeferredTargets() 508{ 509 if (targets.empty() && deferredTargets.empty()) { 510 // nothing to promote 511 return false; 512 } 513 514 // the deferred targets can be generally promoted unless they 515 // contain a cache maintenance request 516 517 // find the first target that is a cache maintenance request 518 auto it = std::find_if(deferredTargets.begin(), deferredTargets.end(), 519 [](MSHR::Target &t) { 520 return t.pkt->req->isCacheMaintenance(); 521 }); 522 if (it == deferredTargets.begin()) { 523 // if the first deferred target is a cache maintenance packet 524 // then we can promote provided the targets list is empty and 525 // we can service it on its own 526 if (targets.empty()) { 527 targets.splice(targets.end(), deferredTargets, it); 528 } 529 } else { 530 // if a cache maintenance operation exists, we promote all the 531 // deferred targets that precede it, or all deferred targets 532 // otherwise 533 targets.splice(targets.end(), deferredTargets, 534 deferredTargets.begin(), it); 535 } 536 537 deferredTargets.populateFlags(); 538 targets.populateFlags(); 539 order = targets.front().order; 540 readyTime = std::max(curTick(), targets.front().readyTime); 541 542 return true; 543} 544 545 546void 547MSHR::promoteWritable() 548{ 549 if (deferredTargets.needsWritable && 550 !(hasPostInvalidate() || hasPostDowngrade())) { 551 // We got a writable response, but we have deferred targets 552 // which are waiting to request a writable copy (not because 553 // of a pending invalidate). This can happen if the original 554 // request was for a read-only block, but we got a writable 555 // response anyway. Since we got the writable copy there's no 556 // need to defer the targets, so move them up to the regular 557 // target list. 558 assert(!targets.needsWritable); 559 targets.needsWritable = true; 560 // if any of the deferred targets were upper-level cache 561 // requests marked downstreamPending, need to clear that 562 assert(!downstreamPending); // not pending here anymore 563 deferredTargets.clearDownstreamPending(); 564 // this clears out deferredTargets too 565 targets.splice(targets.end(), deferredTargets); 566 deferredTargets.resetFlags(); 567 } 568} 569 570 571bool 572MSHR::checkFunctional(PacketPtr pkt) 573{ 574 // For printing, we treat the MSHR as a whole as single entity. 575 // For other requests, we iterate over the individual targets 576 // since that's where the actual data lies. 577 if (pkt->isPrint()) { 578 pkt->checkFunctional(this, blkAddr, isSecure, blkSize, nullptr); 579 return false; 580 } else { 581 return (targets.checkFunctional(pkt) || 582 deferredTargets.checkFunctional(pkt)); 583 } 584} 585 586bool 587MSHR::sendPacket(Cache &cache) 588{ 589 return cache.sendMSHRQueuePacket(this); 590} 591 592void 593MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 594{ 595 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", 596 prefix, blkAddr, blkAddr + blkSize - 1, 597 isSecure ? "s" : "ns", 598 isForward ? "Forward" : "", 599 allocOnFill() ? "AllocOnFill" : "", 600 needsWritable() ? "Wrtbl" : "", 601 _isUncacheable ? "Unc" : "", 602 inService ? "InSvc" : "", 603 downstreamPending ? "DwnPend" : "", 604 postInvalidate ? "PostInv" : "", 605 postDowngrade ? "PostDowngr" : ""); 606 607 if (!targets.empty()) { 608 ccprintf(os, "%s Targets:\n", prefix); 609 targets.print(os, verbosity, prefix + " "); 610 } 611 if (!deferredTargets.empty()) { 612 ccprintf(os, "%s Deferred Targets:\n", prefix); 613 deferredTargets.print(os, verbosity, prefix + " "); 614 } 615} 616 617std::string 618MSHR::print() const 619{
| 63MSHR::MSHR() : downstreamPending(false), 64 pendingModified(false), 65 postInvalidate(false), postDowngrade(false), 66 isForward(false) 67{ 68} 69 70MSHR::TargetList::TargetList() 71 : needsWritable(false), hasUpgrade(false), allocOnFill(false) 72{} 73 74 75void 76MSHR::TargetList::updateFlags(PacketPtr pkt, Target::Source source, 77 bool alloc_on_fill) 78{ 79 if (source != Target::FromSnoop) { 80 if (pkt->needsWritable()) { 81 needsWritable = true; 82 } 83 84 // StoreCondReq is effectively an upgrade if it's in an MSHR 85 // since it would have been failed already if we didn't have a 86 // read-only copy 87 if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 88 hasUpgrade = true; 89 } 90 91 // potentially re-evaluate whether we should allocate on a fill or 92 // not 93 allocOnFill = allocOnFill || alloc_on_fill; 94 } 95} 96 97void 98MSHR::TargetList::populateFlags() 99{ 100 resetFlags(); 101 for (auto& t: *this) { 102 updateFlags(t.pkt, t.source, t.allocOnFill); 103 } 104} 105 106inline void 107MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 108 Counter order, Target::Source source, bool markPending, 109 bool alloc_on_fill) 110{ 111 updateFlags(pkt, source, alloc_on_fill); 112 if (markPending) { 113 // Iterate over the SenderState stack and see if we find 114 // an MSHR entry. If we do, set the downstreamPending 115 // flag. Otherwise, do nothing. 116 MSHR *mshr = pkt->findNextSenderState<MSHR>(); 117 if (mshr != nullptr) { 118 assert(!mshr->downstreamPending); 119 mshr->downstreamPending = true; 120 } else { 121 // No need to clear downstreamPending later 122 markPending = false; 123 } 124 } 125 126 emplace_back(pkt, readyTime, order, source, markPending, alloc_on_fill); 127} 128 129 130static void 131replaceUpgrade(PacketPtr pkt) 132{ 133 // remember if the current packet has data allocated 134 bool has_data = pkt->hasData() || pkt->hasRespData(); 135 136 if (pkt->cmd == MemCmd::UpgradeReq) { 137 pkt->cmd = MemCmd::ReadExReq; 138 DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); 139 } else if (pkt->cmd == MemCmd::SCUpgradeReq) { 140 pkt->cmd = MemCmd::SCUpgradeFailReq; 141 DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); 142 } else if (pkt->cmd == MemCmd::StoreCondReq) { 143 pkt->cmd = MemCmd::StoreCondFailReq; 144 DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n"); 145 } 146 147 if (!has_data) { 148 // there is no sensible way of setting the data field if the 149 // new command actually would carry data 150 assert(!pkt->hasData()); 151 152 if (pkt->hasRespData()) { 153 // we went from a packet that had no data (neither request, 154 // nor response), to one that does, and therefore we need to 155 // actually allocate space for the data payload 156 pkt->allocate(); 157 } 158 } 159} 160 161 162void 163MSHR::TargetList::replaceUpgrades() 164{ 165 if (!hasUpgrade) 166 return; 167 168 for (auto& t : *this) { 169 replaceUpgrade(t.pkt); 170 } 171 172 hasUpgrade = false; 173} 174 175 176void 177MSHR::TargetList::clearDownstreamPending() 178{ 179 for (auto& t : *this) { 180 if (t.markedPending) { 181 // Iterate over the SenderState stack and see if we find 182 // an MSHR entry. If we find one, clear the 183 // downstreamPending flag by calling 184 // clearDownstreamPending(). This recursively clears the 185 // downstreamPending flag in all caches this packet has 186 // passed through. 187 MSHR *mshr = t.pkt->findNextSenderState<MSHR>(); 188 if (mshr != nullptr) { 189 mshr->clearDownstreamPending(); 190 } 191 t.markedPending = false; 192 } 193 } 194} 195 196 197bool 198MSHR::TargetList::checkFunctional(PacketPtr pkt) 199{ 200 for (auto& t : *this) { 201 if (pkt->checkFunctional(t.pkt)) { 202 return true; 203 } 204 } 205 206 return false; 207} 208 209 210void 211MSHR::TargetList::print(std::ostream &os, int verbosity, 212 const std::string &prefix) const 213{ 214 for (auto& t : *this) { 215 const char *s; 216 switch (t.source) { 217 case Target::FromCPU: 218 s = "FromCPU"; 219 break; 220 case Target::FromSnoop: 221 s = "FromSnoop"; 222 break; 223 case Target::FromPrefetcher: 224 s = "FromPrefetcher"; 225 break; 226 default: 227 s = ""; 228 break; 229 } 230 ccprintf(os, "%s%s: ", prefix, s); 231 t.pkt->print(os, verbosity, ""); 232 ccprintf(os, "\n"); 233 } 234} 235 236 237void 238MSHR::allocate(Addr blk_addr, unsigned blk_size, PacketPtr target, 239 Tick when_ready, Counter _order, bool alloc_on_fill) 240{ 241 blkAddr = blk_addr; 242 blkSize = blk_size; 243 isSecure = target->isSecure(); 244 readyTime = when_ready; 245 order = _order; 246 assert(target); 247 isForward = false; 248 _isUncacheable = target->req->isUncacheable(); 249 inService = false; 250 downstreamPending = false; 251 assert(targets.isReset()); 252 // Don't know of a case where we would allocate a new MSHR for a 253 // snoop (mem-side request), so set source according to request here 254 Target::Source source = (target->cmd == MemCmd::HardPFReq) ? 255 Target::FromPrefetcher : Target::FromCPU; 256 targets.add(target, when_ready, _order, source, true, alloc_on_fill); 257 assert(deferredTargets.isReset()); 258} 259 260 261void 262MSHR::clearDownstreamPending() 263{ 264 assert(downstreamPending); 265 downstreamPending = false; 266 // recursively clear flag on any MSHRs we will be forwarding 267 // responses to 268 targets.clearDownstreamPending(); 269} 270 271void 272MSHR::markInService(bool pending_modified_resp) 273{ 274 assert(!inService); 275 276 inService = true; 277 pendingModified = targets.needsWritable || pending_modified_resp; 278 postInvalidate = postDowngrade = false; 279 280 if (!downstreamPending) { 281 // let upstream caches know that the request has made it to a 282 // level where it's going to get a response 283 targets.clearDownstreamPending(); 284 } 285} 286 287 288void 289MSHR::deallocate() 290{ 291 assert(targets.empty()); 292 targets.resetFlags(); 293 assert(deferredTargets.isReset()); 294 inService = false; 295} 296 297/* 298 * Adds a target to an MSHR 299 */ 300void 301MSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order, 302 bool alloc_on_fill) 303{ 304 // assume we'd never issue a prefetch when we've got an 305 // outstanding miss 306 assert(pkt->cmd != MemCmd::HardPFReq); 307 308 // uncacheable accesses always allocate a new MSHR, and cacheable 309 // accesses ignore any uncacheable MSHRs, thus we should never 310 // have targets addded if originally allocated uncacheable 311 assert(!_isUncacheable); 312 313 // if there's a request already in service for this MSHR, we will 314 // have to defer the new target until after the response if any of 315 // the following are true: 316 // - there are other targets already deferred 317 // - there's a pending invalidate to be applied after the response 318 // comes back (but before this target is processed) 319 // - the MSHR's first (and only) non-deferred target is a cache 320 // maintenance packet 321 // - the new target is a cache maintenance packet (this is probably 322 // overly conservative but certainly safe) 323 // - this target requires a writable block and either we're not 324 // getting a writable block back or we have already snooped 325 // another read request that will downgrade our writable block 326 // to non-writable (Shared or Owned) 327 PacketPtr tgt_pkt = targets.front().pkt; 328 if (pkt->req->isCacheMaintenance() || 329 tgt_pkt->req->isCacheMaintenance() || 330 !deferredTargets.empty() || 331 (inService && 332 (hasPostInvalidate() || 333 (pkt->needsWritable() && 334 (!isPendingModified() || hasPostDowngrade() || isForward))))) { 335 // need to put on deferred list 336 if (inService && hasPostInvalidate()) 337 replaceUpgrade(pkt); 338 deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true, 339 alloc_on_fill); 340 } else { 341 // No request outstanding, or still OK to append to 342 // outstanding request: append to regular target list. Only 343 // mark pending if current request hasn't been issued yet 344 // (isn't in service). 345 targets.add(pkt, whenReady, _order, Target::FromCPU, !inService, 346 alloc_on_fill); 347 } 348} 349 350bool 351MSHR::handleSnoop(PacketPtr pkt, Counter _order) 352{ 353 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print()); 354 355 // when we snoop packets the needsWritable and isInvalidate flags 356 // should always be the same, however, this assumes that we never 357 // snoop writes as they are currently not marked as invalidations 358 panic_if((pkt->needsWritable() != pkt->isInvalidate()) && 359 !pkt->req->isCacheMaintenance(), 360 "%s got snoop %s where needsWritable, " 361 "does not match isInvalidate", name(), pkt->print()); 362 363 if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 364 // Request has not been issued yet, or it's been issued 365 // locally but is buffered unissued at some downstream cache 366 // which is forwarding us this snoop. Either way, the packet 367 // we're snooping logically precedes this MSHR's request, so 368 // the snoop has no impact on the MSHR, but must be processed 369 // in the standard way by the cache. The only exception is 370 // that if we're an L2+ cache buffering an UpgradeReq from a 371 // higher-level cache, and the snoop is invalidating, then our 372 // buffered upgrades must be converted to read exclusives, 373 // since the upper-level cache no longer has a valid copy. 374 // That is, even though the upper-level cache got out on its 375 // local bus first, some other invalidating transaction 376 // reached the global bus before the upgrade did. 377 if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 378 targets.replaceUpgrades(); 379 deferredTargets.replaceUpgrades(); 380 } 381 382 return false; 383 } 384 385 // From here on down, the request issued by this MSHR logically 386 // precedes the request we're snooping. 387 if (pkt->needsWritable() || pkt->req->isCacheInvalidate()) { 388 // snooped request still precedes the re-request we'll have to 389 // issue for deferred targets, if any... 390 deferredTargets.replaceUpgrades(); 391 } 392 393 PacketPtr tgt_pkt = targets.front().pkt; 394 if (hasPostInvalidate() || tgt_pkt->req->isCacheInvalidate()) { 395 // a prior snoop has already appended an invalidation or a 396 // cache invalidation operation is in progress, so logically 397 // we don't have the block anymore; no need for further 398 // snooping. 399 return true; 400 } 401 402 if (isPendingModified() || pkt->isInvalidate()) { 403 // We need to save and replay the packet in two cases: 404 // 1. We're awaiting a writable copy (Modified or Exclusive), 405 // so this MSHR is the orgering point, and we need to respond 406 // after we receive data. 407 // 2. It's an invalidation (e.g., UpgradeReq), and we need 408 // to forward the snoop up the hierarchy after the current 409 // transaction completes. 410 411 // Start by determining if we will eventually respond or not, 412 // matching the conditions checked in Cache::handleSnoop 413 bool will_respond = isPendingModified() && pkt->needsResponse() && 414 !pkt->isClean(); 415 416 // The packet we are snooping may be deleted by the time we 417 // actually process the target, and we consequently need to 418 // save a copy here. Clear flags and also allocate new data as 419 // the original packet data storage may have been deleted by 420 // the time we get to process this packet. In the cases where 421 // we are not responding after handling the snoop we also need 422 // to create a copy of the request to be on the safe side. In 423 // the latter case the cache is responsible for deleting both 424 // the packet and the request as part of handling the deferred 425 // snoop. 426 PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : 427 new Packet(new Request(*pkt->req), pkt->cmd, blkSize, pkt->id); 428 429 if (will_respond) { 430 // we are the ordering point, and will consequently 431 // respond, and depending on whether the packet 432 // needsWritable or not we either pass a Shared line or a 433 // Modified line 434 pkt->setCacheResponding(); 435 436 // inform the cache hierarchy that this cache had the line 437 // in the Modified state, even if the response is passed 438 // as Shared (and thus non-writable) 439 pkt->setResponderHadWritable(); 440 441 // in the case of an uncacheable request there is no need 442 // to set the responderHadWritable flag, but since the 443 // recipient does not care there is no harm in doing so 444 } 445 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, 446 downstreamPending && targets.needsWritable, false); 447 448 if (pkt->needsWritable() || pkt->isInvalidate()) { 449 // This transaction will take away our pending copy 450 postInvalidate = true; 451 } 452 453 if (isPendingModified() && pkt->isClean()) { 454 pkt->setSatisfied(); 455 } 456 } 457 458 if (!pkt->needsWritable() && !pkt->req->isUncacheable()) { 459 // This transaction will get a read-shared copy, downgrading 460 // our copy if we had a writable one 461 postDowngrade = true; 462 // make sure that any downstream cache does not respond with a 463 // writable (and dirty) copy even if it has one, unless it was 464 // explicitly asked for one 465 pkt->setHasSharers(); 466 } 467 468 return true; 469} 470 471MSHR::TargetList 472MSHR::extractServiceableTargets(PacketPtr pkt) 473{ 474 TargetList ready_targets; 475 // If the downstream MSHR got an invalidation request then we only 476 // service the first of the FromCPU targets and any other 477 // non-FromCPU target. This way the remaining FromCPU targets 478 // issue a new request and get a fresh copy of the block and we 479 // avoid memory consistency violations. 480 if (pkt->cmd == MemCmd::ReadRespWithInvalidate) { 481 auto it = targets.begin(); 482 assert((it->source == Target::FromCPU) || 483 (it->source == Target::FromPrefetcher)); 484 ready_targets.push_back(*it); 485 it = targets.erase(it); 486 while (it != targets.end()) { 487 if (it->source == Target::FromCPU) { 488 it++; 489 } else { 490 assert(it->source == Target::FromSnoop); 491 ready_targets.push_back(*it); 492 it = targets.erase(it); 493 } 494 } 495 ready_targets.populateFlags(); 496 } else { 497 std::swap(ready_targets, targets); 498 } 499 targets.populateFlags(); 500 501 return ready_targets; 502} 503 504bool 505MSHR::promoteDeferredTargets() 506{ 507 if (targets.empty() && deferredTargets.empty()) { 508 // nothing to promote 509 return false; 510 } 511 512 // the deferred targets can be generally promoted unless they 513 // contain a cache maintenance request 514 515 // find the first target that is a cache maintenance request 516 auto it = std::find_if(deferredTargets.begin(), deferredTargets.end(), 517 [](MSHR::Target &t) { 518 return t.pkt->req->isCacheMaintenance(); 519 }); 520 if (it == deferredTargets.begin()) { 521 // if the first deferred target is a cache maintenance packet 522 // then we can promote provided the targets list is empty and 523 // we can service it on its own 524 if (targets.empty()) { 525 targets.splice(targets.end(), deferredTargets, it); 526 } 527 } else { 528 // if a cache maintenance operation exists, we promote all the 529 // deferred targets that precede it, or all deferred targets 530 // otherwise 531 targets.splice(targets.end(), deferredTargets, 532 deferredTargets.begin(), it); 533 } 534 535 deferredTargets.populateFlags(); 536 targets.populateFlags(); 537 order = targets.front().order; 538 readyTime = std::max(curTick(), targets.front().readyTime); 539 540 return true; 541} 542 543 544void 545MSHR::promoteWritable() 546{ 547 if (deferredTargets.needsWritable && 548 !(hasPostInvalidate() || hasPostDowngrade())) { 549 // We got a writable response, but we have deferred targets 550 // which are waiting to request a writable copy (not because 551 // of a pending invalidate). This can happen if the original 552 // request was for a read-only block, but we got a writable 553 // response anyway. Since we got the writable copy there's no 554 // need to defer the targets, so move them up to the regular 555 // target list. 556 assert(!targets.needsWritable); 557 targets.needsWritable = true; 558 // if any of the deferred targets were upper-level cache 559 // requests marked downstreamPending, need to clear that 560 assert(!downstreamPending); // not pending here anymore 561 deferredTargets.clearDownstreamPending(); 562 // this clears out deferredTargets too 563 targets.splice(targets.end(), deferredTargets); 564 deferredTargets.resetFlags(); 565 } 566} 567 568 569bool 570MSHR::checkFunctional(PacketPtr pkt) 571{ 572 // For printing, we treat the MSHR as a whole as single entity. 573 // For other requests, we iterate over the individual targets 574 // since that's where the actual data lies. 575 if (pkt->isPrint()) { 576 pkt->checkFunctional(this, blkAddr, isSecure, blkSize, nullptr); 577 return false; 578 } else { 579 return (targets.checkFunctional(pkt) || 580 deferredTargets.checkFunctional(pkt)); 581 } 582} 583 584bool 585MSHR::sendPacket(Cache &cache) 586{ 587 return cache.sendMSHRQueuePacket(this); 588} 589 590void 591MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 592{ 593 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", 594 prefix, blkAddr, blkAddr + blkSize - 1, 595 isSecure ? "s" : "ns", 596 isForward ? "Forward" : "", 597 allocOnFill() ? "AllocOnFill" : "", 598 needsWritable() ? "Wrtbl" : "", 599 _isUncacheable ? "Unc" : "", 600 inService ? "InSvc" : "", 601 downstreamPending ? "DwnPend" : "", 602 postInvalidate ? "PostInv" : "", 603 postDowngrade ? "PostDowngr" : ""); 604 605 if (!targets.empty()) { 606 ccprintf(os, "%s Targets:\n", prefix); 607 targets.print(os, verbosity, prefix + " "); 608 } 609 if (!deferredTargets.empty()) { 610 ccprintf(os, "%s Deferred Targets:\n", prefix); 611 deferredTargets.print(os, verbosity, prefix + " "); 612 } 613} 614 615std::string 616MSHR::print() const 617{
|