mshr.cc (11279:3fd1142adad9) | mshr.cc (11284:b3926db25371) |
---|---|
1/* 2 * Copyright (c) 2012-2013, 2015 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 48 unchanged lines hidden (view full) --- 57#include "debug/Cache.hh" 58#include "mem/cache/cache.hh" 59#include "mem/cache/mshr.hh" 60#include "sim/core.hh" 61 62using namespace std; 63 64MSHR::MSHR() : readyTime(0), _isUncacheable(false), downstreamPending(false), | 1/* 2 * Copyright (c) 2012-2013, 2015 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 48 unchanged lines hidden (view full) --- 57#include "debug/Cache.hh" 58#include "mem/cache/cache.hh" 59#include "mem/cache/mshr.hh" 60#include "sim/core.hh" 61 62using namespace std; 63 64MSHR::MSHR() : readyTime(0), _isUncacheable(false), downstreamPending(false), |
65 pendingDirty(false), | 65 pendingModified(false), |
66 postInvalidate(false), postDowngrade(false), 67 queue(NULL), order(0), blkAddr(0), 68 blkSize(0), isSecure(false), inService(false), 69 isForward(false), allocOnFill(false), 70 data(NULL) 71{ 72} 73 74 75MSHR::TargetList::TargetList() | 66 postInvalidate(false), postDowngrade(false), 67 queue(NULL), order(0), blkAddr(0), 68 blkSize(0), isSecure(false), inService(false), 69 isForward(false), allocOnFill(false), 70 data(NULL) 71{ 72} 73 74 75MSHR::TargetList::TargetList() |
76 : needsExclusive(false), hasUpgrade(false) | 76 : needsWritable(false), hasUpgrade(false) |
77{} 78 79 80inline void 81MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 82 Counter order, Target::Source source, bool markPending) 83{ 84 if (source != Target::FromSnoop) { | 77{} 78 79 80inline void 81MSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 82 Counter order, Target::Source source, bool markPending) 83{ 84 if (source != Target::FromSnoop) { |
85 if (pkt->needsExclusive()) { 86 needsExclusive = true; | 85 if (pkt->needsWritable()) { 86 needsWritable = true; |
87 } 88 89 // StoreCondReq is effectively an upgrade if it's in an MSHR 90 // since it would have been failed already if we didn't have a 91 // read-only copy 92 if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 93 hasUpgrade = true; 94 } --- 138 unchanged lines hidden (view full) --- 233 assert(downstreamPending); 234 downstreamPending = false; 235 // recursively clear flag on any MSHRs we will be forwarding 236 // responses to 237 targets.clearDownstreamPending(); 238} 239 240bool | 87 } 88 89 // StoreCondReq is effectively an upgrade if it's in an MSHR 90 // since it would have been failed already if we didn't have a 91 // read-only copy 92 if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 93 hasUpgrade = true; 94 } --- 138 unchanged lines hidden (view full) --- 233 assert(downstreamPending); 234 downstreamPending = false; 235 // recursively clear flag on any MSHRs we will be forwarding 236 // responses to 237 targets.clearDownstreamPending(); 238} 239 240bool |
241MSHR::markInService(bool pending_dirty_resp) | 241MSHR::markInService(bool pending_modified_resp) |
242{ 243 assert(!inService); 244 if (isForwardNoResponse()) { 245 // we just forwarded the request packet & don't expect a 246 // response, so get rid of it 247 assert(getNumTargets() == 1); 248 popTarget(); 249 return true; 250 } 251 252 inService = true; | 242{ 243 assert(!inService); 244 if (isForwardNoResponse()) { 245 // we just forwarded the request packet & don't expect a 246 // response, so get rid of it 247 assert(getNumTargets() == 1); 248 popTarget(); 249 return true; 250 } 251 252 inService = true; |
253 pendingDirty = targets.needsExclusive || pending_dirty_resp; | 253 pendingModified = targets.needsWritable || pending_modified_resp; |
254 postInvalidate = postDowngrade = false; 255 256 if (!downstreamPending) { 257 // let upstream caches know that the request has made it to a 258 // level where it's going to get a response 259 targets.clearDownstreamPending(); 260 } 261 return false; --- 30 unchanged lines hidden (view full) --- 292 allocOnFill = allocOnFill || alloc_on_fill; 293 294 // if there's a request already in service for this MSHR, we will 295 // have to defer the new target until after the response if any of 296 // the following are true: 297 // - there are other targets already deferred 298 // - there's a pending invalidate to be applied after the response 299 // comes back (but before this target is processed) | 254 postInvalidate = postDowngrade = false; 255 256 if (!downstreamPending) { 257 // let upstream caches know that the request has made it to a 258 // level where it's going to get a response 259 targets.clearDownstreamPending(); 260 } 261 return false; --- 30 unchanged lines hidden (view full) --- 292 allocOnFill = allocOnFill || alloc_on_fill; 293 294 // if there's a request already in service for this MSHR, we will 295 // have to defer the new target until after the response if any of 296 // the following are true: 297 // - there are other targets already deferred 298 // - there's a pending invalidate to be applied after the response 299 // comes back (but before this target is processed) |
300 // - this target requires an exclusive block and either we're not 301 // getting an exclusive block back or we have already snooped 302 // another read request that will downgrade our exclusive block 303 // to shared | 300 // - this target requires a writable block and either we're not 301 // getting a writable block back or we have already snooped 302 // another read request that will downgrade our writable block 303 // to non-writable (Shared or Owned) |
304 if (inService && 305 (!deferredTargets.empty() || hasPostInvalidate() || | 304 if (inService && 305 (!deferredTargets.empty() || hasPostInvalidate() || |
306 (pkt->needsExclusive() && 307 (!isPendingDirty() || hasPostDowngrade() || isForward)))) { | 306 (pkt->needsWritable() && 307 (!isPendingModified() || hasPostDowngrade() || isForward)))) { |
308 // need to put on deferred list 309 if (hasPostInvalidate()) 310 replaceUpgrade(pkt); 311 deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true); 312 } else { 313 // No request outstanding, or still OK to append to 314 // outstanding request: append to regular target list. Only 315 // mark pending if current request hasn't been issued yet 316 // (isn't in service). 317 targets.add(pkt, whenReady, _order, Target::FromCPU, !inService); 318 } 319} 320 321bool 322MSHR::handleSnoop(PacketPtr pkt, Counter _order) 323{ 324 DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__, 325 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 326 | 308 // need to put on deferred list 309 if (hasPostInvalidate()) 310 replaceUpgrade(pkt); 311 deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true); 312 } else { 313 // No request outstanding, or still OK to append to 314 // outstanding request: append to regular target list. Only 315 // mark pending if current request hasn't been issued yet 316 // (isn't in service). 317 targets.add(pkt, whenReady, _order, Target::FromCPU, !inService); 318 } 319} 320 321bool 322MSHR::handleSnoop(PacketPtr pkt, Counter _order) 323{ 324 DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__, 325 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 326 |
327 // when we snoop packets the needsExclusive and isInvalidate flags | 327 // when we snoop packets the needsWritable and isInvalidate flags |
328 // should always be the same, however, this assumes that we never 329 // snoop writes as they are currently not marked as invalidations | 328 // should always be the same, however, this assumes that we never 329 // snoop writes as they are currently not marked as invalidations |
330 panic_if(pkt->needsExclusive() != pkt->isInvalidate(), 331 "%s got snoop %s to addr %#llx where needsExclusive, " | 330 panic_if(pkt->needsWritable() != pkt->isInvalidate(), 331 "%s got snoop %s to addr %#llx where needsWritable, " |
332 "does not match isInvalidate", name(), pkt->cmdString(), 333 pkt->getAddr()); 334 335 if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 336 // Request has not been issued yet, or it's been issued 337 // locally but is buffered unissued at some downstream cache 338 // which is forwarding us this snoop. Either way, the packet 339 // we're snooping logically precedes this MSHR's request, so 340 // the snoop has no impact on the MSHR, but must be processed 341 // in the standard way by the cache. The only exception is 342 // that if we're an L2+ cache buffering an UpgradeReq from a 343 // higher-level cache, and the snoop is invalidating, then our 344 // buffered upgrades must be converted to read exclusives, 345 // since the upper-level cache no longer has a valid copy. 346 // That is, even though the upper-level cache got out on its 347 // local bus first, some other invalidating transaction 348 // reached the global bus before the upgrade did. | 332 "does not match isInvalidate", name(), pkt->cmdString(), 333 pkt->getAddr()); 334 335 if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 336 // Request has not been issued yet, or it's been issued 337 // locally but is buffered unissued at some downstream cache 338 // which is forwarding us this snoop. Either way, the packet 339 // we're snooping logically precedes this MSHR's request, so 340 // the snoop has no impact on the MSHR, but must be processed 341 // in the standard way by the cache. The only exception is 342 // that if we're an L2+ cache buffering an UpgradeReq from a 343 // higher-level cache, and the snoop is invalidating, then our 344 // buffered upgrades must be converted to read exclusives, 345 // since the upper-level cache no longer has a valid copy. 346 // That is, even though the upper-level cache got out on its 347 // local bus first, some other invalidating transaction 348 // reached the global bus before the upgrade did. |
349 if (pkt->needsExclusive()) { | 349 if (pkt->needsWritable()) { |
350 targets.replaceUpgrades(); 351 deferredTargets.replaceUpgrades(); 352 } 353 354 return false; 355 } 356 357 // From here on down, the request issued by this MSHR logically 358 // precedes the request we're snooping. | 350 targets.replaceUpgrades(); 351 deferredTargets.replaceUpgrades(); 352 } 353 354 return false; 355 } 356 357 // From here on down, the request issued by this MSHR logically 358 // precedes the request we're snooping. |
359 if (pkt->needsExclusive()) { | 359 if (pkt->needsWritable()) { |
360 // snooped request still precedes the re-request we'll have to 361 // issue for deferred targets, if any... 362 deferredTargets.replaceUpgrades(); 363 } 364 365 if (hasPostInvalidate()) { 366 // a prior snoop has already appended an invalidation, so 367 // logically we don't have the block anymore; no need for 368 // further snooping. 369 return true; 370 } 371 | 360 // snooped request still precedes the re-request we'll have to 361 // issue for deferred targets, if any... 362 deferredTargets.replaceUpgrades(); 363 } 364 365 if (hasPostInvalidate()) { 366 // a prior snoop has already appended an invalidation, so 367 // logically we don't have the block anymore; no need for 368 // further snooping. 369 return true; 370 } 371 |
372 if (isPendingDirty() || pkt->isInvalidate()) { | 372 if (isPendingModified() || pkt->isInvalidate()) { |
373 // We need to save and replay the packet in two cases: | 373 // We need to save and replay the packet in two cases: |
374 // 1. We're awaiting an exclusive copy, so ownership is pending, 375 // and we need to deal with the snoop after we receive data. | 374 // 1. We're awaiting a writable copy (Modified or Exclusive), 375 // so this MSHR is the orgering point, and we need to respond 376 // after we receive data. |
376 // 2. It's an invalidation (e.g., UpgradeReq), and we need 377 // to forward the snoop up the hierarchy after the current 378 // transaction completes. 379 380 // Start by determining if we will eventually respond or not, 381 // matching the conditions checked in Cache::handleSnoop | 377 // 2. It's an invalidation (e.g., UpgradeReq), and we need 378 // to forward the snoop up the hierarchy after the current 379 // transaction completes. 380 381 // Start by determining if we will eventually respond or not, 382 // matching the conditions checked in Cache::handleSnoop |
382 bool will_respond = isPendingDirty() && pkt->needsResponse() && | 383 bool will_respond = isPendingModified() && pkt->needsResponse() && |
383 pkt->cmd != MemCmd::InvalidateReq; 384 385 // The packet we are snooping may be deleted by the time we 386 // actually process the target, and we consequently need to 387 // save a copy here. Clear flags and also allocate new data as 388 // the original packet data storage may have been deleted by 389 // the time we get to process this packet. In the cases where 390 // we are not responding after handling the snoop we also need 391 // to create a copy of the request to be on the safe side. In 392 // the latter case the cache is responsible for deleting both 393 // the packet and the request as part of handling the deferred 394 // snoop. 395 PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : 396 new Packet(new Request(*pkt->req), pkt->cmd); 397 | 384 pkt->cmd != MemCmd::InvalidateReq; 385 386 // The packet we are snooping may be deleted by the time we 387 // actually process the target, and we consequently need to 388 // save a copy here. Clear flags and also allocate new data as 389 // the original packet data storage may have been deleted by 390 // the time we get to process this packet. In the cases where 391 // we are not responding after handling the snoop we also need 392 // to create a copy of the request to be on the safe side. In 393 // the latter case the cache is responsible for deleting both 394 // the packet and the request as part of handling the deferred 395 // snoop. 396 PacketPtr cp_pkt = will_respond ? new Packet(pkt, true, true) : 397 new Packet(new Request(*pkt->req), pkt->cmd); 398 |
398 if (isPendingDirty()) { 399 // The new packet will need to get the response from the 400 // MSHR already queued up here 401 pkt->assertMemInhibit(); | 399 if (isPendingModified()) { 400 // we are the ordering point, and will consequently 401 // respond, and depending on whether the packet 402 // needsWritable or not we either pass a Shared line or a 403 // Modified line 404 pkt->setCacheResponding(); 405 406 // inform the cache hierarchy that this cache had the line 407 // in the Modified state, even if the response is passed 408 // as Shared (and thus non-writable) 409 pkt->setResponderHadWritable(); 410 |
402 // in the case of an uncacheable request there is no need | 411 // in the case of an uncacheable request there is no need |
403 // to set the exclusive flag, but since the recipient does 404 // not care there is no harm in doing so 405 pkt->setSupplyExclusive(); | 412 // to set the responderHadWritable flag, but since the 413 // recipient does not care there is no harm in doing so |
406 } 407 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, | 414 } 415 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, |
408 downstreamPending && targets.needsExclusive); | 416 downstreamPending && targets.needsWritable); |
409 | 417 |
410 if (pkt->needsExclusive()) { | 418 if (pkt->needsWritable()) { |
411 // This transaction will take away our pending copy 412 postInvalidate = true; 413 } 414 } 415 | 419 // This transaction will take away our pending copy 420 postInvalidate = true; 421 } 422 } 423 |
416 if (!pkt->needsExclusive() && !pkt->req->isUncacheable()) { | 424 if (!pkt->needsWritable() && !pkt->req->isUncacheable()) { |
417 // This transaction will get a read-shared copy, downgrading | 425 // This transaction will get a read-shared copy, downgrading |
418 // our copy if we had an exclusive one | 426 // our copy if we had a writable one |
419 postDowngrade = true; | 427 postDowngrade = true; |
420 pkt->assertShared(); | 428 // make sure that any downstream cache does not respond with a 429 // writable (and dirty) copy even if it has one, unless it was 430 // explicitly asked for one 431 pkt->setHasSharers(); |
421 } 422 423 return true; 424} 425 426 427bool 428MSHR::promoteDeferredTargets() --- 12 unchanged lines hidden (view full) --- 441 order = targets.front().order; 442 readyTime = std::max(curTick(), targets.front().readyTime); 443 444 return true; 445} 446 447 448void | 432 } 433 434 return true; 435} 436 437 438bool 439MSHR::promoteDeferredTargets() --- 12 unchanged lines hidden (view full) --- 452 order = targets.front().order; 453 readyTime = std::max(curTick(), targets.front().readyTime); 454 455 return true; 456} 457 458 459void |
449MSHR::promoteExclusive() | 460MSHR::promoteWritable() |
450{ | 461{ |
451 if (deferredTargets.needsExclusive && | 462 if (deferredTargets.needsWritable && |
452 !(hasPostInvalidate() || hasPostDowngrade())) { | 463 !(hasPostInvalidate() || hasPostDowngrade())) { |
453 // We got an exclusive response, but we have deferred targets 454 // which are waiting to request an exclusive copy (not because | 464 // We got a writable response, but we have deferred targets 465 // which are waiting to request a writable copy (not because |
455 // of a pending invalidate). This can happen if the original | 466 // of a pending invalidate). This can happen if the original |
456 // request was for a read-only (non-exclusive) block, but we 457 // got an exclusive copy anyway because of the E part of the 458 // MOESI/MESI protocol. Since we got the exclusive copy 459 // there's no need to defer the targets, so move them up to 460 // the regular target list. 461 assert(!targets.needsExclusive); 462 targets.needsExclusive = true; | 467 // request was for a read-only block, but we got a writable 468 // response anyway. Since we got the writable copy there's no 469 // need to defer the targets, so move them up to the regular 470 // target list. 471 assert(!targets.needsWritable); 472 targets.needsWritable = true; |
463 // if any of the deferred targets were upper-level cache 464 // requests marked downstreamPending, need to clear that 465 assert(!downstreamPending); // not pending here anymore 466 deferredTargets.clearDownstreamPending(); 467 // this clears out deferredTargets too 468 targets.splice(targets.end(), deferredTargets); 469 deferredTargets.resetFlags(); 470 } --- 20 unchanged lines hidden (view full) --- 491MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 492{ 493 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", 494 prefix, blkAddr, blkAddr + blkSize - 1, 495 isSecure ? "s" : "ns", 496 isForward ? "Forward" : "", 497 allocOnFill ? "AllocOnFill" : "", 498 isForwardNoResponse() ? "ForwNoResp" : "", | 473 // if any of the deferred targets were upper-level cache 474 // requests marked downstreamPending, need to clear that 475 assert(!downstreamPending); // not pending here anymore 476 deferredTargets.clearDownstreamPending(); 477 // this clears out deferredTargets too 478 targets.splice(targets.end(), deferredTargets); 479 deferredTargets.resetFlags(); 480 } --- 20 unchanged lines hidden (view full) --- 501MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 502{ 503 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", 504 prefix, blkAddr, blkAddr + blkSize - 1, 505 isSecure ? "s" : "ns", 506 isForward ? "Forward" : "", 507 allocOnFill ? "AllocOnFill" : "", 508 isForwardNoResponse() ? "ForwNoResp" : "", |
499 needsExclusive() ? "Excl" : "", | 509 needsWritable() ? "Wrtbl" : "", |
500 _isUncacheable ? "Unc" : "", 501 inService ? "InSvc" : "", 502 downstreamPending ? "DwnPend" : "", 503 hasPostInvalidate() ? "PostInv" : "", 504 hasPostDowngrade() ? "PostDowngr" : ""); 505 506 ccprintf(os, "%s Targets:\n", prefix); 507 targets.print(os, verbosity, prefix + " "); --- 13 unchanged lines hidden --- | 510 _isUncacheable ? "Unc" : "", 511 inService ? "InSvc" : "", 512 downstreamPending ? "DwnPend" : "", 513 hasPostInvalidate() ? "PostInv" : "", 514 hasPostDowngrade() ? "PostDowngr" : ""); 515 516 ccprintf(os, "%s Targets:\n", prefix); 517 targets.print(os, verbosity, prefix + " "); --- 13 unchanged lines hidden --- |