mshr.cc revision 10764
18706Sandreas.hansson@arm.com/* 29814Sandreas.hansson@arm.com * Copyright (c) 2012-2013, 2015 ARM Limited 38706Sandreas.hansson@arm.com * All rights reserved. 48706Sandreas.hansson@arm.com * 58706Sandreas.hansson@arm.com * The license below extends only to copyright in the software and shall 68706Sandreas.hansson@arm.com * not be construed as granting a license to any other intellectual 78706Sandreas.hansson@arm.com * property including but not limited to intellectual property relating 88706Sandreas.hansson@arm.com * to a hardware implementation of the functionality of the software 98706Sandreas.hansson@arm.com * licensed hereunder. You may use the software subject to the license 108706Sandreas.hansson@arm.com * terms below provided that you ensure that this notice is replicated 118706Sandreas.hansson@arm.com * unmodified and in its entirety in all distributions of the software, 128706Sandreas.hansson@arm.com * modified or unmodified, in source code or in binary form. 138706Sandreas.hansson@arm.com * 148706Sandreas.hansson@arm.com * Copyright (c) 2002-2005 The Regents of The University of Michigan 158706Sandreas.hansson@arm.com * Copyright (c) 2010 Advanced Micro Devices, Inc. 168706Sandreas.hansson@arm.com * All rights reserved. 178706Sandreas.hansson@arm.com * 188706Sandreas.hansson@arm.com * Redistribution and use in source and binary forms, with or without 198706Sandreas.hansson@arm.com * modification, are permitted provided that the following conditions are 208706Sandreas.hansson@arm.com * met: redistributions of source code must retain the above copyright 218706Sandreas.hansson@arm.com * notice, this list of conditions and the following disclaimer; 228706Sandreas.hansson@arm.com * redistributions in binary form must reproduce the above copyright 238706Sandreas.hansson@arm.com * notice, this list of conditions and the following disclaimer in the 248706Sandreas.hansson@arm.com * documentation and/or other materials provided with the distribution; 258706Sandreas.hansson@arm.com * neither the name of the copyright holders nor the names of its 268706Sandreas.hansson@arm.com * contributors may be used to endorse or promote products derived from 278706Sandreas.hansson@arm.com * this software without specific prior written permission. 288706Sandreas.hansson@arm.com * 298706Sandreas.hansson@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 308706Sandreas.hansson@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 318706Sandreas.hansson@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 328706Sandreas.hansson@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 338706Sandreas.hansson@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 348706Sandreas.hansson@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 358706Sandreas.hansson@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 368706Sandreas.hansson@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 378706Sandreas.hansson@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 388706Sandreas.hansson@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 398706Sandreas.hansson@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 408706Sandreas.hansson@arm.com * 418706Sandreas.hansson@arm.com * Authors: Erik Hallnor 428706Sandreas.hansson@arm.com * Dave Greene 438706Sandreas.hansson@arm.com */ 448853Sandreas.hansson@arm.com 458853Sandreas.hansson@arm.com/** 468853Sandreas.hansson@arm.com * @file 478853Sandreas.hansson@arm.com * Miss Status and Handling Register (MSHR) definitions. 488853Sandreas.hansson@arm.com */ 498853Sandreas.hansson@arm.com 508853Sandreas.hansson@arm.com#include <algorithm> 518853Sandreas.hansson@arm.com#include <cassert> 528853Sandreas.hansson@arm.com#include <string> 538853Sandreas.hansson@arm.com#include <vector> 548853Sandreas.hansson@arm.com 558706Sandreas.hansson@arm.com#include "base/misc.hh" 568706Sandreas.hansson@arm.com#include "base/types.hh" 578706Sandreas.hansson@arm.com#include "debug/Cache.hh" 588706Sandreas.hansson@arm.com#include "mem/cache/cache.hh" 598706Sandreas.hansson@arm.com#include "mem/cache/mshr.hh" 608706Sandreas.hansson@arm.com#include "sim/core.hh" 618706Sandreas.hansson@arm.com 628706Sandreas.hansson@arm.comusing namespace std; 639850Sandreas.hansson@arm.com 648706Sandreas.hansson@arm.comMSHR::MSHR() : readyTime(0), _isUncacheable(false), downstreamPending(false), 658706Sandreas.hansson@arm.com pendingDirty(false), 668706Sandreas.hansson@arm.com postInvalidate(false), postDowngrade(false), 678706Sandreas.hansson@arm.com queue(NULL), order(0), blkAddr(0), 688706Sandreas.hansson@arm.com blkSize(0), isSecure(false), inService(false), 698706Sandreas.hansson@arm.com isForward(false), threadNum(InvalidThreadID), data(NULL) 708706Sandreas.hansson@arm.com{ 718853Sandreas.hansson@arm.com} 728853Sandreas.hansson@arm.com 738706Sandreas.hansson@arm.com 748706Sandreas.hansson@arm.comMSHR::TargetList::TargetList() 758706Sandreas.hansson@arm.com : needsExclusive(false), hasUpgrade(false) 768706Sandreas.hansson@arm.com{} 778706Sandreas.hansson@arm.com 788706Sandreas.hansson@arm.com 798706Sandreas.hansson@arm.cominline void 808706Sandreas.hansson@arm.comMSHR::TargetList::add(PacketPtr pkt, Tick readyTime, 818706Sandreas.hansson@arm.com Counter order, Target::Source source, bool markPending) 828706Sandreas.hansson@arm.com{ 838706Sandreas.hansson@arm.com if (source != Target::FromSnoop) { 848706Sandreas.hansson@arm.com if (pkt->needsExclusive()) { 858706Sandreas.hansson@arm.com needsExclusive = true; 868853Sandreas.hansson@arm.com } 878853Sandreas.hansson@arm.com 888853Sandreas.hansson@arm.com // StoreCondReq is effectively an upgrade if it's in an MSHR 898922Swilliam.wang@arm.com // since it would have been failed already if we didn't have a 908706Sandreas.hansson@arm.com // read-only copy 919814Sandreas.hansson@arm.com if (pkt->isUpgrade() || pkt->cmd == MemCmd::StoreCondReq) { 929814Sandreas.hansson@arm.com hasUpgrade = true; 939814Sandreas.hansson@arm.com } 948861Sandreas.hansson@arm.com } 958853Sandreas.hansson@arm.com 968706Sandreas.hansson@arm.com if (markPending) { 979814Sandreas.hansson@arm.com // Iterate over the SenderState stack and see if we find 989814Sandreas.hansson@arm.com // an MSHR entry. If we do, set the downstreamPending 998706Sandreas.hansson@arm.com // flag. Otherwise, do nothing. 1008706Sandreas.hansson@arm.com MSHR *mshr = pkt->findNextSenderState<MSHR>(); 1018706Sandreas.hansson@arm.com if (mshr != NULL) { 1028706Sandreas.hansson@arm.com assert(!mshr->downstreamPending); 1038706Sandreas.hansson@arm.com mshr->downstreamPending = true; 1048861Sandreas.hansson@arm.com } 1058853Sandreas.hansson@arm.com } 1068706Sandreas.hansson@arm.com 1078706Sandreas.hansson@arm.com push_back(Target(pkt, readyTime, order, source, markPending)); 1088706Sandreas.hansson@arm.com} 1098706Sandreas.hansson@arm.com 1108861Sandreas.hansson@arm.com 1118853Sandreas.hansson@arm.comstatic void 1128706Sandreas.hansson@arm.comreplaceUpgrade(PacketPtr pkt) 1138706Sandreas.hansson@arm.com{ 1148706Sandreas.hansson@arm.com if (pkt->cmd == MemCmd::UpgradeReq) { 1158706Sandreas.hansson@arm.com pkt->cmd = MemCmd::ReadExReq; 1168861Sandreas.hansson@arm.com DPRINTF(Cache, "Replacing UpgradeReq with ReadExReq\n"); 1178706Sandreas.hansson@arm.com } else if (pkt->cmd == MemCmd::SCUpgradeReq) { 1188706Sandreas.hansson@arm.com pkt->cmd = MemCmd::SCUpgradeFailReq; 1198706Sandreas.hansson@arm.com DPRINTF(Cache, "Replacing SCUpgradeReq with SCUpgradeFailReq\n"); 1208706Sandreas.hansson@arm.com } else if (pkt->cmd == MemCmd::StoreCondReq) { 1218706Sandreas.hansson@arm.com pkt->cmd = MemCmd::StoreCondFailReq; 1228861Sandreas.hansson@arm.com DPRINTF(Cache, "Replacing StoreCondReq with StoreCondFailReq\n"); 1238706Sandreas.hansson@arm.com } 1248706Sandreas.hansson@arm.com} 1258706Sandreas.hansson@arm.com 1268706Sandreas.hansson@arm.com 1278706Sandreas.hansson@arm.comvoid 1288861Sandreas.hansson@arm.comMSHR::TargetList::replaceUpgrades() 1298706Sandreas.hansson@arm.com{ 1309850Sandreas.hansson@arm.com if (!hasUpgrade) 1318706Sandreas.hansson@arm.com return; 1328706Sandreas.hansson@arm.com 1338706Sandreas.hansson@arm.com Iterator end_i = end(); 1348706Sandreas.hansson@arm.com for (Iterator i = begin(); i != end_i; ++i) { 1358706Sandreas.hansson@arm.com replaceUpgrade(i->pkt); 1368861Sandreas.hansson@arm.com } 1378706Sandreas.hansson@arm.com 1388706Sandreas.hansson@arm.com hasUpgrade = false; 1398706Sandreas.hansson@arm.com} 1408706Sandreas.hansson@arm.com 1418706Sandreas.hansson@arm.com 1428706Sandreas.hansson@arm.comvoid 1438861Sandreas.hansson@arm.comMSHR::TargetList::clearDownstreamPending() 1448706Sandreas.hansson@arm.com{ 1458706Sandreas.hansson@arm.com Iterator end_i = end(); 1468706Sandreas.hansson@arm.com for (Iterator i = begin(); i != end_i; ++i) { 1478706Sandreas.hansson@arm.com if (i->markedPending) { 1488706Sandreas.hansson@arm.com // Iterate over the SenderState stack and see if we find 1498706Sandreas.hansson@arm.com // an MSHR entry. If we find one, clear the 1508861Sandreas.hansson@arm.com // downstreamPending flag by calling 1518706Sandreas.hansson@arm.com // clearDownstreamPending(). This recursively clears the 1528706Sandreas.hansson@arm.com // downstreamPending flag in all caches this packet has 1538706Sandreas.hansson@arm.com // passed through. 1548706Sandreas.hansson@arm.com MSHR *mshr = i->pkt->findNextSenderState<MSHR>(); 1558706Sandreas.hansson@arm.com if (mshr != NULL) { 1568706Sandreas.hansson@arm.com mshr->clearDownstreamPending(); 1578706Sandreas.hansson@arm.com } 1588706Sandreas.hansson@arm.com } 1598861Sandreas.hansson@arm.com } 1608706Sandreas.hansson@arm.com} 1618706Sandreas.hansson@arm.com 1628706Sandreas.hansson@arm.com 1638706Sandreas.hansson@arm.combool 1649850Sandreas.hansson@arm.comMSHR::TargetList::checkFunctional(PacketPtr pkt) 1658706Sandreas.hansson@arm.com{ 1668706Sandreas.hansson@arm.com Iterator end_i = end(); 1678861Sandreas.hansson@arm.com for (Iterator i = begin(); i != end_i; ++i) { 1688706Sandreas.hansson@arm.com if (pkt->checkFunctional(i->pkt)) { 1698706Sandreas.hansson@arm.com return true; 1708706Sandreas.hansson@arm.com } 1718706Sandreas.hansson@arm.com } 1728706Sandreas.hansson@arm.com 1738706Sandreas.hansson@arm.com return false; 1748706Sandreas.hansson@arm.com} 1758706Sandreas.hansson@arm.com 1768861Sandreas.hansson@arm.com 1778706Sandreas.hansson@arm.comvoid 1788706Sandreas.hansson@arm.comMSHR::TargetList:: 1798706Sandreas.hansson@arm.comprint(std::ostream &os, int verbosity, const std::string &prefix) const 1808706Sandreas.hansson@arm.com{ 1818706Sandreas.hansson@arm.com ConstIterator end_i = end(); 1828706Sandreas.hansson@arm.com for (ConstIterator i = begin(); i != end_i; ++i) { 1838706Sandreas.hansson@arm.com const char *s; 184 switch (i->source) { 185 case Target::FromCPU: 186 s = "FromCPU"; 187 break; 188 case Target::FromSnoop: 189 s = "FromSnoop"; 190 break; 191 case Target::FromPrefetcher: 192 s = "FromPrefetcher"; 193 break; 194 default: 195 s = ""; 196 break; 197 } 198 ccprintf(os, "%s%s: ", prefix, s); 199 i->pkt->print(os, verbosity, ""); 200 } 201} 202 203 204void 205MSHR::allocate(Addr blk_addr, unsigned blk_size, PacketPtr target, 206 Tick when_ready, Counter _order) 207{ 208 blkAddr = blk_addr; 209 blkSize = blk_size; 210 isSecure = target->isSecure(); 211 readyTime = when_ready; 212 order = _order; 213 assert(target); 214 isForward = false; 215 _isUncacheable = target->req->isUncacheable(); 216 inService = false; 217 downstreamPending = false; 218 threadNum = 0; 219 assert(targets.isReset()); 220 // Don't know of a case where we would allocate a new MSHR for a 221 // snoop (mem-side request), so set source according to request here 222 Target::Source source = (target->cmd == MemCmd::HardPFReq) ? 223 Target::FromPrefetcher : Target::FromCPU; 224 targets.add(target, when_ready, _order, source, true); 225 assert(deferredTargets.isReset()); 226 data = NULL; 227} 228 229 230void 231MSHR::clearDownstreamPending() 232{ 233 assert(downstreamPending); 234 downstreamPending = false; 235 // recursively clear flag on any MSHRs we will be forwarding 236 // responses to 237 targets.clearDownstreamPending(); 238} 239 240bool 241MSHR::markInService(bool pending_dirty_resp) 242{ 243 assert(!inService); 244 if (isForwardNoResponse()) { 245 // we just forwarded the request packet & don't expect a 246 // response, so get rid of it 247 assert(getNumTargets() == 1); 248 popTarget(); 249 return true; 250 } 251 252 inService = true; 253 pendingDirty = targets.needsExclusive || pending_dirty_resp; 254 postInvalidate = postDowngrade = false; 255 256 if (!downstreamPending) { 257 // let upstream caches know that the request has made it to a 258 // level where it's going to get a response 259 targets.clearDownstreamPending(); 260 } 261 return false; 262} 263 264 265void 266MSHR::deallocate() 267{ 268 assert(targets.empty()); 269 targets.resetFlags(); 270 assert(deferredTargets.isReset()); 271 inService = false; 272} 273 274/* 275 * Adds a target to an MSHR 276 */ 277void 278MSHR::allocateTarget(PacketPtr pkt, Tick whenReady, Counter _order) 279{ 280 // if there's a request already in service for this MSHR, we will 281 // have to defer the new target until after the response if any of 282 // the following are true: 283 // - there are other targets already deferred 284 // - there's a pending invalidate to be applied after the response 285 // comes back (but before this target is processed) 286 // - this target requires an exclusive block and either we're not 287 // getting an exclusive block back or we have already snooped 288 // another read request that will downgrade our exclusive block 289 // to shared 290 291 // assume we'd never issue a prefetch when we've got an 292 // outstanding miss 293 assert(pkt->cmd != MemCmd::HardPFReq); 294 295 if (inService && 296 (!deferredTargets.empty() || hasPostInvalidate() || 297 (pkt->needsExclusive() && 298 (!isPendingDirty() || hasPostDowngrade() || isForward)))) { 299 // need to put on deferred list 300 if (hasPostInvalidate()) 301 replaceUpgrade(pkt); 302 deferredTargets.add(pkt, whenReady, _order, Target::FromCPU, true); 303 } else { 304 // No request outstanding, or still OK to append to 305 // outstanding request: append to regular target list. Only 306 // mark pending if current request hasn't been issued yet 307 // (isn't in service). 308 targets.add(pkt, whenReady, _order, Target::FromCPU, !inService); 309 } 310} 311 312bool 313MSHR::handleSnoop(PacketPtr pkt, Counter _order) 314{ 315 DPRINTF(Cache, "%s for %s addr %#llx size %d\n", __func__, 316 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 317 if (!inService || (pkt->isExpressSnoop() && downstreamPending)) { 318 // Request has not been issued yet, or it's been issued 319 // locally but is buffered unissued at some downstream cache 320 // which is forwarding us this snoop. Either way, the packet 321 // we're snooping logically precedes this MSHR's request, so 322 // the snoop has no impact on the MSHR, but must be processed 323 // in the standard way by the cache. The only exception is 324 // that if we're an L2+ cache buffering an UpgradeReq from a 325 // higher-level cache, and the snoop is invalidating, then our 326 // buffered upgrades must be converted to read exclusives, 327 // since the upper-level cache no longer has a valid copy. 328 // That is, even though the upper-level cache got out on its 329 // local bus first, some other invalidating transaction 330 // reached the global bus before the upgrade did. 331 if (pkt->needsExclusive()) { 332 targets.replaceUpgrades(); 333 deferredTargets.replaceUpgrades(); 334 } 335 336 return false; 337 } 338 339 // From here on down, the request issued by this MSHR logically 340 // precedes the request we're snooping. 341 if (pkt->needsExclusive()) { 342 // snooped request still precedes the re-request we'll have to 343 // issue for deferred targets, if any... 344 deferredTargets.replaceUpgrades(); 345 } 346 347 if (hasPostInvalidate()) { 348 // a prior snoop has already appended an invalidation, so 349 // logically we don't have the block anymore; no need for 350 // further snooping. 351 return true; 352 } 353 354 if (isPendingDirty() || pkt->isInvalidate()) { 355 // We need to save and replay the packet in two cases: 356 // 1. We're awaiting an exclusive copy, so ownership is pending, 357 // and we need to respond after we receive data. 358 // 2. It's an invalidation (e.g., UpgradeReq), and we need 359 // to forward the snoop up the hierarchy after the current 360 // transaction completes. 361 362 // Actual target device (typ. a memory) will delete the 363 // packet on reception, so we need to save a copy here. 364 365 // Clear flags and also allocate new data as the original 366 // packet data storage may have been deleted by the time we 367 // get to send this packet. 368 PacketPtr cp_pkt = new Packet(pkt, true, true); 369 targets.add(cp_pkt, curTick(), _order, Target::FromSnoop, 370 downstreamPending && targets.needsExclusive); 371 372 if (isPendingDirty()) { 373 pkt->assertMemInhibit(); 374 pkt->setSupplyExclusive(); 375 } 376 377 if (pkt->needsExclusive()) { 378 // This transaction will take away our pending copy 379 postInvalidate = true; 380 } 381 } 382 383 if (!pkt->needsExclusive()) { 384 // This transaction will get a read-shared copy, downgrading 385 // our copy if we had an exclusive one 386 postDowngrade = true; 387 pkt->assertShared(); 388 } 389 390 return true; 391} 392 393 394bool 395MSHR::promoteDeferredTargets() 396{ 397 assert(targets.empty()); 398 if (deferredTargets.empty()) { 399 return false; 400 } 401 402 // swap targets & deferredTargets lists 403 std::swap(targets, deferredTargets); 404 405 // clear deferredTargets flags 406 deferredTargets.resetFlags(); 407 408 order = targets.front().order; 409 readyTime = std::max(curTick(), targets.front().readyTime); 410 411 return true; 412} 413 414 415void 416MSHR::handleFill(Packet *pkt, CacheBlk *blk) 417{ 418 if (!pkt->sharedAsserted() 419 && !(hasPostInvalidate() || hasPostDowngrade()) 420 && deferredTargets.needsExclusive) { 421 // We got an exclusive response, but we have deferred targets 422 // which are waiting to request an exclusive copy (not because 423 // of a pending invalidate). This can happen if the original 424 // request was for a read-only (non-exclusive) block, but we 425 // got an exclusive copy anyway because of the E part of the 426 // MOESI/MESI protocol. Since we got the exclusive copy 427 // there's no need to defer the targets, so move them up to 428 // the regular target list. 429 assert(!targets.needsExclusive); 430 targets.needsExclusive = true; 431 // if any of the deferred targets were upper-level cache 432 // requests marked downstreamPending, need to clear that 433 assert(!downstreamPending); // not pending here anymore 434 deferredTargets.clearDownstreamPending(); 435 // this clears out deferredTargets too 436 targets.splice(targets.end(), deferredTargets); 437 deferredTargets.resetFlags(); 438 } 439} 440 441 442bool 443MSHR::checkFunctional(PacketPtr pkt) 444{ 445 // For printing, we treat the MSHR as a whole as single entity. 446 // For other requests, we iterate over the individual targets 447 // since that's where the actual data lies. 448 if (pkt->isPrint()) { 449 pkt->checkFunctional(this, blkAddr, isSecure, blkSize, NULL); 450 return false; 451 } else { 452 return (targets.checkFunctional(pkt) || 453 deferredTargets.checkFunctional(pkt)); 454 } 455} 456 457 458void 459MSHR::print(std::ostream &os, int verbosity, const std::string &prefix) const 460{ 461 ccprintf(os, "%s[%#llx:%#llx](%s) %s %s %s state: %s %s %s %s %s\n", 462 prefix, blkAddr, blkAddr + blkSize - 1, 463 isSecure ? "s" : "ns", 464 isForward ? "Forward" : "", 465 isForwardNoResponse() ? "ForwNoResp" : "", 466 needsExclusive() ? "Excl" : "", 467 _isUncacheable ? "Unc" : "", 468 inService ? "InSvc" : "", 469 downstreamPending ? "DwnPend" : "", 470 hasPostInvalidate() ? "PostInv" : "", 471 hasPostDowngrade() ? "PostDowngr" : ""); 472 473 ccprintf(os, "%s Targets:\n", prefix); 474 targets.print(os, verbosity, prefix + " "); 475 if (!deferredTargets.empty()) { 476 ccprintf(os, "%s Deferred Targets:\n", prefix); 477 deferredTargets.print(os, verbosity, prefix + " "); 478 } 479} 480 481std::string 482MSHR::print() const 483{ 484 ostringstream str; 485 print(str); 486 return str.str(); 487} 488