| 1/* 2 * Copyright (c) 2010-2015 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 56 unchanged lines hidden (view full) --- 65 66Cache::Cache(const CacheParams *p) 67 : BaseCache(p, p->system->cacheLineSize()), 68 tags(p->tags), 69 prefetcher(p->prefetcher), 70 doFastWrites(true), 71 prefetchOnAccess(p->prefetch_on_access), 72 clusivity(p->clusivity), |
| 73 writebackClean(p->writeback_clean), |
| 74 tempBlockWriteback(nullptr), 75 writebackTempBlockAtomicEvent(this, false, 76 EventBase::Delayed_Writeback_Pri) 77{ 78 tempBlock = new CacheBlk(); 79 tempBlock->data = new uint8_t[blkSize]; 80 81 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this, --- 231 unchanged lines hidden (view full) --- 313 if (pkt->req->isUncacheable()) { 314 DPRINTF(Cache, "%s%s addr %#llx uncacheable\n", pkt->cmdString(), 315 pkt->req->isInstFetch() ? " (ifetch)" : "", 316 pkt->getAddr()); 317 318 // flush and invalidate any existing block 319 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure())); 320 if (old_blk && old_blk->isValid()) { |
| 321 if (old_blk->isDirty() || writebackClean) |
| 322 writebacks.push_back(writebackBlk(old_blk)); 323 else 324 writebacks.push_back(cleanEvictBlk(old_blk)); 325 tags->invalidate(old_blk); 326 old_blk->invalidate(); 327 } 328 329 blk = NULL; --- 9 unchanged lines hidden (view full) --- 339 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat, id); 340 341 DPRINTF(Cache, "%s%s addr %#llx size %d (%s) %s\n", pkt->cmdString(), 342 pkt->req->isInstFetch() ? " (ifetch)" : "", 343 pkt->getAddr(), pkt->getSize(), pkt->isSecure() ? "s" : "ns", 344 blk ? "hit " + blk->print() : "miss"); 345 346 |
| 347 if (pkt->isEviction()) { |
| 348 // We check for presence of block in above caches before issuing 349 // Writeback or CleanEvict to write buffer. Therefore the only 350 // possible cases can be of a CleanEvict packet coming from above 351 // encountering a Writeback generated in this cache peer cache and 352 // waiting in the write buffer. Cases of upper level peer caches 353 // generating CleanEvict and Writeback or simply CleanEvict and 354 // CleanEvict almost simultaneously will be caught by snoops sent out 355 // by crossbar. 356 std::vector<MSHR *> outgoing; 357 if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(), 358 outgoing)) { 359 assert(outgoing.size() == 1); |
| 360 MSHR *wb_entry = outgoing[0]; 361 assert(wb_entry->getNumTargets() == 1); 362 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 363 assert(wbPkt->isWriteback()); 364 365 if (pkt->isCleanEviction()) { 366 // The CleanEvict and WritebackClean snoops into other 367 // peer caches of the same level while traversing the 368 // crossbar. If a copy of the block is found, the 369 // packet is deleted in the crossbar. Hence, none of 370 // the other upper level caches connected to this 371 // cache have the block, so we can clear the 372 // BLOCK_CACHED flag in the Writeback if set and 373 // discard the CleanEvict by returning true. 374 wbPkt->clearBlockCached(); 375 return true; 376 } else { 377 assert(pkt->cmd == MemCmd::WritebackDirty); 378 // Dirty writeback from above trumps our clean 379 // writeback... discard here 380 // Note: markInService will remove entry from writeback buffer. 381 markInService(wb_entry, false); 382 delete wbPkt; 383 } |
| 384 } 385 } 386 387 // Writeback handling is special case. We can write the block into 388 // the cache without having a writeable copy (or any copy at all). |
| 389 if (pkt->isWriteback()) { |
| 390 assert(blkSize == pkt->getSize()); |
| 391 392 // we could get a clean writeback while we are having 393 // outstanding accesses to a block, do the simple thing for 394 // now and drop the clean writeback so that we do not upset 395 // any ordering/decisions about ownership already taken 396 if (pkt->cmd == MemCmd::WritebackClean && 397 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) { 398 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, " 399 "dropping\n", pkt->getAddr()); 400 return true; 401 } 402 |
| 403 if (blk == NULL) { 404 // need to do a replacement 405 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks); 406 if (blk == NULL) { 407 // no replaceable block available: give up, fwd to next level. 408 incMissCount(pkt); 409 return false; 410 } 411 tags->insertBlock(pkt, blk); 412 413 blk->status = (BlkValid | BlkReadable); 414 if (pkt->isSecure()) { 415 blk->status |= BlkSecure; 416 } 417 } |
| 418 // only mark the block dirty if we got a writeback command, 419 // and leave it as is for a clean writeback 420 if (pkt->cmd == MemCmd::WritebackDirty) { 421 blk->status |= BlkDirty; 422 } |
| 423 // if shared is not asserted we got the writeback in modified 424 // state, if it is asserted we are in the owned state 425 if (!pkt->sharedAsserted()) { 426 blk->status |= BlkWritable; 427 } 428 // nothing else to do; writeback doesn't expect response 429 assert(!pkt->needsResponse()); 430 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize); --- 55 unchanged lines hidden (view full) --- 486 // in Writebacks and discard CleanEvicts. 487 if (isCachedAbove(wbPkt)) { 488 if (wbPkt->cmd == MemCmd::CleanEvict) { 489 // Delete CleanEvict because cached copies exist above. The 490 // packet destructor will delete the request object because 491 // this is a non-snoop request packet which does not require a 492 // response. 493 delete wbPkt; |
| 494 } else if (wbPkt->cmd == MemCmd::WritebackClean) { 495 // clean writeback, do not send since the block is 496 // still cached above 497 assert(writebackClean); 498 delete wbPkt; |
| 499 } else { |
| 500 assert(wbPkt->cmd == MemCmd::WritebackDirty); |
| 501 // Set BLOCK_CACHED flag in Writeback and send below, so that 502 // the Writeback does not reset the bit corresponding to this 503 // address in the snoop filter below. 504 wbPkt->setBlockCached(); 505 allocateWriteBuffer(wbPkt, forward_time); 506 } 507 } else { 508 // If the block is not cached above, send packet below. Both --- 10 unchanged lines hidden (view full) --- 519Cache::doWritebacksAtomic(PacketList& writebacks) 520{ 521 while (!writebacks.empty()) { 522 PacketPtr wbPkt = writebacks.front(); 523 // Call isCachedAbove for both Writebacks and CleanEvicts. If 524 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks 525 // and discard CleanEvicts. 526 if (isCachedAbove(wbPkt, false)) { |
| 527 if (wbPkt->cmd == MemCmd::WritebackDirty) { |
| 528 // Set BLOCK_CACHED flag in Writeback and send below, 529 // so that the Writeback does not reset the bit 530 // corresponding to this address in the snoop filter 531 // below. We can discard CleanEvicts because cached 532 // copies exist above. Atomic mode isCachedAbove 533 // modifies packet to set BLOCK_CACHED flag 534 memSidePort->sendAtomic(wbPkt); 535 } --- 187 unchanged lines hidden (view full) --- 723 724 // In this case we are considering request_time that takes 725 // into account the delay of the xbar, if any, and just 726 // lat, neglecting responseLatency, modelling hit latency 727 // just as lookupLatency or or the value of lat overriden 728 // by access(), that calls accessBlock() function. 729 cpuSidePort->schedTimingResp(pkt, request_time, true); 730 } else { |
| 731 DPRINTF(Cache, "%s satisfied %s addr %#llx, no response needed\n", 732 __func__, pkt->cmdString(), pkt->getAddr(), 733 pkt->getSize()); 734 |
| 735 // queue the packet for deletion, as the sending cache is 736 // still relying on it; if the block is found in access(), 737 // CleanEvict and Writeback messages will be deleted 738 // here as well 739 pendingDelete.reset(pkt); 740 } 741 } else { 742 // miss --- 55 unchanged lines hidden (view full) --- 798 /// MSHR hit 799 /// @note writebacks will be checked in getNextMSHR() 800 /// for any conflicting requests to the same block 801 802 //@todo remove hw_pf here 803 804 // Coalesce unless it was a software prefetch (see above). 805 if (pkt) { |
| 806 assert(!pkt->isWriteback()); 807 // CleanEvicts corresponding to blocks which have 808 // outstanding requests in MSHRs are simply sunk here |
| 809 if (pkt->cmd == MemCmd::CleanEvict) { 810 pendingDelete.reset(pkt); 811 } else { 812 DPRINTF(Cache, "%s coalescing MSHR for %s addr %#llx size %d\n", 813 __func__, pkt->cmdString(), pkt->getAddr(), 814 pkt->getSize()); 815 816 assert(pkt->req->masterId() < system->maxMasters()); --- 36 unchanged lines hidden (view full) --- 853 // no MSHR 854 assert(pkt->req->masterId() < system->maxMasters()); 855 if (pkt->req->isUncacheable()) { 856 mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++; 857 } else { 858 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++; 859 } 860 |
| 861 if (pkt->isEviction() || |
| 862 (pkt->req->isUncacheable() && pkt->isWrite())) { 863 // We use forward_time here because there is an 864 // uncached memory write, forwarded to WriteBuffer. 865 allocateWriteBuffer(pkt, forward_time); 866 } else { 867 if (blk && blk->isValid()) { 868 // should have flushed and have no valid block 869 assert(!pkt->req->isUncacheable()); --- 51 unchanged lines hidden (view full) --- 921 // and the cache could have filled again before we actually 922 // send out the forwarded uncacheable request (blk could thus 923 // be non-null) 924 return NULL; 925 } 926 927 if (!blkValid && 928 (cpu_pkt->isUpgrade() || |
| 929 cpu_pkt->isEviction())) { |
| 930 // Writebacks that weren't allocated in access() and upgrades 931 // from upper-level caches that missed completely just go 932 // through. 933 return NULL; 934 } 935 936 assert(cpu_pkt->needsResponse()); 937 --- 203 unchanged lines hidden (view full) --- 1141 } else { 1142 // the writeback/clean eviction happens after the call to 1143 // recvAtomic has finished (but before any successive 1144 // calls), so that the response handling from the fill is 1145 // allowed to happen first 1146 schedule(writebackTempBlockAtomicEvent, curTick()); 1147 } 1148 |
| 1149 tempBlockWriteback = (blk->isDirty() || writebackClean) ? 1150 writebackBlk(blk) : cleanEvictBlk(blk); |
| 1151 blk->invalidate(); 1152 } 1153 1154 if (pkt->needsResponse()) { 1155 pkt->makeAtomicResponse(); 1156 } 1157 1158 return lat * clockPeriod(); --- 332 unchanged lines hidden (view full) --- 1491 doWritebacks(writebacks, forward_time); 1492 1493 // if we used temp block, check to see if its valid and then clear it out 1494 if (blk == tempBlock && tempBlock->isValid()) { 1495 // We use forwardLatency here because we are copying 1496 // Writebacks/CleanEvicts to write buffer. It specifies the latency to 1497 // allocate an internal buffer and to schedule an event to the 1498 // queued port. |
| 1499 if (blk->isDirty() || writebackClean) { |
| 1500 PacketPtr wbPkt = writebackBlk(blk); 1501 allocateWriteBuffer(wbPkt, forward_time); 1502 // Set BLOCK_CACHED flag if cached above. 1503 if (isCachedAbove(wbPkt)) 1504 wbPkt->setBlockCached(); 1505 } else { 1506 PacketPtr wcPkt = cleanEvictBlk(blk); 1507 // Check to see if block is cached above. If not allocate --- 9 unchanged lines hidden (view full) --- 1517 DPRINTF(Cache, "Leaving %s with %s for addr %#llx\n", __func__, 1518 pkt->cmdString(), pkt->getAddr()); 1519 delete pkt; 1520} 1521 1522PacketPtr 1523Cache::writebackBlk(CacheBlk *blk) 1524{ |
| 1525 chatty_assert(!isReadOnly || writebackClean, 1526 "Writeback from read-only cache"); 1527 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean)); |
| 1528 1529 writebacks[Request::wbMasterId]++; 1530 |
| 1531 Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set), 1532 blkSize, 0, Request::wbMasterId); |
| 1533 if (blk->isSecure()) |
| 1534 req->setFlags(Request::SECURE); |
| 1535 |
| 1536 req->taskId(blk->task_id); |
| 1537 blk->task_id= ContextSwitchTaskId::Unknown; 1538 blk->tickInserted = curTick(); 1539 |
| 1540 PacketPtr pkt = 1541 new Packet(req, blk->isDirty() ? 1542 MemCmd::WritebackDirty : MemCmd::WritebackClean); 1543 1544 DPRINTF(Cache, "Create Writeback %#llx writable: %d, dirty: %d\n", 1545 pkt->getAddr(), blk->isWritable(), blk->isDirty()); 1546 |
| 1547 if (blk->isWritable()) { 1548 // not asserting shared means we pass the block in modified 1549 // state, mark our own block non-writeable 1550 blk->status &= ~BlkWritable; 1551 } else { 1552 // we are in the owned state, tell the receiver |
| 1553 pkt->assertShared(); |
| 1554 } 1555 |
| 1556 // make sure the block is not marked dirty |
| 1557 blk->status &= ~BlkDirty; |
| 1558 1559 pkt->allocate(); 1560 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize); 1561 1562 return pkt; |
| 1563} 1564 1565PacketPtr 1566Cache::cleanEvictBlk(CacheBlk *blk) 1567{ |
| 1568 assert(!writebackClean); |
| 1569 assert(blk && blk->isValid() && !blk->isDirty()); 1570 // Creating a zero sized write, a message to the snoop filter 1571 Request *req = 1572 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0, 1573 Request::wbMasterId); 1574 if (blk->isSecure()) 1575 req->setFlags(Request::SECURE); 1576 --- 93 unchanged lines hidden (view full) --- 1670 } else { 1671 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx (%s): %s\n", 1672 repl_addr, blk->isSecure() ? "s" : "ns", 1673 addr, is_secure ? "s" : "ns", 1674 blk->isDirty() ? "writeback" : "clean"); 1675 1676 // Will send up Writeback/CleanEvict snoops via isCachedAbove 1677 // when pushing this writeback list into the write buffer. |
| 1678 if (blk->isDirty() || writebackClean) { |
| 1679 // Save writeback packet for handling by caller 1680 writebacks.push_back(writebackBlk(blk)); 1681 } else { 1682 writebacks.push_back(cleanEvictBlk(blk)); 1683 } 1684 } 1685 } 1686 --- 406 unchanged lines hidden (view full) --- 2093 MSHR *wb_entry = writebacks[0]; 2094 // Expect to see only Writebacks and/or CleanEvicts here, both of 2095 // which should not be generated for uncacheable data. 2096 assert(!wb_entry->isUncacheable()); 2097 // There should only be a single request responsible for generating 2098 // Writebacks/CleanEvicts. 2099 assert(wb_entry->getNumTargets() == 1); 2100 PacketPtr wb_pkt = wb_entry->getTarget()->pkt; |
| 2101 assert(wb_pkt->isEviction()); |
| 2102 |
| 2103 if (pkt->isEviction()) { |
| 2104 // if the block is found in the write queue, set the BLOCK_CACHED 2105 // flag for Writeback/CleanEvict snoop. On return the snoop will 2106 // propagate the BLOCK_CACHED flag in Writeback packets and prevent 2107 // any CleanEvicts from travelling down the memory hierarchy. 2108 pkt->setBlockCached(); 2109 DPRINTF(Cache, "Squashing %s from lower cache on writequeue hit" 2110 " %#x\n", pkt->cmdString(), pkt->getAddr()); 2111 return; 2112 } 2113 |
| 2114 if (wb_pkt->cmd == MemCmd::WritebackDirty) { |
| 2115 assert(!pkt->memInhibitAsserted()); 2116 pkt->assertMemInhibit(); 2117 if (!pkt->needsExclusive()) { 2118 pkt->assertShared(); 2119 // the writeback is no longer passing exclusivity (the 2120 // receiving cache should consider the block owned 2121 // rather than modified) 2122 wb_pkt->assertShared(); 2123 } else { 2124 // if we're not asserting the shared line, we need to 2125 // invalidate our copy. we'll do that below as long as 2126 // the packet's invalidate flag is set... 2127 assert(pkt->isInvalidate()); 2128 } 2129 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(), 2130 false, false); 2131 } else { |
| 2132 // on hitting a clean writeback we play it safe and do not 2133 // provide a response, the block may be dirty somewhere 2134 // else 2135 assert(wb_pkt->isCleanEviction()); |
| 2136 // The cache technically holds the block until the |
| 2137 // corresponding message reaches the crossbar |
| 2138 // below. Therefore when a snoop encounters a CleanEvict |
| 2139 // or WritebackClean message we must set assertShared 2140 // (just like when it encounters a Writeback) to avoid the 2141 // snoop filter prematurely clearing the holder bit in the 2142 // crossbar below 2143 if (!pkt->needsExclusive()) { |
| 2144 pkt->assertShared(); |
| 2145 // the writeback is no longer passing exclusivity (the 2146 // receiving cache should consider the block owned 2147 // rather than modified) 2148 wb_pkt->assertShared(); 2149 } else { |
| 2150 assert(pkt->isInvalidate()); |
| 2151 } |
| 2152 } 2153 2154 if (pkt->isInvalidate()) { 2155 // Invalidation trumps our writeback... discard here 2156 // Note: markInService will remove entry from writeback buffer. 2157 markInService(wb_entry, false); 2158 delete wb_pkt; 2159 } --- 133 unchanged lines hidden (view full) --- 2293 // packet, the cache can inform the crossbar below of presence or absence 2294 // of the block. 2295 if (is_timing) { 2296 Packet snoop_pkt(pkt, true, false); 2297 snoop_pkt.setExpressSnoop(); 2298 // Assert that packet is either Writeback or CleanEvict and not a 2299 // prefetch request because prefetch requests need an MSHR and may 2300 // generate a snoop response. |
| 2301 assert(pkt->isEviction()); |
| 2302 snoop_pkt.senderState = NULL; 2303 cpuSidePort->sendTimingSnoopReq(&snoop_pkt); 2304 // Writeback/CleanEvict snoops do not generate a snoop response. 2305 assert(!(snoop_pkt.memInhibitAsserted())); 2306 return snoop_pkt.isBlockCached(); 2307 } else { 2308 cpuSidePort->sendAtomicSnoop(pkt); 2309 return pkt->isBlockCached(); --- 52 unchanged lines hidden (view full) --- 2362 } 2363 2364 if (snoop_pkt.isBlockCached() || blk != NULL) { 2365 DPRINTF(Cache, "Block present, prefetch squashed by cache. " 2366 "Deallocating mshr target %#x.\n", 2367 mshr->blkAddr); 2368 2369 // Deallocate the mshr target |
| 2370 if (!tgt_pkt->isWriteback()) { |
| 2371 if (mshr->queue->forceDeallocateTarget(mshr)) { 2372 // Clear block if this deallocation resulted freed an 2373 // mshr when all had previously been utilized 2374 clearBlocked((BlockedCause)(mshr->queue->index)); 2375 } 2376 return NULL; 2377 } else { 2378 // If this is a Writeback, and the snoops indicate that the blk --- 276 unchanged lines hidden --- |