1/* 2 * Copyright (c) 2012-2013, 2018-2019 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 328 unchanged lines hidden (view full) --- 337 338void 339BaseCache::recvTimingReq(PacketPtr pkt) 340{ 341 // anything that is merely forwarded pays for the forward latency and 342 // the delay provided by the crossbar 343 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 344 |
345 Cycles lat; 346 CacheBlk *blk = nullptr; |
347 bool satisfied = false; 348 { 349 PacketList writebacks; 350 // Note that lat is passed by reference here. The function 351 // access() will set the lat value. 352 satisfied = access(pkt, blk, lat, writebacks); |
353 |
354 // After the evicted blocks are selected, they must be forwarded 355 // to the write buffer to ensure they logically precede anything 356 // happening below 357 doWritebacks(writebacks, clockEdge(lat + forwardLatency)); 358 } 359 |
360 // Here we charge the headerDelay that takes into account the latencies 361 // of the bus, if the packet comes from it. 362 // The latency charged is just the value set by the access() function. 363 // In case of a hit we are neglecting response latency. 364 // In case of a miss we are neglecting forward latency. 365 Tick request_time = clockEdge(lat); 366 // Here we reset the timing of the packet. 367 pkt->headerDelay = pkt->payloadDelay = 0; --- 84 unchanged lines hidden (view full) --- 452 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 453 miss_latency; 454 } else { 455 assert(pkt->req->masterId() < system->maxMasters()); 456 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 457 miss_latency; 458 } 459 |
460 PacketList writebacks; 461 |
462 bool is_fill = !mshr->isForward && 463 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp || 464 mshr->wasWholeLineWrite); 465 466 // make sure that if the mshr was due to a whole line write then 467 // the response is an invalidation 468 assert(!mshr->wasWholeLineWrite || pkt->isInvalidate()); 469 470 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 471 472 if (is_fill && !is_error) { 473 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n", 474 pkt->getAddr()); 475 476 const bool allocate = (writeAllocator && mshr->wasWholeLineWrite) ? 477 writeAllocator->allocate() : mshr->allocOnFill(); |
478 blk = handleFill(pkt, blk, writebacks, allocate); |
479 assert(blk != nullptr); 480 ppFill->notify(pkt); 481 } 482 483 if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) { 484 // The block was marked not readable while there was a pending 485 // cache maintenance operation, restore its flag. 486 blk->status |= BlkReadable; --- 39 unchanged lines hidden (view full) --- 526 clockEdge()); 527 if (next_pf_time != MaxTick) 528 schedMemSideSendEvent(next_pf_time); 529 } 530 } 531 532 // if we used temp block, check to see if its valid and then clear it out 533 if (blk == tempBlock && tempBlock->isValid()) { |
534 evictBlock(blk, writebacks); |
535 } 536 |
537 const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 538 // copy writebacks to write buffer 539 doWritebacks(writebacks, forward_time); 540 |
541 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print()); 542 delete pkt; 543} 544 545 546Tick 547BaseCache::recvAtomic(PacketPtr pkt) 548{ 549 // should assert here that there are no outstanding MSHRs or 550 // writebacks... that would mean that someone used an atomic 551 // access in timing mode 552 553 // We use lookupLatency here because it is used to specify the latency 554 // to access. 555 Cycles lat = lookupLatency; 556 557 CacheBlk *blk = nullptr; |
558 PacketList writebacks; 559 bool satisfied = access(pkt, blk, lat, writebacks); |
560 561 if (pkt->isClean() && blk && blk->isDirty()) { 562 // A cache clean opearation is looking for a dirty 563 // block. If a dirty block is encountered a WriteClean 564 // will update any copies to the path to the memory 565 // until the point of reference. 566 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 567 __func__, pkt->print(), blk->print()); 568 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id); |
569 writebacks.push_back(wb_pkt); |
570 pkt->setSatisfied(); |
571 } 572 |
573 // handle writebacks resulting from the access here to ensure they 574 // logically precede anything happening below 575 doWritebacksAtomic(writebacks); 576 assert(writebacks.empty()); 577 |
578 if (!satisfied) { |
579 lat += handleAtomicReqMiss(pkt, blk, writebacks); |
580 } 581 582 // Note that we don't invoke the prefetcher at all in atomic mode. 583 // It's not clear how to do it properly, particularly for 584 // prefetchers that aggressively generate prefetch candidates and 585 // rely on bandwidth contention to throttle them; these will tend 586 // to pollute the cache in atomic mode since there is no bandwidth 587 // contention. If we ever do want to enable prefetching in atomic 588 // mode, though, this is the place to do it... see timingAccess() 589 // for an example (though we'd want to issue the prefetch(es) 590 // immediately rather than calling requestMemSideBus() as we do 591 // there). 592 |
593 // do any writebacks resulting from the response handling 594 doWritebacksAtomic(writebacks); 595 |
596 // if we used temp block, check to see if its valid and if so 597 // clear it out, but only do so after the call to recvAtomic is 598 // finished so that any downstream observers (such as a snoop 599 // filter), first see the fill, and only then see the eviction 600 if (blk == tempBlock && tempBlock->isValid()) { 601 // the atomic CPU calls recvAtomic for fetch and load/store 602 // sequentuially, and we may already have a tempBlock 603 // writeback from the fetch that we have not yet sent --- 191 unchanged lines hidden (view full) --- 795 } 796 } 797 798 return nullptr; 799} 800 801bool 802BaseCache::updateCompressionData(CacheBlk *blk, const uint64_t* data, |
803 PacketList &writebacks) |
804{ 805 // tempBlock does not exist in the tags, so don't do anything for it. 806 if (blk == tempBlock) { 807 return true; 808 } 809 810 // Get superblock of the given block 811 CompressionBlk* compression_blk = static_cast<CompressionBlk*>(blk); --- 73 unchanged lines hidden (view full) --- 885 compression_blk->setDecompressionLatency(decompression_lat); 886 887 // Evict valid blocks 888 for (const auto& evict_blk : evict_blks) { 889 if (evict_blk->isValid()) { 890 if (evict_blk->wasPrefetched()) { 891 unusedPrefetches++; 892 } |
893 evictBlock(evict_blk, writebacks); |
894 } 895 } 896 897 return true; 898} 899 900void 901BaseCache::satisfyRequest(PacketPtr pkt, CacheBlk *blk, bool, bool) --- 117 unchanged lines hidden (view full) --- 1019 // we find out it is a miss), and use the tag-only latency. 1020 lat = calculateTagOnlyLatency(delay, lookup_lat); 1021 } 1022 1023 return lat; 1024} 1025 1026bool |
1027BaseCache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 1028 PacketList &writebacks) |
1029{ 1030 // sanity check 1031 assert(pkt->isRequest()); 1032 1033 chatty_assert(!(isReadOnly && pkt->isWrite()), 1034 "Should never see a write in a read-only cache %s\n", 1035 name()); 1036 --- 82 unchanged lines hidden (view full) --- 1119 // a tag lookup for simplicity. 1120 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1121 1122 return true; 1123 } 1124 1125 if (!blk) { 1126 // need to do a replacement |
1127 blk = allocateBlock(pkt, writebacks); |
1128 if (!blk) { 1129 // no replaceable block available: give up, fwd to next level. 1130 incMissCount(pkt); 1131 1132 // A writeback searches for the block, then writes the data. 1133 // As the block could not be found, it was a tag-only access. 1134 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1135 1136 return false; 1137 } 1138 1139 blk->status |= BlkReadable; 1140 } else if (compressor) { 1141 // This is an overwrite to an existing block, therefore we need 1142 // to check for data expansion (i.e., block was compressed with 1143 // a smaller size, and now it doesn't fit the entry anymore). 1144 // If that is the case we might need to evict blocks. 1145 if (!updateCompressionData(blk, pkt->getConstPtr<uint64_t>(), |
1146 writebacks)) { |
1147 // This is a failed data expansion (write), which happened 1148 // after finding the replacement entries and accessing the 1149 // block's data. There were no replaceable entries available 1150 // to make room for the expanded block, and since it does not 1151 // fit anymore and it has been properly updated to contain 1152 // the new data, forward it to the next level 1153 lat = calculateAccessLatency(blk, pkt->headerDelay, 1154 tag_latency); --- 59 unchanged lines hidden (view full) --- 1214 // As the block could not be found, it was a tag-only access. 1215 lat = calculateTagOnlyLatency(pkt->headerDelay, tag_latency); 1216 1217 // if this is a write through packet, we don't try to 1218 // allocate if the block is not present 1219 return false; 1220 } else { 1221 // a writeback that misses needs to allocate a new block |
1222 blk = allocateBlock(pkt, writebacks); |
1223 if (!blk) { 1224 // no replaceable block available: give up, fwd to 1225 // next level. 1226 incMissCount(pkt); 1227 1228 // A writeback searches for the block, then writes the 1229 // data. As the block could not be found, it was a tag-only 1230 // access. --- 6 unchanged lines hidden (view full) --- 1237 blk->status |= BlkReadable; 1238 } 1239 } else if (compressor) { 1240 // This is an overwrite to an existing block, therefore we need 1241 // to check for data expansion (i.e., block was compressed with 1242 // a smaller size, and now it doesn't fit the entry anymore). 1243 // If that is the case we might need to evict blocks. 1244 if (!updateCompressionData(blk, pkt->getConstPtr<uint64_t>(), |
1245 writebacks)) { |
1246 // This is a failed data expansion (write), which happened 1247 // after finding the replacement entries and accessing the 1248 // block's data. There were no replaceable entries available 1249 // to make room for the expanded block, and since it does not 1250 // fit anymore and it has been properly updated to contain 1251 // the new data, forward it to the next level 1252 lat = calculateAccessLatency(blk, pkt->headerDelay, 1253 tag_latency); --- 76 unchanged lines hidden (view full) --- 1330 // if we have responded to a cache, and our block is still 1331 // valid, but not dirty, and this cache is mostly exclusive 1332 // with respect to the cache above, drop the block 1333 invalidateBlock(blk); 1334 } 1335} 1336 1337CacheBlk* |
1338BaseCache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks, 1339 bool allocate) |
1340{ 1341 assert(pkt->isResponse()); 1342 Addr addr = pkt->getAddr(); 1343 bool is_secure = pkt->isSecure(); 1344#if TRACING_ON 1345 CacheBlk::State old_state = blk ? blk->status : 0; 1346#endif 1347 1348 // When handling a fill, we should have no writes to this line. 1349 assert(addr == pkt->getBlockAddr(blkSize)); 1350 assert(!writeBuffer.findMatch(addr, is_secure)); 1351 1352 if (!blk) { 1353 // better have read new data... 1354 assert(pkt->hasData() || pkt->cmd == MemCmd::InvalidateResp); 1355 |
1356 // need to do a replacement if allocating, otherwise we stick 1357 // with the temporary storage 1358 blk = allocate ? allocateBlock(pkt, writebacks) : nullptr; |
1359 1360 if (!blk) { 1361 // No replaceable block or a mostly exclusive 1362 // cache... just use temporary storage to complete the 1363 // current request and then get rid of it 1364 blk = tempBlock; 1365 tempBlock->insert(addr, is_secure); 1366 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr, --- 84 unchanged lines hidden (view full) --- 1451 // The block will be ready when the payload arrives and the fill is done 1452 blk->setWhenReady(clockEdge(fillLatency) + pkt->headerDelay + 1453 pkt->payloadDelay); 1454 1455 return blk; 1456} 1457 1458CacheBlk* |
1459BaseCache::allocateBlock(const PacketPtr pkt, PacketList &writebacks) |
1460{ 1461 // Get address 1462 const Addr addr = pkt->getAddr(); 1463 1464 // Get secure bit 1465 const bool is_secure = pkt->isSecure(); 1466 1467 // Block size and compression related access latency. Only relevant if --- 56 unchanged lines hidden (view full) --- 1524 DPRINTF(CacheRepl, "Evicting %s (%#llx) to make room for " \ 1525 "%#llx (%s)\n", blk->print(), regenerateBlkAddr(blk), 1526 addr, is_secure); 1527 1528 if (blk->wasPrefetched()) { 1529 unusedPrefetches++; 1530 } 1531 |
1532 evictBlock(blk, writebacks); |
1533 } 1534 } 1535 1536 replacements++; 1537 } 1538 1539 // If using a compressor, set compression data. This must be done before 1540 // block insertion, as compressed tags use this information. --- 16 unchanged lines hidden (view full) --- 1557 if (blk != tempBlock) { 1558 tags->invalidate(blk); 1559 } else { 1560 tempBlock->invalidate(); 1561 } 1562} 1563 1564void |
1565BaseCache::evictBlock(CacheBlk *blk, PacketList &writebacks) |
1566{ 1567 PacketPtr pkt = evictBlock(blk); 1568 if (pkt) { |
1569 writebacks.push_back(pkt); |
1570 } 1571} 1572 1573PacketPtr 1574BaseCache::writebackBlk(CacheBlk *blk) 1575{ 1576 chatty_assert(!isReadOnly || writebackClean, 1577 "Writeback from read-only cache"); --- 252 unchanged lines hidden (view full) --- 1830 // A cache clean opearation is looking for a dirty 1831 // block. If a dirty block is encountered a WriteClean 1832 // will update any copies to the path to the memory 1833 // until the point of reference. 1834 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n", 1835 __func__, pkt->print(), blk->print()); 1836 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), 1837 pkt->id); |
1838 PacketList writebacks; 1839 writebacks.push_back(wb_pkt); 1840 doWritebacks(writebacks, 0); |
1841 } 1842 1843 return false; 1844 } 1845} 1846 1847bool 1848BaseCache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) --- 846 unchanged lines hidden --- |