cache.cc (11352:4e195fb9ec4f) | cache.cc (11375:f98df9231cdd) |
---|---|
1/* | 1/* |
2 * Copyright (c) 2010-2015 ARM Limited | 2 * Copyright (c) 2010-2016 ARM Limited |
3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated --- 270 unchanged lines hidden (view full) --- 281 // for invalidations we could be looking at the temp block 282 // (for upgrades we always allocate) 283 invalidateBlock(blk); 284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n", 285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 286 } 287} 288 | 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated --- 270 unchanged lines hidden (view full) --- 281 // for invalidations we could be looking at the temp block 282 // (for upgrades we always allocate) 283 invalidateBlock(blk); 284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n", 285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 286 } 287} 288 |
289 | |
290///////////////////////////////////////////////////// 291// | 289///////////////////////////////////////////////////// 290// |
292// MSHR helper functions 293// 294///////////////////////////////////////////////////// 295 296 297void 298Cache::markInService(MSHR *mshr, bool pending_modified_resp) 299{ 300 markInServiceInternal(mshr, pending_modified_resp); 301} 302 303///////////////////////////////////////////////////// 304// | |
305// Access path: requests coming in from the CPU side 306// 307///////////////////////////////////////////////////// 308 309bool 310Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 311 PacketList &writebacks) 312{ --- 45 unchanged lines hidden (view full) --- 358 // We check for presence of block in above caches before issuing 359 // Writeback or CleanEvict to write buffer. Therefore the only 360 // possible cases can be of a CleanEvict packet coming from above 361 // encountering a Writeback generated in this cache peer cache and 362 // waiting in the write buffer. Cases of upper level peer caches 363 // generating CleanEvict and Writeback or simply CleanEvict and 364 // CleanEvict almost simultaneously will be caught by snoops sent out 365 // by crossbar. | 291// Access path: requests coming in from the CPU side 292// 293///////////////////////////////////////////////////// 294 295bool 296Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 297 PacketList &writebacks) 298{ --- 45 unchanged lines hidden (view full) --- 344 // We check for presence of block in above caches before issuing 345 // Writeback or CleanEvict to write buffer. Therefore the only 346 // possible cases can be of a CleanEvict packet coming from above 347 // encountering a Writeback generated in this cache peer cache and 348 // waiting in the write buffer. Cases of upper level peer caches 349 // generating CleanEvict and Writeback or simply CleanEvict and 350 // CleanEvict almost simultaneously will be caught by snoops sent out 351 // by crossbar. |
366 std::vector<MSHR *> outgoing; 367 if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(), 368 outgoing)) { 369 assert(outgoing.size() == 1); 370 MSHR *wb_entry = outgoing[0]; | 352 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(), 353 pkt->isSecure()); 354 if (wb_entry) { |
371 assert(wb_entry->getNumTargets() == 1); 372 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 373 assert(wbPkt->isWriteback()); 374 375 if (pkt->isCleanEviction()) { 376 // The CleanEvict and WritebackClean snoops into other 377 // peer caches of the same level while traversing the 378 // crossbar. If a copy of the block is found, the --- 4 unchanged lines hidden (view full) --- 383 // discard the CleanEvict by returning true. 384 wbPkt->clearBlockCached(); 385 return true; 386 } else { 387 assert(pkt->cmd == MemCmd::WritebackDirty); 388 // Dirty writeback from above trumps our clean 389 // writeback... discard here 390 // Note: markInService will remove entry from writeback buffer. | 355 assert(wb_entry->getNumTargets() == 1); 356 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 357 assert(wbPkt->isWriteback()); 358 359 if (pkt->isCleanEviction()) { 360 // The CleanEvict and WritebackClean snoops into other 361 // peer caches of the same level while traversing the 362 // crossbar. If a copy of the block is found, the --- 4 unchanged lines hidden (view full) --- 367 // discard the CleanEvict by returning true. 368 wbPkt->clearBlockCached(); 369 return true; 370 } else { 371 assert(pkt->cmd == MemCmd::WritebackDirty); 372 // Dirty writeback from above trumps our clean 373 // writeback... discard here 374 // Note: markInService will remove entry from writeback buffer. |
391 markInService(wb_entry, false); | 375 markInService(wb_entry); |
392 delete wbPkt; 393 } 394 } 395 } 396 397 // Writeback handling is special case. We can write the block into 398 // the cache without having a writeable copy (or any copy at all). 399 if (pkt->isWriteback()) { --- 834 unchanged lines hidden (view full) --- 1234///////////////////////////////////////////////////// 1235// 1236// Response handling: responses from the memory side 1237// 1238///////////////////////////////////////////////////// 1239 1240 1241void | 376 delete wbPkt; 377 } 378 } 379 } 380 381 // Writeback handling is special case. We can write the block into 382 // the cache without having a writeable copy (or any copy at all). 383 if (pkt->isWriteback()) { --- 834 unchanged lines hidden (view full) --- 1218///////////////////////////////////////////////////// 1219// 1220// Response handling: responses from the memory side 1221// 1222///////////////////////////////////////////////////// 1223 1224 1225void |
1226Cache::handleUncacheableWriteResp(PacketPtr pkt) 1227{ 1228 WriteQueueEntry *wq_entry = 1229 dynamic_cast<WriteQueueEntry*>(pkt->senderState); 1230 assert(wq_entry); 1231 1232 WriteQueueEntry::Target *target = wq_entry->getTarget(); 1233 Packet *tgt_pkt = target->pkt; 1234 1235 // we send out invalidation reqs and get invalidation 1236 // responses for write-line requests 1237 assert(tgt_pkt->cmd != MemCmd::WriteLineReq); 1238 1239 int stats_cmd_idx = tgt_pkt->cmdToIndex(); 1240 Tick miss_latency = curTick() - target->recvTime; 1241 assert(pkt->req->masterId() < system->maxMasters()); 1242 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1243 miss_latency; 1244 1245 tgt_pkt->makeTimingResponse(); 1246 // if this packet is an error copy that to the new packet 1247 if (pkt->isError()) 1248 tgt_pkt->copyError(pkt); 1249 // Reset the bus additional time as it is now accounted for 1250 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0; 1251 Tick completion_time = clockEdge(responseLatency) + 1252 pkt->headerDelay + pkt->payloadDelay; 1253 1254 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true); 1255 1256 wq_entry->popTarget(); 1257 assert(!wq_entry->hasTargets()); 1258 1259 bool wasFull = writeBuffer.isFull(); 1260 writeBuffer.deallocate(wq_entry); 1261 1262 if (wasFull && !writeBuffer.isFull()) { 1263 clearBlocked(Blocked_NoWBBuffers); 1264 } 1265 1266 delete pkt; 1267} 1268 1269void |
|
1242Cache::recvTimingResp(PacketPtr pkt) 1243{ 1244 assert(pkt->isResponse()); 1245 1246 // all header delay should be paid for by the crossbar, unless 1247 // this is a prefetch response from above 1248 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 1249 "%s saw a non-zero packet delay\n", name()); 1250 | 1270Cache::recvTimingResp(PacketPtr pkt) 1271{ 1272 assert(pkt->isResponse()); 1273 1274 // all header delay should be paid for by the crossbar, unless 1275 // this is a prefetch response from above 1276 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 1277 "%s saw a non-zero packet delay\n", name()); 1278 |
1251 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState); | |
1252 bool is_error = pkt->isError(); 1253 | 1279 bool is_error = pkt->isError(); 1280 |
1254 assert(mshr); 1255 | |
1256 if (is_error) { 1257 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), " 1258 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns", 1259 pkt->cmdString()); 1260 } 1261 1262 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n", 1263 pkt->cmdString(), pkt->getAddr(), pkt->getSize(), 1264 pkt->isSecure() ? "s" : "ns"); 1265 | 1281 if (is_error) { 1282 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), " 1283 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns", 1284 pkt->cmdString()); 1285 } 1286 1287 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n", 1288 pkt->cmdString(), pkt->getAddr(), pkt->getSize(), 1289 pkt->isSecure() ? "s" : "ns"); 1290 |
1266 MSHRQueue *mq = mshr->queue; 1267 bool wasFull = mq->isFull(); | 1291 // if this is a write, we should be looking at an uncacheable 1292 // write 1293 if (pkt->isWrite()) { 1294 assert(pkt->req->isUncacheable()); 1295 handleUncacheableWriteResp(pkt); 1296 return; 1297 } |
1268 | 1298 |
1299 // we have dealt with any (uncacheable) writes above, from here on 1300 // we know we are dealing with an MSHR due to a miss or a prefetch 1301 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState); 1302 assert(mshr); 1303 |
|
1269 if (mshr == noTargetMSHR) { 1270 // we always clear at least one target 1271 clearBlocked(Blocked_NoTargets); 1272 noTargetMSHR = NULL; 1273 } 1274 1275 // Initial target is used just for stats 1276 MSHR::Target *initial_tgt = mshr->getTarget(); 1277 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 1278 Tick miss_latency = curTick() - initial_tgt->recvTime; | 1304 if (mshr == noTargetMSHR) { 1305 // we always clear at least one target 1306 clearBlocked(Blocked_NoTargets); 1307 noTargetMSHR = NULL; 1308 } 1309 1310 // Initial target is used just for stats 1311 MSHR::Target *initial_tgt = mshr->getTarget(); 1312 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 1313 Tick miss_latency = curTick() - initial_tgt->recvTime; |
1279 PacketList writebacks; 1280 // We need forward_time here because we have a call of 1281 // allocateWriteBuffer() that need this parameter to specify the 1282 // time to request the bus. In this case we use forward latency 1283 // because there is a writeback. We pay also here for headerDelay 1284 // that is charged of bus latencies if the packet comes from the 1285 // bus. 1286 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; | |
1287 1288 if (pkt->req->isUncacheable()) { 1289 assert(pkt->req->masterId() < system->maxMasters()); 1290 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1291 miss_latency; 1292 } else { 1293 assert(pkt->req->masterId() < system->maxMasters()); 1294 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 1295 miss_latency; 1296 } 1297 | 1314 1315 if (pkt->req->isUncacheable()) { 1316 assert(pkt->req->masterId() < system->maxMasters()); 1317 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1318 miss_latency; 1319 } else { 1320 assert(pkt->req->masterId() < system->maxMasters()); 1321 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 1322 miss_latency; 1323 } 1324 |
1325 bool wasFull = mshrQueue.isFull(); 1326 1327 PacketList writebacks; 1328 1329 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 1330 |
|
1298 // upgrade deferred targets if the response has no sharers, and is 1299 // thus passing writable 1300 if (!pkt->hasSharers()) { 1301 mshr->promoteWritable(); 1302 } 1303 1304 bool is_fill = !mshr->isForward && 1305 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp); --- 159 unchanged lines hidden (view full) --- 1465 } 1466 1467 if (mshr->promoteDeferredTargets()) { 1468 // avoid later read getting stale data while write miss is 1469 // outstanding.. see comment in timingAccess() 1470 if (blk) { 1471 blk->status &= ~BlkReadable; 1472 } | 1331 // upgrade deferred targets if the response has no sharers, and is 1332 // thus passing writable 1333 if (!pkt->hasSharers()) { 1334 mshr->promoteWritable(); 1335 } 1336 1337 bool is_fill = !mshr->isForward && 1338 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp); --- 159 unchanged lines hidden (view full) --- 1498 } 1499 1500 if (mshr->promoteDeferredTargets()) { 1501 // avoid later read getting stale data while write miss is 1502 // outstanding.. see comment in timingAccess() 1503 if (blk) { 1504 blk->status &= ~BlkReadable; 1505 } |
1473 mq = mshr->queue; 1474 mq->markPending(mshr); | 1506 mshrQueue.markPending(mshr); |
1475 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 1476 } else { | 1507 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 1508 } else { |
1477 mq->deallocate(mshr); 1478 if (wasFull && !mq->isFull()) { 1479 clearBlocked((BlockedCause)mq->index); | 1509 mshrQueue.deallocate(mshr); 1510 if (wasFull && !mshrQueue.isFull()) { 1511 clearBlocked(Blocked_NoMSHRs); |
1480 } 1481 1482 // Request the bus for a prefetch if this deallocation freed enough 1483 // MSHRs for a prefetch to take place | 1512 } 1513 1514 // Request the bus for a prefetch if this deallocation freed enough 1515 // MSHRs for a prefetch to take place |
1484 if (prefetcher && mq == &mshrQueue && mshrQueue.canPrefetch()) { | 1516 if (prefetcher && mshrQueue.canPrefetch()) { |
1485 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 1486 clockEdge()); 1487 if (next_pf_time != MaxTick) 1488 schedMemSideSendEvent(next_pf_time); 1489 } 1490 } 1491 // reset the xbar additional timinig as it is now accounted for 1492 pkt->headerDelay = pkt->payloadDelay = 0; --- 217 unchanged lines hidden (view full) --- 1710{ 1711 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq); 1712 Addr addr = pkt->getAddr(); 1713 bool is_secure = pkt->isSecure(); 1714#if TRACING_ON 1715 CacheBlk::State old_state = blk ? blk->status : 0; 1716#endif 1717 | 1517 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 1518 clockEdge()); 1519 if (next_pf_time != MaxTick) 1520 schedMemSideSendEvent(next_pf_time); 1521 } 1522 } 1523 // reset the xbar additional timinig as it is now accounted for 1524 pkt->headerDelay = pkt->payloadDelay = 0; --- 217 unchanged lines hidden (view full) --- 1742{ 1743 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq); 1744 Addr addr = pkt->getAddr(); 1745 bool is_secure = pkt->isSecure(); 1746#if TRACING_ON 1747 CacheBlk::State old_state = blk ? blk->status : 0; 1748#endif 1749 |
1718 // When handling a fill, discard any CleanEvicts for the 1719 // same address in write buffer. 1720 Addr M5_VAR_USED blk_addr = blockAlign(pkt->getAddr()); 1721 std::vector<MSHR *> M5_VAR_USED wbs; 1722 assert (!writeBuffer.findMatches(blk_addr, is_secure, wbs)); | 1750 // When handling a fill, we should have no writes to this line. 1751 assert(addr == blockAlign(addr)); 1752 assert(!writeBuffer.findMatch(addr, is_secure)); |
1723 1724 if (blk == NULL) { 1725 // better have read new data... 1726 assert(pkt->hasData()); 1727 1728 // only read responses and write-line requests have data; 1729 // note that we don't write the data here for write-line - that 1730 // happens in the subsequent satisfyCpuSideRequest. --- 371 unchanged lines hidden (view full) --- 2102 mshr->print()); 2103 2104 if (mshr->getNumTargets() > numTarget) 2105 warn("allocating bonus target for snoop"); //handle later 2106 return; 2107 } 2108 2109 //We also need to check the writeback buffers and handle those | 1753 1754 if (blk == NULL) { 1755 // better have read new data... 1756 assert(pkt->hasData()); 1757 1758 // only read responses and write-line requests have data; 1759 // note that we don't write the data here for write-line - that 1760 // happens in the subsequent satisfyCpuSideRequest. --- 371 unchanged lines hidden (view full) --- 2132 mshr->print()); 2133 2134 if (mshr->getNumTargets() > numTarget) 2135 warn("allocating bonus target for snoop"); //handle later 2136 return; 2137 } 2138 2139 //We also need to check the writeback buffers and handle those |
2110 std::vector<MSHR *> writebacks; 2111 if (writeBuffer.findMatches(blk_addr, is_secure, writebacks)) { | 2140 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure); 2141 if (wb_entry) { |
2112 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n", 2113 pkt->getAddr(), is_secure ? "s" : "ns"); | 2142 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n", 2143 pkt->getAddr(), is_secure ? "s" : "ns"); |
2114 2115 // Look through writebacks for any cachable writes. 2116 // We should only ever find a single match 2117 assert(writebacks.size() == 1); 2118 MSHR *wb_entry = writebacks[0]; | |
2119 // Expect to see only Writebacks and/or CleanEvicts here, both of 2120 // which should not be generated for uncacheable data. 2121 assert(!wb_entry->isUncacheable()); 2122 // There should only be a single request responsible for generating 2123 // Writebacks/CleanEvicts. 2124 assert(wb_entry->getNumTargets() == 1); 2125 PacketPtr wb_pkt = wb_entry->getTarget()->pkt; 2126 assert(wb_pkt->isEviction()); --- 34 unchanged lines hidden (view full) --- 2161 2162 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(), 2163 false, false); 2164 } 2165 2166 if (invalidate) { 2167 // Invalidation trumps our writeback... discard here 2168 // Note: markInService will remove entry from writeback buffer. | 2144 // Expect to see only Writebacks and/or CleanEvicts here, both of 2145 // which should not be generated for uncacheable data. 2146 assert(!wb_entry->isUncacheable()); 2147 // There should only be a single request responsible for generating 2148 // Writebacks/CleanEvicts. 2149 assert(wb_entry->getNumTargets() == 1); 2150 PacketPtr wb_pkt = wb_entry->getTarget()->pkt; 2151 assert(wb_pkt->isEviction()); --- 34 unchanged lines hidden (view full) --- 2186 2187 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(), 2188 false, false); 2189 } 2190 2191 if (invalidate) { 2192 // Invalidation trumps our writeback... discard here 2193 // Note: markInService will remove entry from writeback buffer. |
2169 markInService(wb_entry, false); | 2194 markInService(wb_entry); |
2170 delete wb_pkt; 2171 } 2172 } 2173 2174 // If this was a shared writeback, there may still be 2175 // other shared copies above that require invalidation. 2176 // We could be more selective and return here if the 2177 // request is non-exclusive or if the writeback is --- 26 unchanged lines hidden (view full) --- 2204 } 2205 2206 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 2207 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false); 2208 return snoop_delay + lookupLatency * clockPeriod(); 2209} 2210 2211 | 2195 delete wb_pkt; 2196 } 2197 } 2198 2199 // If this was a shared writeback, there may still be 2200 // other shared copies above that require invalidation. 2201 // We could be more selective and return here if the 2202 // request is non-exclusive or if the writeback is --- 26 unchanged lines hidden (view full) --- 2229 } 2230 2231 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 2232 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false); 2233 return snoop_delay + lookupLatency * clockPeriod(); 2234} 2235 2236 |
2212MSHR * 2213Cache::getNextMSHR() | 2237QueueEntry* 2238Cache::getNextQueueEntry() |
2214{ 2215 // Check both MSHR queue and write buffer for potential requests, 2216 // note that null does not mean there is no request, it could 2217 // simply be that it is not ready | 2239{ 2240 // Check both MSHR queue and write buffer for potential requests, 2241 // note that null does not mean there is no request, it could 2242 // simply be that it is not ready |
2218 MSHR *miss_mshr = mshrQueue.getNextMSHR(); 2219 MSHR *write_mshr = writeBuffer.getNextMSHR(); | 2243 MSHR *miss_mshr = mshrQueue.getNext(); 2244 WriteQueueEntry *wq_entry = writeBuffer.getNext(); |
2220 2221 // If we got a write buffer request ready, first priority is a | 2245 2246 // If we got a write buffer request ready, first priority is a |
2222 // full write buffer, otherwhise we favour the miss requests 2223 if (write_mshr && 2224 ((writeBuffer.isFull() && writeBuffer.inServiceEntries == 0) || | 2247 // full write buffer (but only if we have no uncacheable write 2248 // responses outstanding, possibly revisit this last part), 2249 // otherwhise we favour the miss requests 2250 if (wq_entry && 2251 ((writeBuffer.isFull() && writeBuffer.numInService() == 0) || |
2225 !miss_mshr)) { 2226 // need to search MSHR queue for conflicting earlier miss. 2227 MSHR *conflict_mshr = | 2252 !miss_mshr)) { 2253 // need to search MSHR queue for conflicting earlier miss. 2254 MSHR *conflict_mshr = |
2228 mshrQueue.findPending(write_mshr->blkAddr, 2229 write_mshr->isSecure); | 2255 mshrQueue.findPending(wq_entry->blkAddr, 2256 wq_entry->isSecure); |
2230 | 2257 |
2231 if (conflict_mshr && conflict_mshr->order < write_mshr->order) { | 2258 if (conflict_mshr && conflict_mshr->order < wq_entry->order) { |
2232 // Service misses in order until conflict is cleared. 2233 return conflict_mshr; 2234 2235 // @todo Note that we ignore the ready time of the conflict here 2236 } 2237 2238 // No conflicts; issue write | 2259 // Service misses in order until conflict is cleared. 2260 return conflict_mshr; 2261 2262 // @todo Note that we ignore the ready time of the conflict here 2263 } 2264 2265 // No conflicts; issue write |
2239 return write_mshr; | 2266 return wq_entry; |
2240 } else if (miss_mshr) { 2241 // need to check for conflicting earlier writeback | 2267 } else if (miss_mshr) { 2268 // need to check for conflicting earlier writeback |
2242 MSHR *conflict_mshr = | 2269 WriteQueueEntry *conflict_mshr = |
2243 writeBuffer.findPending(miss_mshr->blkAddr, 2244 miss_mshr->isSecure); 2245 if (conflict_mshr) { 2246 // not sure why we don't check order here... it was in the 2247 // original code but commented out. 2248 2249 // The only way this happens is if we are 2250 // doing a write and we didn't have permissions 2251 // then subsequently saw a writeback (owned got evicted) 2252 // We need to make sure to perform the writeback first 2253 // To preserve the dirty data, then we can issue the write 2254 | 2270 writeBuffer.findPending(miss_mshr->blkAddr, 2271 miss_mshr->isSecure); 2272 if (conflict_mshr) { 2273 // not sure why we don't check order here... it was in the 2274 // original code but commented out. 2275 2276 // The only way this happens is if we are 2277 // doing a write and we didn't have permissions 2278 // then subsequently saw a writeback (owned got evicted) 2279 // We need to make sure to perform the writeback first 2280 // To preserve the dirty data, then we can issue the write 2281 |
2255 // should we return write_mshr here instead? I.e. do we | 2282 // should we return wq_entry here instead? I.e. do we |
2256 // have to flush writes in order? I don't think so... not 2257 // for Alpha anyway. Maybe for x86? 2258 return conflict_mshr; 2259 2260 // @todo Note that we ignore the ready time of the conflict here 2261 } 2262 2263 // No conflicts; issue read 2264 return miss_mshr; 2265 } 2266 2267 // fall through... no pending requests. Try a prefetch. | 2283 // have to flush writes in order? I don't think so... not 2284 // for Alpha anyway. Maybe for x86? 2285 return conflict_mshr; 2286 2287 // @todo Note that we ignore the ready time of the conflict here 2288 } 2289 2290 // No conflicts; issue read 2291 return miss_mshr; 2292 } 2293 2294 // fall through... no pending requests. Try a prefetch. |
2268 assert(!miss_mshr && !write_mshr); | 2295 assert(!miss_mshr && !wq_entry); |
2269 if (prefetcher && mshrQueue.canPrefetch()) { 2270 // If we have a miss queue slot, we can try a prefetch 2271 PacketPtr pkt = prefetcher->getPacket(); 2272 if (pkt) { 2273 Addr pf_addr = blockAlign(pkt->getAddr()); 2274 if (!tags->findBlock(pf_addr, pkt->isSecure()) && 2275 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 2276 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { --- 9 unchanged lines hidden (view full) --- 2286 } else { 2287 // free the request and packet 2288 delete pkt->req; 2289 delete pkt; 2290 } 2291 } 2292 } 2293 | 2296 if (prefetcher && mshrQueue.canPrefetch()) { 2297 // If we have a miss queue slot, we can try a prefetch 2298 PacketPtr pkt = prefetcher->getPacket(); 2299 if (pkt) { 2300 Addr pf_addr = blockAlign(pkt->getAddr()); 2301 if (!tags->findBlock(pf_addr, pkt->isSecure()) && 2302 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 2303 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { --- 9 unchanged lines hidden (view full) --- 2313 } else { 2314 // free the request and packet 2315 delete pkt->req; 2316 delete pkt; 2317 } 2318 } 2319 } 2320 |
2294 return NULL; | 2321 return nullptr; |
2295} 2296 2297bool 2298Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const 2299{ 2300 if (!forwardSnoops) 2301 return false; 2302 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and --- 14 unchanged lines hidden (view full) --- 2317 assert(!(snoop_pkt.cacheResponding())); 2318 return snoop_pkt.isBlockCached(); 2319 } else { 2320 cpuSidePort->sendAtomicSnoop(pkt); 2321 return pkt->isBlockCached(); 2322 } 2323} 2324 | 2322} 2323 2324bool 2325Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const 2326{ 2327 if (!forwardSnoops) 2328 return false; 2329 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and --- 14 unchanged lines hidden (view full) --- 2344 assert(!(snoop_pkt.cacheResponding())); 2345 return snoop_pkt.isBlockCached(); 2346 } else { 2347 cpuSidePort->sendAtomicSnoop(pkt); 2348 return pkt->isBlockCached(); 2349 } 2350} 2351 |
2325PacketPtr 2326Cache::getTimingPacket() | 2352Tick 2353Cache::nextQueueReadyTime() const |
2327{ | 2354{ |
2328 MSHR *mshr = getNextMSHR(); | 2355 Tick nextReady = std::min(mshrQueue.nextReadyTime(), 2356 writeBuffer.nextReadyTime()); |
2329 | 2357 |
2330 if (mshr == NULL) { 2331 return NULL; | 2358 // Don't signal prefetch ready time if no MSHRs available 2359 // Will signal once enoguh MSHRs are deallocated 2360 if (prefetcher && mshrQueue.canPrefetch()) { 2361 nextReady = std::min(nextReady, 2362 prefetcher->nextPrefetchReadyTime()); |
2332 } 2333 | 2363 } 2364 |
2365 return nextReady; 2366} 2367 2368bool 2369Cache::sendMSHRQueuePacket(MSHR* mshr) 2370{ 2371 assert(mshr); 2372 |
|
2334 // use request from 1st target 2335 PacketPtr tgt_pkt = mshr->getTarget()->pkt; | 2373 // use request from 1st target 2374 PacketPtr tgt_pkt = mshr->getTarget()->pkt; |
2336 PacketPtr pkt = NULL; | |
2337 | 2375 |
2338 DPRINTF(CachePort, "%s %s for addr %#llx size %d\n", __func__, 2339 tgt_pkt->cmdString(), tgt_pkt->getAddr(), tgt_pkt->getSize()); | 2376 DPRINTF(Cache, "%s MSHR %s for addr %#llx size %d\n", __func__, 2377 tgt_pkt->cmdString(), tgt_pkt->getAddr(), 2378 tgt_pkt->getSize()); |
2340 2341 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 2342 2343 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) { | 2379 2380 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 2381 2382 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) { |
2383 // we should never have hardware prefetches to allocated 2384 // blocks 2385 assert(blk == NULL); 2386 |
|
2344 // We need to check the caches above us to verify that 2345 // they don't have a copy of this block in the dirty state 2346 // at the moment. Without this check we could get a stale 2347 // copy from memory that might get used in place of the 2348 // dirty one. 2349 Packet snoop_pkt(tgt_pkt, true, false); 2350 snoop_pkt.setExpressSnoop(); 2351 // We are sending this packet upwards, but if it hits we will --- 22 unchanged lines hidden (view full) --- 2374 // if we are getting a snoop response with no sharers it 2375 // will be allocated as Modified 2376 bool pending_modified_resp = !snoop_pkt.hasSharers(); 2377 markInService(mshr, pending_modified_resp); 2378 2379 DPRINTF(Cache, "Upward snoop of prefetch for addr" 2380 " %#x (%s) hit\n", 2381 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns"); | 2387 // We need to check the caches above us to verify that 2388 // they don't have a copy of this block in the dirty state 2389 // at the moment. Without this check we could get a stale 2390 // copy from memory that might get used in place of the 2391 // dirty one. 2392 Packet snoop_pkt(tgt_pkt, true, false); 2393 snoop_pkt.setExpressSnoop(); 2394 // We are sending this packet upwards, but if it hits we will --- 22 unchanged lines hidden (view full) --- 2417 // if we are getting a snoop response with no sharers it 2418 // will be allocated as Modified 2419 bool pending_modified_resp = !snoop_pkt.hasSharers(); 2420 markInService(mshr, pending_modified_resp); 2421 2422 DPRINTF(Cache, "Upward snoop of prefetch for addr" 2423 " %#x (%s) hit\n", 2424 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns"); |
2382 return NULL; | 2425 return false; |
2383 } 2384 | 2426 } 2427 |
2385 if (snoop_pkt.isBlockCached() || blk != NULL) { | 2428 if (snoop_pkt.isBlockCached()) { |
2386 DPRINTF(Cache, "Block present, prefetch squashed by cache. " 2387 "Deallocating mshr target %#x.\n", 2388 mshr->blkAddr); | 2429 DPRINTF(Cache, "Block present, prefetch squashed by cache. " 2430 "Deallocating mshr target %#x.\n", 2431 mshr->blkAddr); |
2432 |
|
2389 // Deallocate the mshr target | 2433 // Deallocate the mshr target |
2390 if (mshr->queue->forceDeallocateTarget(mshr)) { | 2434 if (mshrQueue.forceDeallocateTarget(mshr)) { |
2391 // Clear block if this deallocation resulted freed an 2392 // mshr when all had previously been utilized | 2435 // Clear block if this deallocation resulted freed an 2436 // mshr when all had previously been utilized |
2393 clearBlocked((BlockedCause)(mshr->queue->index)); | 2437 clearBlocked(Blocked_NoMSHRs); |
2394 } | 2438 } |
2395 return NULL; | 2439 return false; |
2396 } 2397 } 2398 | 2440 } 2441 } 2442 |
2399 if (mshr->isForwardNoResponse()) { 2400 // no response expected, just forward packet as it is 2401 assert(tags->findBlock(mshr->blkAddr, mshr->isSecure) == NULL); 2402 pkt = tgt_pkt; 2403 } else { 2404 pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable()); | 2443 // either a prefetch that is not present upstream, or a normal 2444 // MSHR request, proceed to get the packet to send downstream 2445 PacketPtr pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable()); |
2405 | 2446 |
2406 mshr->isForward = (pkt == NULL); | 2447 mshr->isForward = (pkt == NULL); |
2407 | 2448 |
2408 if (mshr->isForward) { 2409 // not a cache block request, but a response is expected 2410 // make copy of current packet to forward, keep current 2411 // copy for response handling 2412 pkt = new Packet(tgt_pkt, false, true); 2413 if (pkt->isWrite()) { 2414 pkt->setData(tgt_pkt->getConstPtr<uint8_t>()); 2415 } 2416 } | 2449 if (mshr->isForward) { 2450 // not a cache block request, but a response is expected 2451 // make copy of current packet to forward, keep current 2452 // copy for response handling 2453 pkt = new Packet(tgt_pkt, false, true); 2454 assert(!pkt->isWrite()); |
2417 } 2418 | 2455 } 2456 |
2419 assert(pkt != NULL); 2420 // play it safe and append (rather than set) the sender state, as 2421 // forwarded packets may already have existing state | 2457 // play it safe and append (rather than set) the sender state, 2458 // as forwarded packets may already have existing state |
2422 pkt->pushSenderState(mshr); | 2459 pkt->pushSenderState(mshr); |
2423 return pkt; 2424} | |
2425 | 2460 |
2461 if (!memSidePort->sendTimingReq(pkt)) { 2462 // we are awaiting a retry, but we 2463 // delete the packet and will be creating a new packet 2464 // when we get the opportunity 2465 delete pkt; |
|
2426 | 2466 |
2427Tick 2428Cache::nextMSHRReadyTime() const | 2467 // note that we have now masked any requestBus and 2468 // schedSendEvent (we will wait for a retry before 2469 // doing anything), and this is so even if we do not 2470 // care about this packet and might override it before 2471 // it gets retried 2472 return true; 2473 } else { 2474 // As part of the call to sendTimingReq the packet is 2475 // forwarded to all neighbouring caches (and any caches 2476 // above them) as a snoop. Thus at this point we know if 2477 // any of the neighbouring caches are responding, and if 2478 // so, we know it is dirty, and we can determine if it is 2479 // being passed as Modified, making our MSHR the ordering 2480 // point 2481 bool pending_modified_resp = !pkt->hasSharers() && 2482 pkt->cacheResponding(); 2483 markInService(mshr, pending_modified_resp); 2484 return false; 2485 } 2486} 2487 2488bool 2489Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) |
2429{ | 2490{ |
2430 Tick nextReady = std::min(mshrQueue.nextMSHRReadyTime(), 2431 writeBuffer.nextMSHRReadyTime()); | 2491 assert(wq_entry); |
2432 | 2492 |
2433 // Don't signal prefetch ready time if no MSHRs available 2434 // Will signal once enoguh MSHRs are deallocated 2435 if (prefetcher && mshrQueue.canPrefetch()) { 2436 nextReady = std::min(nextReady, 2437 prefetcher->nextPrefetchReadyTime()); | 2493 // always a single target for write queue entries 2494 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 2495 2496 DPRINTF(Cache, "%s write %s for addr %#llx size %d\n", __func__, 2497 tgt_pkt->cmdString(), tgt_pkt->getAddr(), 2498 tgt_pkt->getSize()); 2499 2500 PacketPtr pkt = nullptr; 2501 bool delete_pkt = false; 2502 2503 if (tgt_pkt->isEviction()) { 2504 assert(!wq_entry->isUncacheable()); 2505 // no response expected, just forward packet as it is 2506 pkt = tgt_pkt; 2507 } else { 2508 // the only thing we deal with besides eviction commands 2509 // are uncacheable writes 2510 assert(tgt_pkt->req->isUncacheable() && tgt_pkt->isWrite()); 2511 // not a cache block request, but a response is expected 2512 // make copy of current packet to forward, keep current 2513 // copy for response handling 2514 pkt = new Packet(tgt_pkt, false, true); 2515 pkt->setData(tgt_pkt->getConstPtr<uint8_t>()); 2516 delete_pkt = true; |
2438 } 2439 | 2517 } 2518 |
2440 return nextReady; | 2519 pkt->pushSenderState(wq_entry); 2520 2521 if (!memSidePort->sendTimingReq(pkt)) { 2522 if (delete_pkt) { 2523 // we are awaiting a retry, but we 2524 // delete the packet and will be creating a new packet 2525 // when we get the opportunity 2526 delete pkt; 2527 } 2528 // note that we have now masked any requestBus and 2529 // schedSendEvent (we will wait for a retry before 2530 // doing anything), and this is so even if we do not 2531 // care about this packet and might override it before 2532 // it gets retried 2533 return true; 2534 } else { 2535 markInService(wq_entry); 2536 return false; 2537 } |
2441} 2442 2443void 2444Cache::serialize(CheckpointOut &cp) const 2445{ 2446 bool dirty(isDirty()); 2447 2448 if (dirty) { --- 132 unchanged lines hidden (view full) --- 2581 assert(!waitingOnRetry); 2582 2583 // there should never be any deferred request packets in the 2584 // queue, instead we resly on the cache to provide the packets 2585 // from the MSHR queue or write queue 2586 assert(deferredPacketReadyTime() == MaxTick); 2587 2588 // check for request packets (requests & writebacks) | 2538} 2539 2540void 2541Cache::serialize(CheckpointOut &cp) const 2542{ 2543 bool dirty(isDirty()); 2544 2545 if (dirty) { --- 132 unchanged lines hidden (view full) --- 2678 assert(!waitingOnRetry); 2679 2680 // there should never be any deferred request packets in the 2681 // queue, instead we resly on the cache to provide the packets 2682 // from the MSHR queue or write queue 2683 assert(deferredPacketReadyTime() == MaxTick); 2684 2685 // check for request packets (requests & writebacks) |
2589 PacketPtr pkt = cache.getTimingPacket(); 2590 if (pkt == NULL) { | 2686 QueueEntry* entry = cache.getNextQueueEntry(); 2687 2688 if (!entry) { |
2591 // can happen if e.g. we attempt a writeback and fail, but 2592 // before the retry, the writeback is eliminated because 2593 // we snoop another cache's ReadEx. 2594 } else { | 2689 // can happen if e.g. we attempt a writeback and fail, but 2690 // before the retry, the writeback is eliminated because 2691 // we snoop another cache's ReadEx. 2692 } else { |
2595 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState); 2596 // in most cases getTimingPacket allocates a new packet, and 2597 // we must delete it unless it is successfully sent 2598 bool delete_pkt = !mshr->isForwardNoResponse(); 2599 | |
2600 // let our snoop responses go first if there are responses to | 2693 // let our snoop responses go first if there are responses to |
2601 // the same addresses we are about to writeback, note that 2602 // this creates a dependency between requests and snoop 2603 // responses, but that should not be a problem since there is 2604 // a chain already and the key is that the snoop responses can 2605 // sink unconditionally 2606 if (snoopRespQueue.hasAddr(pkt->getAddr())) { 2607 DPRINTF(CachePort, "Waiting for snoop response to be sent\n"); 2608 Tick when = snoopRespQueue.deferredPacketReadyTime(); 2609 schedSendEvent(when); 2610 2611 if (delete_pkt) 2612 delete pkt; 2613 | 2694 // the same addresses 2695 if (checkConflictingSnoop(entry->blkAddr)) { |
2614 return; 2615 } | 2696 return; 2697 } |
2616 2617 2618 waitingOnRetry = !masterPort.sendTimingReq(pkt); 2619 2620 if (waitingOnRetry) { 2621 DPRINTF(CachePort, "now waiting on a retry\n"); 2622 if (delete_pkt) { 2623 // we are awaiting a retry, but we 2624 // delete the packet and will be creating a new packet 2625 // when we get the opportunity 2626 delete pkt; 2627 } 2628 // note that we have now masked any requestBus and 2629 // schedSendEvent (we will wait for a retry before 2630 // doing anything), and this is so even if we do not 2631 // care about this packet and might override it before 2632 // it gets retried 2633 } else { 2634 // As part of the call to sendTimingReq the packet is 2635 // forwarded to all neighbouring caches (and any caches 2636 // above them) as a snoop. Thus at this point we know if 2637 // any of the neighbouring caches are responding, and if 2638 // so, we know it is dirty, and we can determine if it is 2639 // being passed as Modified, making our MSHR the ordering 2640 // point 2641 bool pending_modified_resp = !pkt->hasSharers() && 2642 pkt->cacheResponding(); 2643 2644 cache.markInService(mshr, pending_modified_resp); 2645 } | 2698 waitingOnRetry = entry->sendPacket(cache); |
2646 } 2647 2648 // if we succeeded and are not waiting for a retry, schedule the | 2699 } 2700 2701 // if we succeeded and are not waiting for a retry, schedule the |
2649 // next send considering when the next MSHR is ready, note that | 2702 // next send considering when the next queue is ready, note that |
2650 // snoop responses have their own packet queue and thus schedule 2651 // their own events 2652 if (!waitingOnRetry) { | 2703 // snoop responses have their own packet queue and thus schedule 2704 // their own events 2705 if (!waitingOnRetry) { |
2653 schedSendEvent(cache.nextMSHRReadyTime()); | 2706 schedSendEvent(cache.nextQueueReadyTime()); |
2654 } 2655} 2656 2657Cache:: 2658MemSidePort::MemSidePort(const std::string &_name, Cache *_cache, 2659 const std::string &_label) 2660 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2661 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2662 _snoopRespQueue(*_cache, *this, _label), cache(_cache) 2663{ 2664} | 2707 } 2708} 2709 2710Cache:: 2711MemSidePort::MemSidePort(const std::string &_name, Cache *_cache, 2712 const std::string &_label) 2713 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2714 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2715 _snoopRespQueue(*_cache, *this, _label), cache(_cache) 2716{ 2717} |