1/* |
2 * Copyright (c) 2010-2016 ARM Limited |
3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated --- 270 unchanged lines hidden (view full) --- 281 // for invalidations we could be looking at the temp block 282 // (for upgrades we always allocate) 283 invalidateBlock(blk); 284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n", 285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 286 } 287} 288 |
289///////////////////////////////////////////////////// 290// |
291// Access path: requests coming in from the CPU side 292// 293///////////////////////////////////////////////////// 294 295bool 296Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat, 297 PacketList &writebacks) 298{ --- 45 unchanged lines hidden (view full) --- 344 // We check for presence of block in above caches before issuing 345 // Writeback or CleanEvict to write buffer. Therefore the only 346 // possible cases can be of a CleanEvict packet coming from above 347 // encountering a Writeback generated in this cache peer cache and 348 // waiting in the write buffer. Cases of upper level peer caches 349 // generating CleanEvict and Writeback or simply CleanEvict and 350 // CleanEvict almost simultaneously will be caught by snoops sent out 351 // by crossbar. |
352 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(), 353 pkt->isSecure()); 354 if (wb_entry) { |
355 assert(wb_entry->getNumTargets() == 1); 356 PacketPtr wbPkt = wb_entry->getTarget()->pkt; 357 assert(wbPkt->isWriteback()); 358 359 if (pkt->isCleanEviction()) { 360 // The CleanEvict and WritebackClean snoops into other 361 // peer caches of the same level while traversing the 362 // crossbar. If a copy of the block is found, the --- 4 unchanged lines hidden (view full) --- 367 // discard the CleanEvict by returning true. 368 wbPkt->clearBlockCached(); 369 return true; 370 } else { 371 assert(pkt->cmd == MemCmd::WritebackDirty); 372 // Dirty writeback from above trumps our clean 373 // writeback... discard here 374 // Note: markInService will remove entry from writeback buffer. |
375 markInService(wb_entry); |
376 delete wbPkt; 377 } 378 } 379 } 380 381 // Writeback handling is special case. We can write the block into 382 // the cache without having a writeable copy (or any copy at all). 383 if (pkt->isWriteback()) { --- 834 unchanged lines hidden (view full) --- 1218///////////////////////////////////////////////////// 1219// 1220// Response handling: responses from the memory side 1221// 1222///////////////////////////////////////////////////// 1223 1224 1225void |
1226Cache::handleUncacheableWriteResp(PacketPtr pkt) 1227{ 1228 WriteQueueEntry *wq_entry = 1229 dynamic_cast<WriteQueueEntry*>(pkt->senderState); 1230 assert(wq_entry); 1231 1232 WriteQueueEntry::Target *target = wq_entry->getTarget(); 1233 Packet *tgt_pkt = target->pkt; 1234 1235 // we send out invalidation reqs and get invalidation 1236 // responses for write-line requests 1237 assert(tgt_pkt->cmd != MemCmd::WriteLineReq); 1238 1239 int stats_cmd_idx = tgt_pkt->cmdToIndex(); 1240 Tick miss_latency = curTick() - target->recvTime; 1241 assert(pkt->req->masterId() < system->maxMasters()); 1242 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1243 miss_latency; 1244 1245 tgt_pkt->makeTimingResponse(); 1246 // if this packet is an error copy that to the new packet 1247 if (pkt->isError()) 1248 tgt_pkt->copyError(pkt); 1249 // Reset the bus additional time as it is now accounted for 1250 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0; 1251 Tick completion_time = clockEdge(responseLatency) + 1252 pkt->headerDelay + pkt->payloadDelay; 1253 1254 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true); 1255 1256 wq_entry->popTarget(); 1257 assert(!wq_entry->hasTargets()); 1258 1259 bool wasFull = writeBuffer.isFull(); 1260 writeBuffer.deallocate(wq_entry); 1261 1262 if (wasFull && !writeBuffer.isFull()) { 1263 clearBlocked(Blocked_NoWBBuffers); 1264 } 1265 1266 delete pkt; 1267} 1268 1269void |
1270Cache::recvTimingResp(PacketPtr pkt) 1271{ 1272 assert(pkt->isResponse()); 1273 1274 // all header delay should be paid for by the crossbar, unless 1275 // this is a prefetch response from above 1276 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 1277 "%s saw a non-zero packet delay\n", name()); 1278 |
1279 bool is_error = pkt->isError(); 1280 |
1281 if (is_error) { 1282 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), " 1283 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns", 1284 pkt->cmdString()); 1285 } 1286 1287 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n", 1288 pkt->cmdString(), pkt->getAddr(), pkt->getSize(), 1289 pkt->isSecure() ? "s" : "ns"); 1290 |
1291 // if this is a write, we should be looking at an uncacheable 1292 // write 1293 if (pkt->isWrite()) { 1294 assert(pkt->req->isUncacheable()); 1295 handleUncacheableWriteResp(pkt); 1296 return; 1297 } |
1298 |
1299 // we have dealt with any (uncacheable) writes above, from here on 1300 // we know we are dealing with an MSHR due to a miss or a prefetch 1301 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState); 1302 assert(mshr); 1303 |
1304 if (mshr == noTargetMSHR) { 1305 // we always clear at least one target 1306 clearBlocked(Blocked_NoTargets); 1307 noTargetMSHR = NULL; 1308 } 1309 1310 // Initial target is used just for stats 1311 MSHR::Target *initial_tgt = mshr->getTarget(); 1312 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex(); 1313 Tick miss_latency = curTick() - initial_tgt->recvTime; |
1314 1315 if (pkt->req->isUncacheable()) { 1316 assert(pkt->req->masterId() < system->maxMasters()); 1317 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1318 miss_latency; 1319 } else { 1320 assert(pkt->req->masterId() < system->maxMasters()); 1321 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 1322 miss_latency; 1323 } 1324 |
1325 bool wasFull = mshrQueue.isFull(); 1326 1327 PacketList writebacks; 1328 1329 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 1330 |
1331 // upgrade deferred targets if the response has no sharers, and is 1332 // thus passing writable 1333 if (!pkt->hasSharers()) { 1334 mshr->promoteWritable(); 1335 } 1336 1337 bool is_fill = !mshr->isForward && 1338 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp); --- 159 unchanged lines hidden (view full) --- 1498 } 1499 1500 if (mshr->promoteDeferredTargets()) { 1501 // avoid later read getting stale data while write miss is 1502 // outstanding.. see comment in timingAccess() 1503 if (blk) { 1504 blk->status &= ~BlkReadable; 1505 } |
1506 mshrQueue.markPending(mshr); |
1507 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay); 1508 } else { |
1509 mshrQueue.deallocate(mshr); 1510 if (wasFull && !mshrQueue.isFull()) { 1511 clearBlocked(Blocked_NoMSHRs); |
1512 } 1513 1514 // Request the bus for a prefetch if this deallocation freed enough 1515 // MSHRs for a prefetch to take place |
1516 if (prefetcher && mshrQueue.canPrefetch()) { |
1517 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(), 1518 clockEdge()); 1519 if (next_pf_time != MaxTick) 1520 schedMemSideSendEvent(next_pf_time); 1521 } 1522 } 1523 // reset the xbar additional timinig as it is now accounted for 1524 pkt->headerDelay = pkt->payloadDelay = 0; --- 217 unchanged lines hidden (view full) --- 1742{ 1743 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq); 1744 Addr addr = pkt->getAddr(); 1745 bool is_secure = pkt->isSecure(); 1746#if TRACING_ON 1747 CacheBlk::State old_state = blk ? blk->status : 0; 1748#endif 1749 |
1750 // When handling a fill, we should have no writes to this line. 1751 assert(addr == blockAlign(addr)); 1752 assert(!writeBuffer.findMatch(addr, is_secure)); |
1753 1754 if (blk == NULL) { 1755 // better have read new data... 1756 assert(pkt->hasData()); 1757 1758 // only read responses and write-line requests have data; 1759 // note that we don't write the data here for write-line - that 1760 // happens in the subsequent satisfyCpuSideRequest. --- 371 unchanged lines hidden (view full) --- 2132 mshr->print()); 2133 2134 if (mshr->getNumTargets() > numTarget) 2135 warn("allocating bonus target for snoop"); //handle later 2136 return; 2137 } 2138 2139 //We also need to check the writeback buffers and handle those |
2140 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure); 2141 if (wb_entry) { |
2142 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n", 2143 pkt->getAddr(), is_secure ? "s" : "ns"); |
2144 // Expect to see only Writebacks and/or CleanEvicts here, both of 2145 // which should not be generated for uncacheable data. 2146 assert(!wb_entry->isUncacheable()); 2147 // There should only be a single request responsible for generating 2148 // Writebacks/CleanEvicts. 2149 assert(wb_entry->getNumTargets() == 1); 2150 PacketPtr wb_pkt = wb_entry->getTarget()->pkt; 2151 assert(wb_pkt->isEviction()); --- 34 unchanged lines hidden (view full) --- 2186 2187 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(), 2188 false, false); 2189 } 2190 2191 if (invalidate) { 2192 // Invalidation trumps our writeback... discard here 2193 // Note: markInService will remove entry from writeback buffer. |
2194 markInService(wb_entry); |
2195 delete wb_pkt; 2196 } 2197 } 2198 2199 // If this was a shared writeback, there may still be 2200 // other shared copies above that require invalidation. 2201 // We could be more selective and return here if the 2202 // request is non-exclusive or if the writeback is --- 26 unchanged lines hidden (view full) --- 2229 } 2230 2231 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 2232 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false); 2233 return snoop_delay + lookupLatency * clockPeriod(); 2234} 2235 2236 |
2237QueueEntry* 2238Cache::getNextQueueEntry() |
2239{ 2240 // Check both MSHR queue and write buffer for potential requests, 2241 // note that null does not mean there is no request, it could 2242 // simply be that it is not ready |
2243 MSHR *miss_mshr = mshrQueue.getNext(); 2244 WriteQueueEntry *wq_entry = writeBuffer.getNext(); |
2245 2246 // If we got a write buffer request ready, first priority is a |
2247 // full write buffer (but only if we have no uncacheable write 2248 // responses outstanding, possibly revisit this last part), 2249 // otherwhise we favour the miss requests 2250 if (wq_entry && 2251 ((writeBuffer.isFull() && writeBuffer.numInService() == 0) || |
2252 !miss_mshr)) { 2253 // need to search MSHR queue for conflicting earlier miss. 2254 MSHR *conflict_mshr = |
2255 mshrQueue.findPending(wq_entry->blkAddr, 2256 wq_entry->isSecure); |
2257 |
2258 if (conflict_mshr && conflict_mshr->order < wq_entry->order) { |
2259 // Service misses in order until conflict is cleared. 2260 return conflict_mshr; 2261 2262 // @todo Note that we ignore the ready time of the conflict here 2263 } 2264 2265 // No conflicts; issue write |
2266 return wq_entry; |
2267 } else if (miss_mshr) { 2268 // need to check for conflicting earlier writeback |
2269 WriteQueueEntry *conflict_mshr = |
2270 writeBuffer.findPending(miss_mshr->blkAddr, 2271 miss_mshr->isSecure); 2272 if (conflict_mshr) { 2273 // not sure why we don't check order here... it was in the 2274 // original code but commented out. 2275 2276 // The only way this happens is if we are 2277 // doing a write and we didn't have permissions 2278 // then subsequently saw a writeback (owned got evicted) 2279 // We need to make sure to perform the writeback first 2280 // To preserve the dirty data, then we can issue the write 2281 |
2282 // should we return wq_entry here instead? I.e. do we |
2283 // have to flush writes in order? I don't think so... not 2284 // for Alpha anyway. Maybe for x86? 2285 return conflict_mshr; 2286 2287 // @todo Note that we ignore the ready time of the conflict here 2288 } 2289 2290 // No conflicts; issue read 2291 return miss_mshr; 2292 } 2293 2294 // fall through... no pending requests. Try a prefetch. |
2295 assert(!miss_mshr && !wq_entry); |
2296 if (prefetcher && mshrQueue.canPrefetch()) { 2297 // If we have a miss queue slot, we can try a prefetch 2298 PacketPtr pkt = prefetcher->getPacket(); 2299 if (pkt) { 2300 Addr pf_addr = blockAlign(pkt->getAddr()); 2301 if (!tags->findBlock(pf_addr, pkt->isSecure()) && 2302 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) && 2303 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) { --- 9 unchanged lines hidden (view full) --- 2313 } else { 2314 // free the request and packet 2315 delete pkt->req; 2316 delete pkt; 2317 } 2318 } 2319 } 2320 |
2321 return nullptr; |
2322} 2323 2324bool 2325Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const 2326{ 2327 if (!forwardSnoops) 2328 return false; 2329 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and --- 14 unchanged lines hidden (view full) --- 2344 assert(!(snoop_pkt.cacheResponding())); 2345 return snoop_pkt.isBlockCached(); 2346 } else { 2347 cpuSidePort->sendAtomicSnoop(pkt); 2348 return pkt->isBlockCached(); 2349 } 2350} 2351 |
2352Tick 2353Cache::nextQueueReadyTime() const |
2354{ |
2355 Tick nextReady = std::min(mshrQueue.nextReadyTime(), 2356 writeBuffer.nextReadyTime()); |
2357 |
2358 // Don't signal prefetch ready time if no MSHRs available 2359 // Will signal once enoguh MSHRs are deallocated 2360 if (prefetcher && mshrQueue.canPrefetch()) { 2361 nextReady = std::min(nextReady, 2362 prefetcher->nextPrefetchReadyTime()); |
2363 } 2364 |
2365 return nextReady; 2366} 2367 2368bool 2369Cache::sendMSHRQueuePacket(MSHR* mshr) 2370{ 2371 assert(mshr); 2372 |
2373 // use request from 1st target 2374 PacketPtr tgt_pkt = mshr->getTarget()->pkt; |
2375 |
2376 DPRINTF(Cache, "%s MSHR %s for addr %#llx size %d\n", __func__, 2377 tgt_pkt->cmdString(), tgt_pkt->getAddr(), 2378 tgt_pkt->getSize()); |
2379 2380 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 2381 2382 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) { |
2383 // we should never have hardware prefetches to allocated 2384 // blocks 2385 assert(blk == NULL); 2386 |
2387 // We need to check the caches above us to verify that 2388 // they don't have a copy of this block in the dirty state 2389 // at the moment. Without this check we could get a stale 2390 // copy from memory that might get used in place of the 2391 // dirty one. 2392 Packet snoop_pkt(tgt_pkt, true, false); 2393 snoop_pkt.setExpressSnoop(); 2394 // We are sending this packet upwards, but if it hits we will --- 22 unchanged lines hidden (view full) --- 2417 // if we are getting a snoop response with no sharers it 2418 // will be allocated as Modified 2419 bool pending_modified_resp = !snoop_pkt.hasSharers(); 2420 markInService(mshr, pending_modified_resp); 2421 2422 DPRINTF(Cache, "Upward snoop of prefetch for addr" 2423 " %#x (%s) hit\n", 2424 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns"); |
2425 return false; |
2426 } 2427 |
2428 if (snoop_pkt.isBlockCached()) { |
2429 DPRINTF(Cache, "Block present, prefetch squashed by cache. " 2430 "Deallocating mshr target %#x.\n", 2431 mshr->blkAddr); |
2432 |
2433 // Deallocate the mshr target |
2434 if (mshrQueue.forceDeallocateTarget(mshr)) { |
2435 // Clear block if this deallocation resulted freed an 2436 // mshr when all had previously been utilized |
2437 clearBlocked(Blocked_NoMSHRs); |
2438 } |
2439 return false; |
2440 } 2441 } 2442 |
2443 // either a prefetch that is not present upstream, or a normal 2444 // MSHR request, proceed to get the packet to send downstream 2445 PacketPtr pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable()); |
2446 |
2447 mshr->isForward = (pkt == NULL); |
2448 |
2449 if (mshr->isForward) { 2450 // not a cache block request, but a response is expected 2451 // make copy of current packet to forward, keep current 2452 // copy for response handling 2453 pkt = new Packet(tgt_pkt, false, true); 2454 assert(!pkt->isWrite()); |
2455 } 2456 |
2457 // play it safe and append (rather than set) the sender state, 2458 // as forwarded packets may already have existing state |
2459 pkt->pushSenderState(mshr); |
2460 |
2461 if (!memSidePort->sendTimingReq(pkt)) { 2462 // we are awaiting a retry, but we 2463 // delete the packet and will be creating a new packet 2464 // when we get the opportunity 2465 delete pkt; |
2466 |
2467 // note that we have now masked any requestBus and 2468 // schedSendEvent (we will wait for a retry before 2469 // doing anything), and this is so even if we do not 2470 // care about this packet and might override it before 2471 // it gets retried 2472 return true; 2473 } else { 2474 // As part of the call to sendTimingReq the packet is 2475 // forwarded to all neighbouring caches (and any caches 2476 // above them) as a snoop. Thus at this point we know if 2477 // any of the neighbouring caches are responding, and if 2478 // so, we know it is dirty, and we can determine if it is 2479 // being passed as Modified, making our MSHR the ordering 2480 // point 2481 bool pending_modified_resp = !pkt->hasSharers() && 2482 pkt->cacheResponding(); 2483 markInService(mshr, pending_modified_resp); 2484 return false; 2485 } 2486} 2487 2488bool 2489Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) |
2490{ |
2491 assert(wq_entry); |
2492 |
2493 // always a single target for write queue entries 2494 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 2495 2496 DPRINTF(Cache, "%s write %s for addr %#llx size %d\n", __func__, 2497 tgt_pkt->cmdString(), tgt_pkt->getAddr(), 2498 tgt_pkt->getSize()); 2499 2500 PacketPtr pkt = nullptr; 2501 bool delete_pkt = false; 2502 2503 if (tgt_pkt->isEviction()) { 2504 assert(!wq_entry->isUncacheable()); 2505 // no response expected, just forward packet as it is 2506 pkt = tgt_pkt; 2507 } else { 2508 // the only thing we deal with besides eviction commands 2509 // are uncacheable writes 2510 assert(tgt_pkt->req->isUncacheable() && tgt_pkt->isWrite()); 2511 // not a cache block request, but a response is expected 2512 // make copy of current packet to forward, keep current 2513 // copy for response handling 2514 pkt = new Packet(tgt_pkt, false, true); 2515 pkt->setData(tgt_pkt->getConstPtr<uint8_t>()); 2516 delete_pkt = true; |
2517 } 2518 |
2519 pkt->pushSenderState(wq_entry); 2520 2521 if (!memSidePort->sendTimingReq(pkt)) { 2522 if (delete_pkt) { 2523 // we are awaiting a retry, but we 2524 // delete the packet and will be creating a new packet 2525 // when we get the opportunity 2526 delete pkt; 2527 } 2528 // note that we have now masked any requestBus and 2529 // schedSendEvent (we will wait for a retry before 2530 // doing anything), and this is so even if we do not 2531 // care about this packet and might override it before 2532 // it gets retried 2533 return true; 2534 } else { 2535 markInService(wq_entry); 2536 return false; 2537 } |
2538} 2539 2540void 2541Cache::serialize(CheckpointOut &cp) const 2542{ 2543 bool dirty(isDirty()); 2544 2545 if (dirty) { --- 132 unchanged lines hidden (view full) --- 2678 assert(!waitingOnRetry); 2679 2680 // there should never be any deferred request packets in the 2681 // queue, instead we resly on the cache to provide the packets 2682 // from the MSHR queue or write queue 2683 assert(deferredPacketReadyTime() == MaxTick); 2684 2685 // check for request packets (requests & writebacks) |
2686 QueueEntry* entry = cache.getNextQueueEntry(); 2687 2688 if (!entry) { |
2689 // can happen if e.g. we attempt a writeback and fail, but 2690 // before the retry, the writeback is eliminated because 2691 // we snoop another cache's ReadEx. 2692 } else { |
2693 // let our snoop responses go first if there are responses to |
2694 // the same addresses 2695 if (checkConflictingSnoop(entry->blkAddr)) { |
2696 return; 2697 } |
2698 waitingOnRetry = entry->sendPacket(cache); |
2699 } 2700 2701 // if we succeeded and are not waiting for a retry, schedule the |
2702 // next send considering when the next queue is ready, note that |
2703 // snoop responses have their own packet queue and thus schedule 2704 // their own events 2705 if (!waitingOnRetry) { |
2706 schedSendEvent(cache.nextQueueReadyTime()); |
2707 } 2708} 2709 2710Cache:: 2711MemSidePort::MemSidePort(const std::string &_name, Cache *_cache, 2712 const std::string &_label) 2713 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue), 2714 _reqQueue(*_cache, *this, _snoopRespQueue, _label), 2715 _snoopRespQueue(*_cache, *this, _label), cache(_cache) 2716{ 2717} |