1/*
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated

--- 270 unchanged lines hidden (view full) ---

281 // for invalidations we could be looking at the temp block
282 // (for upgrades we always allocate)
283 invalidateBlock(blk);
284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n",
285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
286 }
287}
288
289/////////////////////////////////////////////////////
290//
291// Access path: requests coming in from the CPU side
292//
293/////////////////////////////////////////////////////
294
295bool
296Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
297 PacketList &writebacks)
298{

--- 45 unchanged lines hidden (view full) ---

344 // We check for presence of block in above caches before issuing
345 // Writeback or CleanEvict to write buffer. Therefore the only
346 // possible cases can be of a CleanEvict packet coming from above
347 // encountering a Writeback generated in this cache peer cache and
348 // waiting in the write buffer. Cases of upper level peer caches
349 // generating CleanEvict and Writeback or simply CleanEvict and
350 // CleanEvict almost simultaneously will be caught by snoops sent out
351 // by crossbar.
352 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
353 pkt->isSecure());
354 if (wb_entry) {
355 assert(wb_entry->getNumTargets() == 1);
356 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
357 assert(wbPkt->isWriteback());
358
359 if (pkt->isCleanEviction()) {
360 // The CleanEvict and WritebackClean snoops into other
361 // peer caches of the same level while traversing the
362 // crossbar. If a copy of the block is found, the

--- 4 unchanged lines hidden (view full) ---

367 // discard the CleanEvict by returning true.
368 wbPkt->clearBlockCached();
369 return true;
370 } else {
371 assert(pkt->cmd == MemCmd::WritebackDirty);
372 // Dirty writeback from above trumps our clean
373 // writeback... discard here
374 // Note: markInService will remove entry from writeback buffer.
375 markInService(wb_entry);
376 delete wbPkt;
377 }
378 }
379 }
380
381 // Writeback handling is special case. We can write the block into
382 // the cache without having a writeable copy (or any copy at all).
383 if (pkt->isWriteback()) {

--- 834 unchanged lines hidden (view full) ---

1218/////////////////////////////////////////////////////
1219//
1220// Response handling: responses from the memory side
1221//
1222/////////////////////////////////////////////////////
1223
1224
1225void
1226Cache::handleUncacheableWriteResp(PacketPtr pkt)
1227{
1228 WriteQueueEntry *wq_entry =
1229 dynamic_cast<WriteQueueEntry*>(pkt->senderState);
1230 assert(wq_entry);
1231
1232 WriteQueueEntry::Target *target = wq_entry->getTarget();
1233 Packet *tgt_pkt = target->pkt;
1234
1235 // we send out invalidation reqs and get invalidation
1236 // responses for write-line requests
1237 assert(tgt_pkt->cmd != MemCmd::WriteLineReq);
1238
1239 int stats_cmd_idx = tgt_pkt->cmdToIndex();
1240 Tick miss_latency = curTick() - target->recvTime;
1241 assert(pkt->req->masterId() < system->maxMasters());
1242 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1243 miss_latency;
1244
1245 tgt_pkt->makeTimingResponse();
1246 // if this packet is an error copy that to the new packet
1247 if (pkt->isError())
1248 tgt_pkt->copyError(pkt);
1249 // Reset the bus additional time as it is now accounted for
1250 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1251 Tick completion_time = clockEdge(responseLatency) +
1252 pkt->headerDelay + pkt->payloadDelay;
1253
1254 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1255
1256 wq_entry->popTarget();
1257 assert(!wq_entry->hasTargets());
1258
1259 bool wasFull = writeBuffer.isFull();
1260 writeBuffer.deallocate(wq_entry);
1261
1262 if (wasFull && !writeBuffer.isFull()) {
1263 clearBlocked(Blocked_NoWBBuffers);
1264 }
1265
1266 delete pkt;
1267}
1268
1269void
1270Cache::recvTimingResp(PacketPtr pkt)
1271{
1272 assert(pkt->isResponse());
1273
1274 // all header delay should be paid for by the crossbar, unless
1275 // this is a prefetch response from above
1276 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1277 "%s saw a non-zero packet delay\n", name());
1278
1279 bool is_error = pkt->isError();
1280
1281 if (is_error) {
1282 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), "
1283 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns",
1284 pkt->cmdString());
1285 }
1286
1287 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n",
1288 pkt->cmdString(), pkt->getAddr(), pkt->getSize(),
1289 pkt->isSecure() ? "s" : "ns");
1290
1291 // if this is a write, we should be looking at an uncacheable
1292 // write
1293 if (pkt->isWrite()) {
1294 assert(pkt->req->isUncacheable());
1295 handleUncacheableWriteResp(pkt);
1296 return;
1297 }
1298
1299 // we have dealt with any (uncacheable) writes above, from here on
1300 // we know we are dealing with an MSHR due to a miss or a prefetch
1301 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
1302 assert(mshr);
1303
1304 if (mshr == noTargetMSHR) {
1305 // we always clear at least one target
1306 clearBlocked(Blocked_NoTargets);
1307 noTargetMSHR = NULL;
1308 }
1309
1310 // Initial target is used just for stats
1311 MSHR::Target *initial_tgt = mshr->getTarget();
1312 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1313 Tick miss_latency = curTick() - initial_tgt->recvTime;
1314
1315 if (pkt->req->isUncacheable()) {
1316 assert(pkt->req->masterId() < system->maxMasters());
1317 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1318 miss_latency;
1319 } else {
1320 assert(pkt->req->masterId() < system->maxMasters());
1321 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1322 miss_latency;
1323 }
1324
1325 bool wasFull = mshrQueue.isFull();
1326
1327 PacketList writebacks;
1328
1329 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1330
1331 // upgrade deferred targets if the response has no sharers, and is
1332 // thus passing writable
1333 if (!pkt->hasSharers()) {
1334 mshr->promoteWritable();
1335 }
1336
1337 bool is_fill = !mshr->isForward &&
1338 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);

--- 159 unchanged lines hidden (view full) ---

1498 }
1499
1500 if (mshr->promoteDeferredTargets()) {
1501 // avoid later read getting stale data while write miss is
1502 // outstanding.. see comment in timingAccess()
1503 if (blk) {
1504 blk->status &= ~BlkReadable;
1505 }
1506 mshrQueue.markPending(mshr);
1507 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1508 } else {
1509 mshrQueue.deallocate(mshr);
1510 if (wasFull && !mshrQueue.isFull()) {
1511 clearBlocked(Blocked_NoMSHRs);
1512 }
1513
1514 // Request the bus for a prefetch if this deallocation freed enough
1515 // MSHRs for a prefetch to take place
1516 if (prefetcher && mshrQueue.canPrefetch()) {
1517 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1518 clockEdge());
1519 if (next_pf_time != MaxTick)
1520 schedMemSideSendEvent(next_pf_time);
1521 }
1522 }
1523 // reset the xbar additional timinig as it is now accounted for
1524 pkt->headerDelay = pkt->payloadDelay = 0;

--- 217 unchanged lines hidden (view full) ---

1742{
1743 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1744 Addr addr = pkt->getAddr();
1745 bool is_secure = pkt->isSecure();
1746#if TRACING_ON
1747 CacheBlk::State old_state = blk ? blk->status : 0;
1748#endif
1749
1750 // When handling a fill, we should have no writes to this line.
1751 assert(addr == blockAlign(addr));
1752 assert(!writeBuffer.findMatch(addr, is_secure));
1753
1754 if (blk == NULL) {
1755 // better have read new data...
1756 assert(pkt->hasData());
1757
1758 // only read responses and write-line requests have data;
1759 // note that we don't write the data here for write-line - that
1760 // happens in the subsequent satisfyCpuSideRequest.

--- 371 unchanged lines hidden (view full) ---

2132 mshr->print());
2133
2134 if (mshr->getNumTargets() > numTarget)
2135 warn("allocating bonus target for snoop"); //handle later
2136 return;
2137 }
2138
2139 //We also need to check the writeback buffers and handle those
2140 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2141 if (wb_entry) {
2142 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2143 pkt->getAddr(), is_secure ? "s" : "ns");
2144 // Expect to see only Writebacks and/or CleanEvicts here, both of
2145 // which should not be generated for uncacheable data.
2146 assert(!wb_entry->isUncacheable());
2147 // There should only be a single request responsible for generating
2148 // Writebacks/CleanEvicts.
2149 assert(wb_entry->getNumTargets() == 1);
2150 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2151 assert(wb_pkt->isEviction());

--- 34 unchanged lines hidden (view full) ---

2186
2187 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2188 false, false);
2189 }
2190
2191 if (invalidate) {
2192 // Invalidation trumps our writeback... discard here
2193 // Note: markInService will remove entry from writeback buffer.
2194 markInService(wb_entry);
2195 delete wb_pkt;
2196 }
2197 }
2198
2199 // If this was a shared writeback, there may still be
2200 // other shared copies above that require invalidation.
2201 // We could be more selective and return here if the
2202 // request is non-exclusive or if the writeback is

--- 26 unchanged lines hidden (view full) ---

2229 }
2230
2231 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2232 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2233 return snoop_delay + lookupLatency * clockPeriod();
2234}
2235
2236
2237QueueEntry*
2238Cache::getNextQueueEntry()
2239{
2240 // Check both MSHR queue and write buffer for potential requests,
2241 // note that null does not mean there is no request, it could
2242 // simply be that it is not ready
2243 MSHR *miss_mshr = mshrQueue.getNext();
2244 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2245
2246 // If we got a write buffer request ready, first priority is a
2247 // full write buffer (but only if we have no uncacheable write
2248 // responses outstanding, possibly revisit this last part),
2249 // otherwhise we favour the miss requests
2250 if (wq_entry &&
2251 ((writeBuffer.isFull() && writeBuffer.numInService() == 0) ||
2252 !miss_mshr)) {
2253 // need to search MSHR queue for conflicting earlier miss.
2254 MSHR *conflict_mshr =
2255 mshrQueue.findPending(wq_entry->blkAddr,
2256 wq_entry->isSecure);
2257
2258 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2259 // Service misses in order until conflict is cleared.
2260 return conflict_mshr;
2261
2262 // @todo Note that we ignore the ready time of the conflict here
2263 }
2264
2265 // No conflicts; issue write
2266 return wq_entry;
2267 } else if (miss_mshr) {
2268 // need to check for conflicting earlier writeback
2269 WriteQueueEntry *conflict_mshr =
2270 writeBuffer.findPending(miss_mshr->blkAddr,
2271 miss_mshr->isSecure);
2272 if (conflict_mshr) {
2273 // not sure why we don't check order here... it was in the
2274 // original code but commented out.
2275
2276 // The only way this happens is if we are
2277 // doing a write and we didn't have permissions
2278 // then subsequently saw a writeback (owned got evicted)
2279 // We need to make sure to perform the writeback first
2280 // To preserve the dirty data, then we can issue the write
2281
2282 // should we return wq_entry here instead? I.e. do we
2283 // have to flush writes in order? I don't think so... not
2284 // for Alpha anyway. Maybe for x86?
2285 return conflict_mshr;
2286
2287 // @todo Note that we ignore the ready time of the conflict here
2288 }
2289
2290 // No conflicts; issue read
2291 return miss_mshr;
2292 }
2293
2294 // fall through... no pending requests. Try a prefetch.
2295 assert(!miss_mshr && !wq_entry);
2296 if (prefetcher && mshrQueue.canPrefetch()) {
2297 // If we have a miss queue slot, we can try a prefetch
2298 PacketPtr pkt = prefetcher->getPacket();
2299 if (pkt) {
2300 Addr pf_addr = blockAlign(pkt->getAddr());
2301 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2302 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2303 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {

--- 9 unchanged lines hidden (view full) ---

2313 } else {
2314 // free the request and packet
2315 delete pkt->req;
2316 delete pkt;
2317 }
2318 }
2319 }
2320
2321 return nullptr;
2322}
2323
2324bool
2325Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2326{
2327 if (!forwardSnoops)
2328 return false;
2329 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and

--- 14 unchanged lines hidden (view full) ---

2344 assert(!(snoop_pkt.cacheResponding()));
2345 return snoop_pkt.isBlockCached();
2346 } else {
2347 cpuSidePort->sendAtomicSnoop(pkt);
2348 return pkt->isBlockCached();
2349 }
2350}
2351
2352Tick
2353Cache::nextQueueReadyTime() const
2354{
2355 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2356 writeBuffer.nextReadyTime());
2357
2358 // Don't signal prefetch ready time if no MSHRs available
2359 // Will signal once enoguh MSHRs are deallocated
2360 if (prefetcher && mshrQueue.canPrefetch()) {
2361 nextReady = std::min(nextReady,
2362 prefetcher->nextPrefetchReadyTime());
2363 }
2364
2365 return nextReady;
2366}
2367
2368bool
2369Cache::sendMSHRQueuePacket(MSHR* mshr)
2370{
2371 assert(mshr);
2372
2373 // use request from 1st target
2374 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2375
2376 DPRINTF(Cache, "%s MSHR %s for addr %#llx size %d\n", __func__,
2377 tgt_pkt->cmdString(), tgt_pkt->getAddr(),
2378 tgt_pkt->getSize());
2379
2380 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2381
2382 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2383 // we should never have hardware prefetches to allocated
2384 // blocks
2385 assert(blk == NULL);
2386
2387 // We need to check the caches above us to verify that
2388 // they don't have a copy of this block in the dirty state
2389 // at the moment. Without this check we could get a stale
2390 // copy from memory that might get used in place of the
2391 // dirty one.
2392 Packet snoop_pkt(tgt_pkt, true, false);
2393 snoop_pkt.setExpressSnoop();
2394 // We are sending this packet upwards, but if it hits we will

--- 22 unchanged lines hidden (view full) ---

2417 // if we are getting a snoop response with no sharers it
2418 // will be allocated as Modified
2419 bool pending_modified_resp = !snoop_pkt.hasSharers();
2420 markInService(mshr, pending_modified_resp);
2421
2422 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2423 " %#x (%s) hit\n",
2424 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2425 return false;
2426 }
2427
2428 if (snoop_pkt.isBlockCached()) {
2429 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2430 "Deallocating mshr target %#x.\n",
2431 mshr->blkAddr);
2432
2433 // Deallocate the mshr target
2434 if (mshrQueue.forceDeallocateTarget(mshr)) {
2435 // Clear block if this deallocation resulted freed an
2436 // mshr when all had previously been utilized
2437 clearBlocked(Blocked_NoMSHRs);
2438 }
2439 return false;
2440 }
2441 }
2442
2443 // either a prefetch that is not present upstream, or a normal
2444 // MSHR request, proceed to get the packet to send downstream
2445 PacketPtr pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable());
2446
2447 mshr->isForward = (pkt == NULL);
2448
2449 if (mshr->isForward) {
2450 // not a cache block request, but a response is expected
2451 // make copy of current packet to forward, keep current
2452 // copy for response handling
2453 pkt = new Packet(tgt_pkt, false, true);
2454 assert(!pkt->isWrite());
2455 }
2456
2457 // play it safe and append (rather than set) the sender state,
2458 // as forwarded packets may already have existing state
2459 pkt->pushSenderState(mshr);
2460
2461 if (!memSidePort->sendTimingReq(pkt)) {
2462 // we are awaiting a retry, but we
2463 // delete the packet and will be creating a new packet
2464 // when we get the opportunity
2465 delete pkt;
2466
2467 // note that we have now masked any requestBus and
2468 // schedSendEvent (we will wait for a retry before
2469 // doing anything), and this is so even if we do not
2470 // care about this packet and might override it before
2471 // it gets retried
2472 return true;
2473 } else {
2474 // As part of the call to sendTimingReq the packet is
2475 // forwarded to all neighbouring caches (and any caches
2476 // above them) as a snoop. Thus at this point we know if
2477 // any of the neighbouring caches are responding, and if
2478 // so, we know it is dirty, and we can determine if it is
2479 // being passed as Modified, making our MSHR the ordering
2480 // point
2481 bool pending_modified_resp = !pkt->hasSharers() &&
2482 pkt->cacheResponding();
2483 markInService(mshr, pending_modified_resp);
2484 return false;
2485 }
2486}
2487
2488bool
2489Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2490{
2491 assert(wq_entry);
2492
2493 // always a single target for write queue entries
2494 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2495
2496 DPRINTF(Cache, "%s write %s for addr %#llx size %d\n", __func__,
2497 tgt_pkt->cmdString(), tgt_pkt->getAddr(),
2498 tgt_pkt->getSize());
2499
2500 PacketPtr pkt = nullptr;
2501 bool delete_pkt = false;
2502
2503 if (tgt_pkt->isEviction()) {
2504 assert(!wq_entry->isUncacheable());
2505 // no response expected, just forward packet as it is
2506 pkt = tgt_pkt;
2507 } else {
2508 // the only thing we deal with besides eviction commands
2509 // are uncacheable writes
2510 assert(tgt_pkt->req->isUncacheable() && tgt_pkt->isWrite());
2511 // not a cache block request, but a response is expected
2512 // make copy of current packet to forward, keep current
2513 // copy for response handling
2514 pkt = new Packet(tgt_pkt, false, true);
2515 pkt->setData(tgt_pkt->getConstPtr<uint8_t>());
2516 delete_pkt = true;
2517 }
2518
2519 pkt->pushSenderState(wq_entry);
2520
2521 if (!memSidePort->sendTimingReq(pkt)) {
2522 if (delete_pkt) {
2523 // we are awaiting a retry, but we
2524 // delete the packet and will be creating a new packet
2525 // when we get the opportunity
2526 delete pkt;
2527 }
2528 // note that we have now masked any requestBus and
2529 // schedSendEvent (we will wait for a retry before
2530 // doing anything), and this is so even if we do not
2531 // care about this packet and might override it before
2532 // it gets retried
2533 return true;
2534 } else {
2535 markInService(wq_entry);
2536 return false;
2537 }
2538}
2539
2540void
2541Cache::serialize(CheckpointOut &cp) const
2542{
2543 bool dirty(isDirty());
2544
2545 if (dirty) {

--- 132 unchanged lines hidden (view full) ---

2678 assert(!waitingOnRetry);
2679
2680 // there should never be any deferred request packets in the
2681 // queue, instead we resly on the cache to provide the packets
2682 // from the MSHR queue or write queue
2683 assert(deferredPacketReadyTime() == MaxTick);
2684
2685 // check for request packets (requests & writebacks)
2686 QueueEntry* entry = cache.getNextQueueEntry();
2687
2688 if (!entry) {
2689 // can happen if e.g. we attempt a writeback and fail, but
2690 // before the retry, the writeback is eliminated because
2691 // we snoop another cache's ReadEx.
2692 } else {
2693 // let our snoop responses go first if there are responses to
2694 // the same addresses
2695 if (checkConflictingSnoop(entry->blkAddr)) {
2696 return;
2697 }
2698 waitingOnRetry = entry->sendPacket(cache);
2699 }
2700
2701 // if we succeeded and are not waiting for a retry, schedule the
2702 // next send considering when the next queue is ready, note that
2703 // snoop responses have their own packet queue and thus schedule
2704 // their own events
2705 if (!waitingOnRetry) {
2706 schedSendEvent(cache.nextQueueReadyTime());
2707 }
2708}
2709
2710Cache::
2711MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2712 const std::string &_label)
2713 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2714 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2715 _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2716{
2717}