1/*
2 * Copyright (c) 2010-2015 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 56 unchanged lines hidden (view full) ---

65
66Cache::Cache(const CacheParams *p)
67 : BaseCache(p, p->system->cacheLineSize()),
68 tags(p->tags),
69 prefetcher(p->prefetcher),
70 doFastWrites(true),
71 prefetchOnAccess(p->prefetch_on_access),
72 clusivity(p->clusivity),
73 tempBlockWriteback(nullptr),
74 writebackTempBlockAtomicEvent(this, false,
75 EventBase::Delayed_Writeback_Pri)
76{
77 tempBlock = new CacheBlk();
78 tempBlock->data = new uint8_t[blkSize];
79
80 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,

--- 231 unchanged lines hidden (view full) ---

312 if (pkt->req->isUncacheable()) {
313 DPRINTF(Cache, "%s%s addr %#llx uncacheable\n", pkt->cmdString(),
314 pkt->req->isInstFetch() ? " (ifetch)" : "",
315 pkt->getAddr());
316
317 // flush and invalidate any existing block
318 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
319 if (old_blk && old_blk->isValid()) {
320 if (old_blk->isDirty())
321 writebacks.push_back(writebackBlk(old_blk));
322 else
323 writebacks.push_back(cleanEvictBlk(old_blk));
324 tags->invalidate(old_blk);
325 old_blk->invalidate();
326 }
327
328 blk = NULL;

--- 9 unchanged lines hidden (view full) ---

338 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat, id);
339
340 DPRINTF(Cache, "%s%s addr %#llx size %d (%s) %s\n", pkt->cmdString(),
341 pkt->req->isInstFetch() ? " (ifetch)" : "",
342 pkt->getAddr(), pkt->getSize(), pkt->isSecure() ? "s" : "ns",
343 blk ? "hit " + blk->print() : "miss");
344
345
346 if (pkt->evictingBlock()) {
347 // We check for presence of block in above caches before issuing
348 // Writeback or CleanEvict to write buffer. Therefore the only
349 // possible cases can be of a CleanEvict packet coming from above
350 // encountering a Writeback generated in this cache peer cache and
351 // waiting in the write buffer. Cases of upper level peer caches
352 // generating CleanEvict and Writeback or simply CleanEvict and
353 // CleanEvict almost simultaneously will be caught by snoops sent out
354 // by crossbar.
355 std::vector<MSHR *> outgoing;
356 if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(),
357 outgoing)) {
358 assert(outgoing.size() == 1);
359 PacketPtr wbPkt = outgoing[0]->getTarget()->pkt;
360 assert(pkt->cmd == MemCmd::CleanEvict &&
361 wbPkt->cmd == MemCmd::Writeback);
362 // As the CleanEvict is coming from above, it would have snooped
363 // into other peer caches of the same level while traversing the
364 // crossbar. If a copy of the block had been found, the CleanEvict
365 // would have been deleted in the crossbar. Now that the
366 // CleanEvict is here we can be sure none of the other upper level
367 // caches connected to this cache have the block, so we can clear
368 // the BLOCK_CACHED flag in the Writeback if set and discard the
369 // CleanEvict by returning true.
370 wbPkt->clearBlockCached();
371 return true;
372 }
373 }
374
375 // Writeback handling is special case. We can write the block into
376 // the cache without having a writeable copy (or any copy at all).
377 if (pkt->cmd == MemCmd::Writeback) {
378 assert(blkSize == pkt->getSize());
379 if (blk == NULL) {
380 // need to do a replacement
381 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
382 if (blk == NULL) {
383 // no replaceable block available: give up, fwd to next level.
384 incMissCount(pkt);
385 return false;
386 }
387 tags->insertBlock(pkt, blk);
388
389 blk->status = (BlkValid | BlkReadable);
390 if (pkt->isSecure()) {
391 blk->status |= BlkSecure;
392 }
393 }
394 blk->status |= BlkDirty;
395 // if shared is not asserted we got the writeback in modified
396 // state, if it is asserted we are in the owned state
397 if (!pkt->sharedAsserted()) {
398 blk->status |= BlkWritable;
399 }
400 // nothing else to do; writeback doesn't expect response
401 assert(!pkt->needsResponse());
402 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);

--- 55 unchanged lines hidden (view full) ---

458 // in Writebacks and discard CleanEvicts.
459 if (isCachedAbove(wbPkt)) {
460 if (wbPkt->cmd == MemCmd::CleanEvict) {
461 // Delete CleanEvict because cached copies exist above. The
462 // packet destructor will delete the request object because
463 // this is a non-snoop request packet which does not require a
464 // response.
465 delete wbPkt;
466 } else {
467 // Set BLOCK_CACHED flag in Writeback and send below, so that
468 // the Writeback does not reset the bit corresponding to this
469 // address in the snoop filter below.
470 wbPkt->setBlockCached();
471 allocateWriteBuffer(wbPkt, forward_time);
472 }
473 } else {
474 // If the block is not cached above, send packet below. Both

--- 10 unchanged lines hidden (view full) ---

485Cache::doWritebacksAtomic(PacketList& writebacks)
486{
487 while (!writebacks.empty()) {
488 PacketPtr wbPkt = writebacks.front();
489 // Call isCachedAbove for both Writebacks and CleanEvicts. If
490 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
491 // and discard CleanEvicts.
492 if (isCachedAbove(wbPkt, false)) {
493 if (wbPkt->cmd == MemCmd::Writeback) {
494 // Set BLOCK_CACHED flag in Writeback and send below,
495 // so that the Writeback does not reset the bit
496 // corresponding to this address in the snoop filter
497 // below. We can discard CleanEvicts because cached
498 // copies exist above. Atomic mode isCachedAbove
499 // modifies packet to set BLOCK_CACHED flag
500 memSidePort->sendAtomic(wbPkt);
501 }

--- 187 unchanged lines hidden (view full) ---

689
690 // In this case we are considering request_time that takes
691 // into account the delay of the xbar, if any, and just
692 // lat, neglecting responseLatency, modelling hit latency
693 // just as lookupLatency or or the value of lat overriden
694 // by access(), that calls accessBlock() function.
695 cpuSidePort->schedTimingResp(pkt, request_time, true);
696 } else {
697 // queue the packet for deletion, as the sending cache is
698 // still relying on it; if the block is found in access(),
699 // CleanEvict and Writeback messages will be deleted
700 // here as well
701 pendingDelete.reset(pkt);
702 }
703 } else {
704 // miss

--- 55 unchanged lines hidden (view full) ---

760 /// MSHR hit
761 /// @note writebacks will be checked in getNextMSHR()
762 /// for any conflicting requests to the same block
763
764 //@todo remove hw_pf here
765
766 // Coalesce unless it was a software prefetch (see above).
767 if (pkt) {
768 assert(pkt->cmd != MemCmd::Writeback);
769 // CleanEvicts corresponding to blocks which have outstanding
770 // requests in MSHRs can be deleted here.
771 if (pkt->cmd == MemCmd::CleanEvict) {
772 pendingDelete.reset(pkt);
773 } else {
774 DPRINTF(Cache, "%s coalescing MSHR for %s addr %#llx size %d\n",
775 __func__, pkt->cmdString(), pkt->getAddr(),
776 pkt->getSize());
777
778 assert(pkt->req->masterId() < system->maxMasters());

--- 36 unchanged lines hidden (view full) ---

815 // no MSHR
816 assert(pkt->req->masterId() < system->maxMasters());
817 if (pkt->req->isUncacheable()) {
818 mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
819 } else {
820 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
821 }
822
823 if (pkt->evictingBlock() ||
824 (pkt->req->isUncacheable() && pkt->isWrite())) {
825 // We use forward_time here because there is an
826 // uncached memory write, forwarded to WriteBuffer.
827 allocateWriteBuffer(pkt, forward_time);
828 } else {
829 if (blk && blk->isValid()) {
830 // should have flushed and have no valid block
831 assert(!pkt->req->isUncacheable());

--- 51 unchanged lines hidden (view full) ---

883 // and the cache could have filled again before we actually
884 // send out the forwarded uncacheable request (blk could thus
885 // be non-null)
886 return NULL;
887 }
888
889 if (!blkValid &&
890 (cpu_pkt->isUpgrade() ||
891 cpu_pkt->evictingBlock())) {
892 // Writebacks that weren't allocated in access() and upgrades
893 // from upper-level caches that missed completely just go
894 // through.
895 return NULL;
896 }
897
898 assert(cpu_pkt->needsResponse());
899

--- 203 unchanged lines hidden (view full) ---

1103 } else {
1104 // the writeback/clean eviction happens after the call to
1105 // recvAtomic has finished (but before any successive
1106 // calls), so that the response handling from the fill is
1107 // allowed to happen first
1108 schedule(writebackTempBlockAtomicEvent, curTick());
1109 }
1110
1111 tempBlockWriteback = blk->isDirty() ? writebackBlk(blk) :
1112 cleanEvictBlk(blk);
1113 blk->invalidate();
1114 }
1115
1116 if (pkt->needsResponse()) {
1117 pkt->makeAtomicResponse();
1118 }
1119
1120 return lat * clockPeriod();

--- 332 unchanged lines hidden (view full) ---

1453 doWritebacks(writebacks, forward_time);
1454
1455 // if we used temp block, check to see if its valid and then clear it out
1456 if (blk == tempBlock && tempBlock->isValid()) {
1457 // We use forwardLatency here because we are copying
1458 // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1459 // allocate an internal buffer and to schedule an event to the
1460 // queued port.
1461 if (blk->isDirty()) {
1462 PacketPtr wbPkt = writebackBlk(blk);
1463 allocateWriteBuffer(wbPkt, forward_time);
1464 // Set BLOCK_CACHED flag if cached above.
1465 if (isCachedAbove(wbPkt))
1466 wbPkt->setBlockCached();
1467 } else {
1468 PacketPtr wcPkt = cleanEvictBlk(blk);
1469 // Check to see if block is cached above. If not allocate

--- 9 unchanged lines hidden (view full) ---

1479 DPRINTF(Cache, "Leaving %s with %s for addr %#llx\n", __func__,
1480 pkt->cmdString(), pkt->getAddr());
1481 delete pkt;
1482}
1483
1484PacketPtr
1485Cache::writebackBlk(CacheBlk *blk)
1486{
1487 chatty_assert(!isReadOnly, "Writeback from read-only cache");
1488 assert(blk && blk->isValid() && blk->isDirty());
1489
1490 writebacks[Request::wbMasterId]++;
1491
1492 Request *writebackReq =
1493 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1494 Request::wbMasterId);
1495 if (blk->isSecure())
1496 writebackReq->setFlags(Request::SECURE);
1497
1498 writebackReq->taskId(blk->task_id);
1499 blk->task_id= ContextSwitchTaskId::Unknown;
1500 blk->tickInserted = curTick();
1501
1502 PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback);
1503 if (blk->isWritable()) {
1504 // not asserting shared means we pass the block in modified
1505 // state, mark our own block non-writeable
1506 blk->status &= ~BlkWritable;
1507 } else {
1508 // we are in the owned state, tell the receiver
1509 writeback->assertShared();
1510 }
1511
1512 writeback->allocate();
1513 std::memcpy(writeback->getPtr<uint8_t>(), blk->data, blkSize);
1514
1515 blk->status &= ~BlkDirty;
1516 return writeback;
1517}
1518
1519PacketPtr
1520Cache::cleanEvictBlk(CacheBlk *blk)
1521{
1522 assert(blk && blk->isValid() && !blk->isDirty());
1523 // Creating a zero sized write, a message to the snoop filter
1524 Request *req =
1525 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1526 Request::wbMasterId);
1527 if (blk->isSecure())
1528 req->setFlags(Request::SECURE);
1529

--- 93 unchanged lines hidden (view full) ---

1623 } else {
1624 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx (%s): %s\n",
1625 repl_addr, blk->isSecure() ? "s" : "ns",
1626 addr, is_secure ? "s" : "ns",
1627 blk->isDirty() ? "writeback" : "clean");
1628
1629 // Will send up Writeback/CleanEvict snoops via isCachedAbove
1630 // when pushing this writeback list into the write buffer.
1631 if (blk->isDirty()) {
1632 // Save writeback packet for handling by caller
1633 writebacks.push_back(writebackBlk(blk));
1634 } else {
1635 writebacks.push_back(cleanEvictBlk(blk));
1636 }
1637 }
1638 }
1639

--- 406 unchanged lines hidden (view full) ---

2046 MSHR *wb_entry = writebacks[0];
2047 // Expect to see only Writebacks and/or CleanEvicts here, both of
2048 // which should not be generated for uncacheable data.
2049 assert(!wb_entry->isUncacheable());
2050 // There should only be a single request responsible for generating
2051 // Writebacks/CleanEvicts.
2052 assert(wb_entry->getNumTargets() == 1);
2053 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2054 assert(wb_pkt->evictingBlock());
2055
2056 if (pkt->evictingBlock()) {
2057 // if the block is found in the write queue, set the BLOCK_CACHED
2058 // flag for Writeback/CleanEvict snoop. On return the snoop will
2059 // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2060 // any CleanEvicts from travelling down the memory hierarchy.
2061 pkt->setBlockCached();
2062 DPRINTF(Cache, "Squashing %s from lower cache on writequeue hit"
2063 " %#x\n", pkt->cmdString(), pkt->getAddr());
2064 return;
2065 }
2066
2067 if (wb_pkt->cmd == MemCmd::Writeback) {
2068 assert(!pkt->memInhibitAsserted());
2069 pkt->assertMemInhibit();
2070 if (!pkt->needsExclusive()) {
2071 pkt->assertShared();
2072 // the writeback is no longer passing exclusivity (the
2073 // receiving cache should consider the block owned
2074 // rather than modified)
2075 wb_pkt->assertShared();
2076 } else {
2077 // if we're not asserting the shared line, we need to
2078 // invalidate our copy. we'll do that below as long as
2079 // the packet's invalidate flag is set...
2080 assert(pkt->isInvalidate());
2081 }
2082 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2083 false, false);
2084 } else {
2085 assert(wb_pkt->cmd == MemCmd::CleanEvict);
2086 // The cache technically holds the block until the
2087 // corresponding CleanEvict message reaches the crossbar
2088 // below. Therefore when a snoop encounters a CleanEvict
2089 // message we must set assertShared (just like when it
2090 // encounters a Writeback) to avoid the snoop filter
2091 // prematurely clearing the holder bit in the crossbar
2092 // below
2093 if (!pkt->needsExclusive())
2094 pkt->assertShared();
2095 else
2096 assert(pkt->isInvalidate());
2097 }
2098
2099 if (pkt->isInvalidate()) {
2100 // Invalidation trumps our writeback... discard here
2101 // Note: markInService will remove entry from writeback buffer.
2102 markInService(wb_entry, false);
2103 delete wb_pkt;
2104 }

--- 133 unchanged lines hidden (view full) ---

2238 // packet, the cache can inform the crossbar below of presence or absence
2239 // of the block.
2240 if (is_timing) {
2241 Packet snoop_pkt(pkt, true, false);
2242 snoop_pkt.setExpressSnoop();
2243 // Assert that packet is either Writeback or CleanEvict and not a
2244 // prefetch request because prefetch requests need an MSHR and may
2245 // generate a snoop response.
2246 assert(pkt->evictingBlock());
2247 snoop_pkt.senderState = NULL;
2248 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2249 // Writeback/CleanEvict snoops do not generate a snoop response.
2250 assert(!(snoop_pkt.memInhibitAsserted()));
2251 return snoop_pkt.isBlockCached();
2252 } else {
2253 cpuSidePort->sendAtomicSnoop(pkt);
2254 return pkt->isBlockCached();

--- 52 unchanged lines hidden (view full) ---

2307 }
2308
2309 if (snoop_pkt.isBlockCached() || blk != NULL) {
2310 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2311 "Deallocating mshr target %#x.\n",
2312 mshr->blkAddr);
2313
2314 // Deallocate the mshr target
2315 if (tgt_pkt->cmd != MemCmd::Writeback) {
2316 if (mshr->queue->forceDeallocateTarget(mshr)) {
2317 // Clear block if this deallocation resulted freed an
2318 // mshr when all had previously been utilized
2319 clearBlocked((BlockedCause)(mshr->queue->index));
2320 }
2321 return NULL;
2322 } else {
2323 // If this is a Writeback, and the snoops indicate that the blk

--- 276 unchanged lines hidden ---