| 1/* 2 * Copyright (c) 2010-2016 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 166 unchanged lines hidden (view full) --- 175 if (blk->checkWrite(pkt)) { 176 pkt->writeDataToBlock(blk->data, blkSize); 177 } 178 // Always mark the line as dirty (and thus transition to the 179 // Modified state) even if we are a failed StoreCond so we 180 // supply data to any snoops that have appended themselves to 181 // this cache before knowing the store will fail. 182 blk->status |= BlkDirty; |
| 183 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print()); |
| 184 } else if (pkt->isRead()) { 185 if (pkt->isLLSC()) { 186 blk->trackLoadLocked(pkt); 187 } 188 189 // all read responses have a data payload 190 assert(pkt->hasRespData()); 191 pkt->setDataFromBlock(blk->data, blkSize); --- 74 unchanged lines hidden (view full) --- 266 // has the line in Shared state needs to be made aware 267 // that the data it already has is in fact dirty 268 pkt->setCacheResponding(); 269 blk->status &= ~BlkDirty; 270 } 271 } else { 272 assert(pkt->isInvalidate()); 273 invalidateBlock(blk); |
| 274 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__, 275 pkt->print()); |
| 276 } 277} 278 279///////////////////////////////////////////////////// 280// 281// Access path: requests coming in from the CPU side 282// 283///////////////////////////////////////////////////// --- 4 unchanged lines hidden (view full) --- 288{ 289 // sanity check 290 assert(pkt->isRequest()); 291 292 chatty_assert(!(isReadOnly && pkt->isWrite()), 293 "Should never see a write in a read-only cache %s\n", 294 name()); 295 |
| 296 DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print()); |
| 297 298 if (pkt->req->isUncacheable()) { |
| 299 DPRINTF(Cache, "uncacheable: %s\n", pkt->print()); |
| 300 301 // flush and invalidate any existing block 302 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure())); 303 if (old_blk && old_blk->isValid()) { 304 if (old_blk->isDirty() || writebackClean) 305 writebacks.push_back(writebackBlk(old_blk)); 306 else 307 writebacks.push_back(cleanEvictBlk(old_blk)); --- 8 unchanged lines hidden (view full) --- 316 } 317 318 ContextID id = pkt->req->hasContextId() ? 319 pkt->req->contextId() : InvalidContextID; 320 // Here lat is the value passed as parameter to accessBlock() function 321 // that can modify its value. 322 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat, id); 323 |
| 324 DPRINTF(Cache, "%s %s\n", pkt->print(), |
| 325 blk ? "hit " + blk->print() : "miss"); 326 327 328 if (pkt->isEviction()) { 329 // We check for presence of block in above caches before issuing 330 // Writeback or CleanEvict to write buffer. Therefore the only 331 // possible cases can be of a CleanEvict packet coming from above 332 // encountering a Writeback generated in this cache peer cache and --- 199 unchanged lines hidden (view full) --- 532 delete wbPkt; 533 } 534} 535 536 537void 538Cache::recvTimingSnoopResp(PacketPtr pkt) 539{ |
| 540 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print()); |
| 541 542 assert(pkt->isResponse()); 543 assert(!system->bypassCaches()); 544 545 // determine if the response is from a snoop request we created 546 // (in which case it should be in the outstandingSnoop), or if we 547 // merely forwarded someone else's snoop request 548 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == --- 49 unchanged lines hidden (view full) --- 598 } 599 600 promoteWholeLineWrites(pkt); 601 602 if (pkt->cacheResponding()) { 603 // a cache above us (but not where the packet came from) is 604 // responding to the request, in other words it has the line 605 // in Modified or Owned state |
| 606 DPRINTF(Cache, "Cache above responding to %s: not responding\n", 607 pkt->print()); |
| 608 609 // if the packet needs the block to be writable, and the cache 610 // that has promised to respond (setting the cache responding 611 // flag) is not providing writable (it is in Owned rather than 612 // the Modified state), we know that there may be other Shared 613 // copies in the system; go out and invalidate them all 614 assert(pkt->needsWritable() && !pkt->responderHadWritable()); 615 --- 107 unchanged lines hidden (view full) --- 723 724 // In this case we are considering request_time that takes 725 // into account the delay of the xbar, if any, and just 726 // lat, neglecting responseLatency, modelling hit latency 727 // just as lookupLatency or or the value of lat overriden 728 // by access(), that calls accessBlock() function. 729 cpuSidePort->schedTimingResp(pkt, request_time, true); 730 } else { |
| 731 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__, 732 pkt->print()); |
| 733 734 // queue the packet for deletion, as the sending cache is 735 // still relying on it; if the block is found in access(), 736 // CleanEvict and Writeback messages will be deleted 737 // here as well 738 pendingDelete.reset(pkt); 739 } 740 } else { --- 59 unchanged lines hidden (view full) --- 800 // Coalesce unless it was a software prefetch (see above). 801 if (pkt) { 802 assert(!pkt->isWriteback()); 803 // CleanEvicts corresponding to blocks which have 804 // outstanding requests in MSHRs are simply sunk here 805 if (pkt->cmd == MemCmd::CleanEvict) { 806 pendingDelete.reset(pkt); 807 } else { |
| 808 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__, 809 pkt->print()); |
| 810 811 assert(pkt->req->masterId() < system->maxMasters()); 812 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++; 813 // We use forward_time here because it is the same 814 // considering new targets. We have multiple 815 // requests for the same address here. It 816 // specifies the latency to allocate an internal 817 // buffer and to schedule an event to the queued --- 135 unchanged lines hidden (view full) --- 953 // packet as having sharers (not passing writable), pass that info 954 // downstream 955 if (cpu_pkt->hasSharers() && !needsWritable) { 956 // note that cpu_pkt may have spent a considerable time in the 957 // MSHR queue and that the information could possibly be out 958 // of date, however, there is no harm in conservatively 959 // assuming the block has sharers 960 pkt->setHasSharers(); |
| 961 DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n", 962 __func__, cpu_pkt->print(), pkt->print()); |
| 963 } 964 965 // the packet should be block aligned 966 assert(pkt->getAddr() == blockAlign(pkt->getAddr())); 967 968 pkt->allocate(); |
| 969 DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(), 970 cpu_pkt->print()); |
| 971 return pkt; 972} 973 974 975Tick 976Cache::recvAtomic(PacketPtr pkt) 977{ 978 // We are in atomic mode so we pay just for lookupLatency here. 979 Cycles lat = lookupLatency; 980 981 // Forward the request if the system is in cache bypass mode. 982 if (system->bypassCaches()) 983 return ticksToCycles(memSidePort->sendAtomic(pkt)); 984 985 promoteWholeLineWrites(pkt); 986 987 // follow the same flow as in recvTimingReq, and check if a cache 988 // above us is responding 989 if (pkt->cacheResponding()) { |
| 990 DPRINTF(Cache, "Cache above responding to %s: not responding\n", 991 pkt->print()); |
| 992 993 // if a cache is responding, and it had the line in Owned 994 // rather than Modified state, we need to invalidate any 995 // copies that are not on the same path to memory 996 assert(pkt->needsWritable() && !pkt->responderHadWritable()); 997 lat += ticksToCycles(memSidePort->sendAtomic(pkt)); 998 999 return lat * clockPeriod(); --- 28 unchanged lines hidden (view full) --- 1028 bool is_forward = (bus_pkt == nullptr); 1029 1030 if (is_forward) { 1031 // just forwarding the same request to the next level 1032 // no local cache operation involved 1033 bus_pkt = pkt; 1034 } 1035 |
| 1036 DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__, 1037 bus_pkt->print()); |
| 1038 1039#if TRACING_ON 1040 CacheBlk::State old_state = blk ? blk->status : 0; 1041#endif 1042 1043 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt)); 1044 1045 bool is_invalidate = bus_pkt->isInvalidate(); 1046 1047 // We are now dealing with the response handling |
| 1048 DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__, 1049 bus_pkt->print(), old_state); |
| 1050 1051 // If packet was a forward, the response (if any) is already 1052 // in place in the bus_pkt == pkt structure, so we don't need 1053 // to do anything. Otherwise, use the separate bus_pkt to 1054 // generate response to pkt and then delete it. 1055 if (!is_forward) { 1056 if (pkt->needsResponse()) { 1057 assert(bus_pkt->isResponse()); --- 119 unchanged lines hidden (view full) --- 1177 (mshr && mshr->inService && mshr->isPendingModified())); 1178 1179 bool done = have_dirty 1180 || cpuSidePort->checkFunctional(pkt) 1181 || mshrQueue.checkFunctional(pkt, blk_addr) 1182 || writeBuffer.checkFunctional(pkt, blk_addr) 1183 || memSidePort->checkFunctional(pkt); 1184 |
| 1185 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(), |
| 1186 (blk && blk->isValid()) ? "valid " : "", 1187 have_data ? "data " : "", done ? "done " : ""); 1188 1189 // We're leaving the cache, so pop cache->name() label 1190 pkt->popLabel(); 1191 1192 if (done) { 1193 pkt->makeResponse(); --- 38 unchanged lines hidden (view full) --- 1232 // all header delay should be paid for by the crossbar, unless 1233 // this is a prefetch response from above 1234 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp, 1235 "%s saw a non-zero packet delay\n", name()); 1236 1237 bool is_error = pkt->isError(); 1238 1239 if (is_error) { |
| 1240 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__, 1241 pkt->print()); |
| 1242 } 1243 |
| 1244 DPRINTF(Cache, "%s: Handling response %s\n", __func__, 1245 pkt->print()); |
| 1246 1247 // if this is a write, we should be looking at an uncacheable 1248 // write 1249 if (pkt->isWrite()) { 1250 assert(pkt->req->isUncacheable()); 1251 handleUncacheableWriteResp(pkt); 1252 return; 1253 } --- 156 unchanged lines hidden (view full) --- 1410 if (is_error) 1411 tgt_pkt->copyError(pkt); 1412 if (tgt_pkt->cmd == MemCmd::ReadResp && 1413 (is_invalidate || mshr->hasPostInvalidate())) { 1414 // If intermediate cache got ReadRespWithInvalidate, 1415 // propagate that. Response should not have 1416 // isInvalidate() set otherwise. 1417 tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate; |
| 1418 DPRINTF(Cache, "%s: updated cmd to %s\n", __func__, 1419 tgt_pkt->print()); |
| 1420 } 1421 // Reset the bus additional time as it is now accounted for 1422 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0; 1423 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true); 1424 break; 1425 1426 case MSHR::Target::FromPrefetcher: 1427 assert(tgt_pkt->cmd == MemCmd::HardPFReq); --- 78 unchanged lines hidden (view full) --- 1506 if (isCachedAbove(wcPkt)) 1507 delete wcPkt; 1508 else 1509 allocateWriteBuffer(wcPkt, forward_time); 1510 } 1511 blk->invalidate(); 1512 } 1513 |
| 1514 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print()); |
| 1515 delete pkt; 1516} 1517 1518PacketPtr 1519Cache::writebackBlk(CacheBlk *blk) 1520{ 1521 chatty_assert(!isReadOnly || writebackClean, 1522 "Writeback from read-only cache"); --- 9 unchanged lines hidden (view full) --- 1532 req->taskId(blk->task_id); 1533 blk->task_id= ContextSwitchTaskId::Unknown; 1534 blk->tickInserted = curTick(); 1535 1536 PacketPtr pkt = 1537 new Packet(req, blk->isDirty() ? 1538 MemCmd::WritebackDirty : MemCmd::WritebackClean); 1539 |
| 1540 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n", 1541 pkt->print(), blk->isWritable(), blk->isDirty()); |
| 1542 1543 if (blk->isWritable()) { 1544 // not asserting shared means we pass the block in modified 1545 // state, mark our own block non-writeable 1546 blk->status &= ~BlkWritable; 1547 } else { 1548 // we are in the Owned state, tell the receiver 1549 pkt->setHasSharers(); --- 21 unchanged lines hidden (view full) --- 1571 req->setFlags(Request::SECURE); 1572 1573 req->taskId(blk->task_id); 1574 blk->task_id = ContextSwitchTaskId::Unknown; 1575 blk->tickInserted = curTick(); 1576 1577 PacketPtr pkt = new Packet(req, MemCmd::CleanEvict); 1578 pkt->allocate(); |
| 1579 DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print()); |
| 1580 1581 return pkt; 1582} 1583 1584void 1585Cache::memWriteback() 1586{ 1587 CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor); --- 231 unchanged lines hidden (view full) --- 1819void 1820Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data, 1821 bool already_copied, bool pending_inval) 1822{ 1823 // sanity check 1824 assert(req_pkt->isRequest()); 1825 assert(req_pkt->needsResponse()); 1826 |
| 1827 DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print()); |
| 1828 // timing-mode snoop responses require a new packet, unless we 1829 // already made a copy... 1830 PacketPtr pkt = req_pkt; 1831 if (!already_copied) 1832 // do not clear flags, and allocate space for data if the 1833 // packet needs it (the only packets that carry data are read 1834 // responses) 1835 pkt = new Packet(req_pkt, false, req_pkt->isRead()); --- 15 unchanged lines hidden (view full) --- 1851 pkt->cmd = MemCmd::ReadRespWithInvalidate; 1852 } 1853 // Here we consider forward_time, paying for just forward latency and 1854 // also charging the delay provided by the xbar. 1855 // forward_time is used as send_time in next allocateWriteBuffer(). 1856 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 1857 // Here we reset the timing of the packet. 1858 pkt->headerDelay = pkt->payloadDelay = 0; |
| 1859 DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__, 1860 pkt->print(), forward_time); |
| 1861 memSidePort->schedTimingSnoopResp(pkt, forward_time, true); 1862} 1863 1864uint32_t 1865Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing, 1866 bool is_deferred, bool pending_inval) 1867{ |
| 1868 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print()); |
| 1869 // deferred snoops can only happen in timing mode 1870 assert(!(is_deferred && !is_timing)); 1871 // pending_inval only makes sense on deferred snoops 1872 assert(!(pending_inval && !is_deferred)); 1873 assert(pkt->isRequest()); 1874 1875 // the packet may get modified if we or a forwarded snooper 1876 // responds in atomic mode, so remember a few things about the 1877 // original packet up front 1878 bool invalidate = pkt->isInvalidate(); 1879 bool M5_VAR_USED needs_writable = pkt->needsWritable(); 1880 1881 // at the moment we could get an uncacheable write which does not 1882 // have the invalidate flag, and we need a suitable way of dealing 1883 // with this case 1884 panic_if(invalidate && pkt->req->isUncacheable(), |
| 1885 "%s got an invalidating uncacheable snoop request %s", 1886 name(), pkt->print()); |
| 1887 1888 uint32_t snoop_delay = 0; 1889 1890 if (forwardSnoops) { 1891 // first propagate snoop upward to see if anyone above us wants to 1892 // handle it. save & restore packet src since it will get 1893 // rewritten to be relative to cpu-side bus (if any) 1894 bool alreadyResponded = pkt->cacheResponding(); --- 36 unchanged lines hidden (view full) --- 1931 // cache-to-cache response from some upper cache: 1932 // forward response to original requester 1933 assert(pkt->isResponse()); 1934 } 1935 } 1936 } 1937 1938 if (!blk || !blk->isValid()) { |
| 1939 DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__, 1940 pkt->print()); |
| 1941 if (is_deferred) { 1942 // we no longer have the block, and will not respond, but a 1943 // packet was allocated in MSHR::handleSnoop and we have 1944 // to delete it 1945 assert(pkt->needsResponse()); 1946 1947 // we have passed the block to a cache upstream, that 1948 // cache should be responding 1949 assert(pkt->cacheResponding()); 1950 1951 delete pkt; 1952 } |
| 1953 return snoop_delay; 1954 } else { |
| 1955 DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__, 1956 pkt->print(), blk->print()); |
| 1957 } 1958 1959 chatty_assert(!(isReadOnly && blk->isDirty()), 1960 "Should never have a dirty block in a read-only cache %s\n", 1961 name()); 1962 1963 // We may end up modifying both the block state and the packet (if 1964 // we respond in atomic mode), so just figure out what to do now --- 5 unchanged lines hidden (view full) --- 1970 bool have_writable = blk->isWritable(); 1971 1972 // Invalidate any prefetch's from below that would strip write permissions 1973 // MemCmd::HardPFReq is only observed by upstream caches. After missing 1974 // above and in it's own cache, a new MemCmd::ReadReq is created that 1975 // downstream caches observe. 1976 if (pkt->mustCheckAbove()) { 1977 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s " |
| 1978 "from lower cache\n", pkt->getAddr(), pkt->print()); |
| 1979 pkt->setBlockCached(); 1980 return snoop_delay; 1981 } 1982 1983 if (pkt->isRead() && !invalidate) { 1984 // reading without requiring the line in a writable state 1985 assert(!needs_writable); 1986 pkt->setHasSharers(); --- 26 unchanged lines hidden (view full) --- 2013 // copy below and all other copies will be invalidates 2014 // through express snoops, and if needsWritable is not set 2015 // we already called setHasSharers above 2016 } 2017 2018 // if we are returning a writable and dirty (Modified) line, 2019 // we should be invalidating the line 2020 panic_if(!invalidate && !pkt->hasSharers(), |
| 2021 "%s is passing a Modified line through %s, " 2022 "but keeping the block", name(), pkt->print()); |
| 2023 2024 if (is_timing) { 2025 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval); 2026 } else { 2027 pkt->makeAtomicResponse(); 2028 // packets such as upgrades do not actually have any data 2029 // payload 2030 if (pkt->hasData()) --- 25 unchanged lines hidden (view full) --- 2056 2057 return snoop_delay; 2058} 2059 2060 2061void 2062Cache::recvTimingSnoopReq(PacketPtr pkt) 2063{ |
| 2064 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print()); |
| 2065 2066 // Snoops shouldn't happen when bypassing caches 2067 assert(!system->bypassCaches()); 2068 2069 // no need to snoop requests that are not in range 2070 if (!inRange(pkt->getAddr())) { 2071 return; 2072 } --- 10 unchanged lines hidden (view full) --- 2083 // tentative, for cases where we return before an upward snoop 2084 // happens below. 2085 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, 2086 lookupLatency * clockPeriod()); 2087 2088 // Inform request(Prefetch, CleanEvict or Writeback) from below of 2089 // MSHR hit, set setBlockCached. 2090 if (mshr && pkt->mustCheckAbove()) { |
| 2091 DPRINTF(Cache, "Setting block cached for %s from lower cache on " 2092 "mshr hit\n", pkt->print()); |
| 2093 pkt->setBlockCached(); 2094 return; 2095 } 2096 2097 // Let the MSHR itself track the snoop and decide whether we want 2098 // to go ahead and do the regular cache snoop 2099 if (mshr && mshr->handleSnoop(pkt, order++)) { 2100 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)." --- 20 unchanged lines hidden (view full) --- 2121 assert(wb_pkt->isEviction()); 2122 2123 if (pkt->isEviction()) { 2124 // if the block is found in the write queue, set the BLOCK_CACHED 2125 // flag for Writeback/CleanEvict snoop. On return the snoop will 2126 // propagate the BLOCK_CACHED flag in Writeback packets and prevent 2127 // any CleanEvicts from travelling down the memory hierarchy. 2128 pkt->setBlockCached(); |
| 2129 DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue " 2130 "hit\n", __func__, pkt->print()); |
| 2131 return; 2132 } 2133 2134 // conceptually writebacks are no different to other blocks in 2135 // this cache, so the behaviour is modelled after handleSnoop, 2136 // the difference being that instead of querying the block 2137 // state to determine if it is dirty and writable, we use the 2138 // command and fields of the writeback packet --- 195 unchanged lines hidden (view full) --- 2334bool 2335Cache::sendMSHRQueuePacket(MSHR* mshr) 2336{ 2337 assert(mshr); 2338 2339 // use request from 1st target 2340 PacketPtr tgt_pkt = mshr->getTarget()->pkt; 2341 |
| 2342 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print()); |
| 2343 2344 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure); 2345 2346 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) { 2347 // we should never have hardware prefetches to allocated 2348 // blocks 2349 assert(blk == nullptr); 2350 --- 101 unchanged lines hidden (view full) --- 2452bool 2453Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry) 2454{ 2455 assert(wq_entry); 2456 2457 // always a single target for write queue entries 2458 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt; 2459 |
| 2460 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print()); |
| 2461 2462 // forward as is, both for evictions and uncacheable writes 2463 if (!memSidePort->sendTimingReq(tgt_pkt)) { 2464 // note that we have now masked any requestBus and 2465 // schedSendEvent (we will wait for a retry before 2466 // doing anything), and this is so even if we do not 2467 // care about this packet and might override it before 2468 // it gets retried --- 185 unchanged lines hidden --- |