1/* 2 * Copyright (c) 2010-2015 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 143 unchanged lines hidden (view full) --- 152 153 assert(blk && blk->isValid()); 154 // Occasionally this is not true... if we are a lower-level cache 155 // satisfying a string of Read and ReadEx requests from 156 // upper-level caches, a Read will mark the block as shared but we 157 // can satisfy a following ReadEx anyway since we can rely on the 158 // Read requester(s) to have buffered the ReadEx snoop and to 159 // invalidate their blocks after receiving them. |
160 // assert(!pkt->needsWritable() || blk->isWritable()); |
161 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize); 162 163 // Check RMW operations first since both isRead() and 164 // isWrite() will be true for them 165 if (pkt->cmd == MemCmd::SwapReq) { 166 cmpAndSwap(blk, pkt); 167 } else if (pkt->isWrite()) { |
168 // we have the block in a writable state and can go ahead, 169 // note that the line may be also be considered writable in 170 // downstream caches along the path to memory, but always 171 // Exclusive, and never Modified |
172 assert(blk->isWritable()); |
173 // Write or WriteLine at the first cache with block in writable state |
174 if (blk->checkWrite(pkt)) { 175 pkt->writeDataToBlock(blk->data, blkSize); 176 } |
177 // Always mark the line as dirty (and thus transition to the 178 // Modified state) even if we are a failed StoreCond so we 179 // supply data to any snoops that have appended themselves to 180 // this cache before knowing the store will fail. |
181 blk->status |= BlkDirty; 182 DPRINTF(Cache, "%s for %s addr %#llx size %d (write)\n", __func__, 183 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 184 } else if (pkt->isRead()) { 185 if (pkt->isLLSC()) { 186 blk->trackLoadLocked(pkt); 187 } 188 pkt->setDataFromBlock(blk->data, blkSize); 189 // determine if this read is from a (coherent) cache, or not 190 // by looking at the command type; we could potentially add a 191 // packet attribute such as 'FromCache' to make this check a 192 // bit cleaner 193 if (pkt->cmd == MemCmd::ReadExReq || 194 pkt->cmd == MemCmd::ReadSharedReq || 195 pkt->cmd == MemCmd::ReadCleanReq || 196 pkt->cmd == MemCmd::SCUpgradeFailReq) { 197 assert(pkt->getSize() == blkSize); 198 // special handling for coherent block requests from 199 // upper-level caches |
200 if (pkt->needsWritable()) { |
201 // sanity check 202 assert(pkt->cmd == MemCmd::ReadExReq || 203 pkt->cmd == MemCmd::SCUpgradeFailReq); 204 205 // if we have a dirty copy, make sure the recipient |
206 // keeps it marked dirty (in the modified state) |
207 if (blk->isDirty()) { |
208 pkt->setCacheResponding(); |
209 } 210 // on ReadExReq we give up our copy unconditionally, 211 // even if this cache is mostly inclusive, we may want 212 // to revisit this 213 invalidateBlock(blk); 214 } else if (blk->isWritable() && !pending_downgrade && |
215 !pkt->hasSharers() && |
216 pkt->cmd != MemCmd::ReadCleanReq) { |
217 // we can give the requester a writable copy on a read 218 // request if: 219 // - we have a writable copy at this level (& below) |
220 // - we don't have a pending snoop from below 221 // signaling another read request 222 // - no other cache above has a copy (otherwise it |
223 // would have set hasSharers flag when 224 // snooping the packet) 225 // - the read has explicitly asked for a clean 226 // copy of the line |
227 if (blk->isDirty()) { 228 // special considerations if we're owner: 229 if (!deferred_response) { |
230 // respond with the line in Modified state 231 // (cacheResponding set, hasSharers not set) 232 pkt->setCacheResponding(); |
233 |
234 if (clusivity == Enums::mostly_excl) { |
235 // if this cache is mostly exclusive with 236 // respect to the cache above, drop the 237 // block, no need to first unset the dirty 238 // bit |
239 invalidateBlock(blk); |
240 } else { 241 // if this cache is mostly inclusive, we 242 // keep the block in the Exclusive state, 243 // and pass it upwards as Modified 244 // (writable and dirty), hence we have 245 // multiple caches, all on the same path 246 // towards memory, all considering the 247 // same block writable, but only one 248 // considering it Modified 249 250 // we get away with multiple caches (on 251 // the same path to memory) considering 252 // the block writeable as we always enter 253 // the cache hierarchy through a cache, 254 // and first snoop upwards in all other 255 // branches 256 blk->status &= ~BlkDirty; |
257 } 258 } else { 259 // if we're responding after our own miss, 260 // there's a window where the recipient didn't 261 // know it was getting ownership and may not 262 // have responded to snoops correctly, so we |
263 // have to respond with a shared line 264 pkt->setHasSharers(); |
265 } 266 } 267 } else { 268 // otherwise only respond with a shared copy |
269 pkt->setHasSharers(); |
270 } 271 } 272 } else { |
273 // Upgrade or Invalidate |
274 assert(pkt->isUpgrade() || pkt->isInvalidate()); 275 276 // for invalidations we could be looking at the temp block 277 // (for upgrades we always allocate) 278 invalidateBlock(blk); 279 DPRINTF(Cache, "%s for %s addr %#llx size %d (invalidation)\n", 280 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 281 } 282} 283 284 285///////////////////////////////////////////////////// 286// 287// MSHR helper functions 288// 289///////////////////////////////////////////////////// 290 291 292void |
293Cache::markInService(MSHR *mshr, bool pending_modified_resp) |
294{ |
295 markInServiceInternal(mshr, pending_modified_resp); |
296} 297 298///////////////////////////////////////////////////// 299// 300// Access path: requests coming in from the CPU side 301// 302///////////////////////////////////////////////////// 303 --- 116 unchanged lines hidden (view full) --- 420 blk->status |= BlkSecure; 421 } 422 } 423 // only mark the block dirty if we got a writeback command, 424 // and leave it as is for a clean writeback 425 if (pkt->cmd == MemCmd::WritebackDirty) { 426 blk->status |= BlkDirty; 427 } |
428 // if the packet does not have sharers, it is passing 429 // writable, and we got the writeback in Modified or Exclusive 430 // state, if not we are in the Owned or Shared state 431 if (!pkt->hasSharers()) { |
432 blk->status |= BlkWritable; 433 } 434 // nothing else to do; writeback doesn't expect response 435 assert(!pkt->needsResponse()); 436 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize); 437 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print()); 438 incHitCount(pkt); 439 return true; --- 6 unchanged lines hidden (view full) --- 446 return true; 447 } 448 // We didn't find the block here, propagate the CleanEvict further 449 // down the memory hierarchy. Returning false will treat the CleanEvict 450 // like a Writeback which could not find a replaceable block so has to 451 // go to next level. 452 return false; 453 } else if ((blk != NULL) && |
454 (pkt->needsWritable() ? blk->isWritable() : blk->isReadable())) { |
455 // OK to satisfy access 456 incHitCount(pkt); 457 satisfyCpuSideRequest(pkt, blk); 458 return true; 459 } 460 461 // Can't satisfy access normally... either no block (blk == NULL) |
462 // or have block but need writable |
463 464 incMissCount(pkt); 465 466 if (blk == NULL && pkt->isLLSC() && pkt->isWrite()) { 467 // complete miss on store conditional... just give up now 468 pkt->req->setExtraData(0); 469 return true; 470 } --- 136 unchanged lines hidden (view full) --- 607 // @todo This should really enqueue the packet rather 608 bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt); 609 assert(success); 610 return true; 611 } 612 613 promoteWholeLineWrites(pkt); 614 |
615 if (pkt->cacheResponding()) { |
616 // a cache above us (but not where the packet came from) is |
617 // responding to the request, in other words it has the line 618 // in Modified or Owned state 619 DPRINTF(Cache, "Cache above responding to %#llx (%s): " 620 "not responding\n", |
621 pkt->getAddr(), pkt->isSecure() ? "s" : "ns"); 622 |
623 // if the packet needs the block to be writable, and the cache 624 // that has promised to respond (setting the cache responding 625 // flag) is not providing writable (it is in Owned rather than 626 // the Modified state), we know that there may be other Shared 627 // copies in the system; go out and invalidate them all 628 if (pkt->needsWritable() && !pkt->responderHadWritable()) { 629 // an upstream cache that had the line in Owned state 630 // (dirty, but not writable), is responding and thus 631 // transferring the dirty line from one branch of the 632 // cache hierarchy to another 633 634 // send out an express snoop and invalidate all other 635 // copies (snooping a packet that needs writable is the 636 // same as an invalidation), thus turning the Owned line 637 // into a Modified line, note that we don't invalidate the 638 // block in the current cache or any other cache on the 639 // path to memory 640 |
641 // create a downstream express snoop with cleared packet 642 // flags, there is no need to allocate any data as the 643 // packet is merely used to co-ordinate state transitions 644 Packet *snoop_pkt = new Packet(pkt, true, false); 645 646 // also reset the bus time that the original packet has 647 // not yet paid for 648 snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0; 649 650 // make this an instantaneous express snoop, and let the |
651 // other caches in the system know that the another cache 652 // is responding, because we have found the authorative 653 // copy (Modified or Owned) that will supply the right 654 // data |
655 snoop_pkt->setExpressSnoop(); |
656 snoop_pkt->setCacheResponding(); |
657 658 // this express snoop travels towards the memory, and at 659 // every crossbar it is snooped upwards thus reaching 660 // every cache in the system 661 bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt); 662 // express snoops always succeed 663 assert(success); 664 |
665 // main memory will delete the snoop packet |
666 } 667 |
668 // queue for deletion, as opposed to immediate deletion, as 669 // the sending cache is still relying on the packet |
670 pendingDelete.reset(pkt); 671 |
672 // no need to take any action in this particular cache as an 673 // upstream cache has already committed to responding, and 674 // either the packet does not need writable (and we can let 675 // the cache that set the cache responding flag pass on the 676 // line without any need for intervention), or if the packet 677 // needs writable it is provided, or we have already sent out 678 // any express snoops in the section above |
679 return true; 680 } 681 682 // anything that is merely forwarded pays for the forward latency and 683 // the delay provided by the crossbar 684 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 685 686 // We use lookupLatency here because it is used to specify the latency --- 203 unchanged lines hidden (view full) --- 890 // that yet. Note that we do need to leave the 891 // block valid so that it stays in the cache, in 892 // case we get an upgrade response (and hence no 893 // new data) when the write miss completes. 894 // As long as CPUs do proper store/load forwarding 895 // internally, and have a sufficiently weak memory 896 // model, this is probably unnecessary, but at some 897 // point it must have seemed like we needed it... |
898 assert(pkt->needsWritable()); |
899 assert(!blk->isWritable()); 900 blk->status &= ~BlkReadable; 901 } 902 // Here we are using forward_time, modelling the latency of 903 // a miss (outbound) just as forwardLatency, neglecting the 904 // lookupLatency component. 905 allocateMissBuffer(pkt, forward_time); 906 } --- 11 unchanged lines hidden (view full) --- 918 919 return true; 920} 921 922 923// See comment in cache.hh. 924PacketPtr 925Cache::getBusPacket(PacketPtr cpu_pkt, CacheBlk *blk, |
926 bool needsWritable) const |
927{ 928 bool blkValid = blk && blk->isValid(); 929 930 if (cpu_pkt->req->isUncacheable()) { 931 // note that at the point we see the uncacheable request we 932 // flush any block, but there could be an outstanding MSHR, 933 // and the cache could have filled again before we actually 934 // send out the forwarded uncacheable request (blk could thus --- 14 unchanged lines hidden (view full) --- 949 950 MemCmd cmd; 951 // @TODO make useUpgrades a parameter. 952 // Note that ownership protocols require upgrade, otherwise a 953 // write miss on a shared owned block will generate a ReadExcl, 954 // which will clobber the owned copy. 955 const bool useUpgrades = true; 956 if (blkValid && useUpgrades) { |
957 // only reason to be here is that blk is read only and we need 958 // it to be writable 959 assert(needsWritable); |
960 assert(!blk->isWritable()); 961 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq; 962 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq || 963 cpu_pkt->cmd == MemCmd::StoreCondFailReq) { 964 // Even though this SC will fail, we still need to send out the 965 // request and get the data to supply it to other snoopers in the case 966 // where the determination the StoreCond fails is delayed due to 967 // all caches not being on the same local bus. 968 cmd = MemCmd::SCUpgradeFailReq; 969 } else if (cpu_pkt->cmd == MemCmd::WriteLineReq) { 970 // forward as invalidate to all other caches, this gives us |
971 // the line in Exclusive state, and invalidates all other |
972 // copies 973 cmd = MemCmd::InvalidateReq; 974 } else { 975 // block is invalid |
976 cmd = needsWritable ? MemCmd::ReadExReq : |
977 (isReadOnly ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq); 978 } 979 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize); 980 |
981 // if there are upstream caches that have already marked the 982 // packet as having sharers (not passing writable), pass that info 983 // downstream 984 if (cpu_pkt->hasSharers()) { |
985 // note that cpu_pkt may have spent a considerable time in the 986 // MSHR queue and that the information could possibly be out 987 // of date, however, there is no harm in conservatively |
988 // assuming the block has sharers 989 pkt->setHasSharers(); 990 DPRINTF(Cache, "%s passing hasSharers from %s to %s addr %#llx " 991 "size %d\n", |
992 __func__, cpu_pkt->cmdString(), pkt->cmdString(), 993 pkt->getAddr(), pkt->getSize()); 994 } 995 996 // the packet should be block aligned 997 assert(pkt->getAddr() == blockAlign(pkt->getAddr())); 998 999 pkt->allocate(); --- 13 unchanged lines hidden (view full) --- 1013 bool last_level_cache = false; 1014 1015 // Forward the request if the system is in cache bypass mode. 1016 if (system->bypassCaches()) 1017 return ticksToCycles(memSidePort->sendAtomic(pkt)); 1018 1019 promoteWholeLineWrites(pkt); 1020 |
1021 if (pkt->cacheResponding()) { |
1022 // have to invalidate ourselves and any lower caches even if 1023 // upper cache will be responding 1024 if (pkt->isInvalidate()) { 1025 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 1026 if (blk && blk->isValid()) { 1027 tags->invalidate(blk); 1028 blk->invalidate(); |
1029 DPRINTF(Cache, "Other cache responding to %s on %#llx (%s):" |
1030 " invalidating\n", 1031 pkt->cmdString(), pkt->getAddr(), 1032 pkt->isSecure() ? "s" : "ns"); 1033 } 1034 if (!last_level_cache) { |
1035 DPRINTF(Cache, "Other cache responding to %s on %#llx (%s):" 1036 " forwarding\n", |
1037 pkt->cmdString(), pkt->getAddr(), 1038 pkt->isSecure() ? "s" : "ns"); 1039 lat += ticksToCycles(memSidePort->sendAtomic(pkt)); 1040 } 1041 } else { |
1042 DPRINTF(Cache, "Other cache responding to %s on %#llx: " 1043 "not responding\n", |
1044 pkt->cmdString(), pkt->getAddr()); 1045 } 1046 1047 return lat * clockPeriod(); 1048 } 1049 1050 // should assert here that there are no outstanding MSHRs or 1051 // writebacks... that would mean that someone used an atomic --- 5 unchanged lines hidden (view full) --- 1057 1058 // handle writebacks resulting from the access here to ensure they 1059 // logically proceed anything happening below 1060 doWritebacksAtomic(writebacks); 1061 1062 if (!satisfied) { 1063 // MISS 1064 |
1065 PacketPtr bus_pkt = getBusPacket(pkt, blk, pkt->needsWritable()); |
1066 1067 bool is_forward = (bus_pkt == NULL); 1068 1069 if (is_forward) { 1070 // just forwarding the same request to the next level 1071 // no local cache operation involved 1072 bus_pkt = pkt; 1073 } --- 130 unchanged lines hidden (view full) --- 1204 // needs to be found. As a result we always update the request if 1205 // we have it, but only declare it satisfied if we are the owner. 1206 1207 // see if we have data at all (owned or otherwise) 1208 bool have_data = blk && blk->isValid() 1209 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize, 1210 blk->data); 1211 |
1212 // data we have is dirty if marked as such or if we have an 1213 // in-service MSHR that is pending a modified line |
1214 bool have_dirty = 1215 have_data && (blk->isDirty() || |
1216 (mshr && mshr->inService && mshr->isPendingModified())); |
1217 1218 bool done = have_dirty 1219 || cpuSidePort->checkFunctional(pkt) 1220 || mshrQueue.checkFunctional(pkt, blk_addr) 1221 || writeBuffer.checkFunctional(pkt, blk_addr) 1222 || memSidePort->checkFunctional(pkt); 1223 1224 DPRINTF(Cache, "functional %s %#llx (%s) %s%s%s\n", --- 79 unchanged lines hidden (view full) --- 1304 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] += 1305 miss_latency; 1306 } else { 1307 assert(pkt->req->masterId() < system->maxMasters()); 1308 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] += 1309 miss_latency; 1310 } 1311 |
1312 // upgrade deferred targets if the response has no sharers, and is 1313 // thus passing writable 1314 if (!pkt->hasSharers()) { 1315 mshr->promoteWritable(); |
1316 } 1317 1318 bool is_fill = !mshr->isForward && 1319 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp); 1320 1321 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure()); 1322 1323 if (is_fill && !is_error) { --- 35 unchanged lines hidden (view full) --- 1359 // unlike the other packet flows, where data is found in other 1360 // caches or memory and brought back, write-line requests always 1361 // have the data right away, so the above check for "is fill?" 1362 // cannot actually be determined until examining the stored MSHR 1363 // state. We "catch up" with that logic here, which is duplicated 1364 // from above. 1365 if (tgt_pkt->cmd == MemCmd::WriteLineReq) { 1366 assert(!is_error); |
1367 // we got the block in a writable state, so promote 1368 // any deferred targets if possible 1369 mshr->promoteWritable(); |
1370 // NB: we use the original packet here and not the response! 1371 blk = handleFill(tgt_pkt, blk, writebacks, mshr->allocOnFill); 1372 assert(blk != NULL); 1373 1374 // treat as a fill, and discard the invalidation 1375 // response 1376 is_fill = true; 1377 is_invalidate = false; --- 184 unchanged lines hidden (view full) --- 1562 DPRINTF(Cache, "Create Writeback %#llx writable: %d, dirty: %d\n", 1563 pkt->getAddr(), blk->isWritable(), blk->isDirty()); 1564 1565 if (blk->isWritable()) { 1566 // not asserting shared means we pass the block in modified 1567 // state, mark our own block non-writeable 1568 blk->status &= ~BlkWritable; 1569 } else { |
1570 // we are in the Owned state, tell the receiver 1571 pkt->setHasSharers(); |
1572 } 1573 1574 // make sure the block is not marked dirty 1575 blk->status &= ~BlkDirty; 1576 1577 pkt->allocate(); 1578 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize); 1579 --- 96 unchanged lines hidden (view full) --- 1676 1677 if (blk->isValid()) { 1678 Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set); 1679 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure()); 1680 if (repl_mshr) { 1681 // must be an outstanding upgrade request 1682 // on a block we're about to replace... 1683 assert(!blk->isWritable() || blk->isDirty()); |
1684 assert(repl_mshr->needsWritable()); |
1685 // too hard to replace block with transient state 1686 // allocation failed, block not inserted 1687 return NULL; 1688 } else { 1689 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx (%s): %s\n", 1690 repl_addr, blk->isSecure() ? "s" : "ns", 1691 addr, is_secure ? "s" : "ns", 1692 blk->isDirty() ? "writeback" : "clean"); --- 84 unchanged lines hidden (view full) --- 1777 if (is_secure) 1778 blk->status |= BlkSecure; 1779 blk->status |= BlkValid | BlkReadable; 1780 1781 // sanity check for whole-line writes, which should always be 1782 // marked as writable as part of the fill, and then later marked 1783 // dirty as part of satisfyCpuSideRequest 1784 if (pkt->cmd == MemCmd::WriteLineReq) { |
1785 assert(!pkt->hasSharers()); |
1786 // at the moment other caches do not respond to the 1787 // invalidation requests corresponding to a whole-line write |
1788 assert(!pkt->cacheResponding()); |
1789 } 1790 |
1791 // here we deal with setting the appropriate state of the line, 1792 // and we start by looking at the hasSharers flag, and ignore the 1793 // cacheResponding flag (normally signalling dirty data) if the 1794 // packet has sharers, thus the line is never allocated as Owned 1795 // (dirty but not writable), and always ends up being either 1796 // Shared, Exclusive or Modified, see Packet::setCacheResponding 1797 // for more details 1798 if (!pkt->hasSharers()) { 1799 // we could get a writable line from memory (rather than a 1800 // cache) even in a read-only cache, note that we set this bit 1801 // even for a read-only cache, possibly revisit this decision |
1802 blk->status |= BlkWritable; 1803 |
1804 // check if we got this via cache-to-cache transfer (i.e., from a 1805 // cache that had the block in Modified or Owned state) 1806 if (pkt->cacheResponding()) { 1807 // we got the block in Modified state, and invalidated the 1808 // owners copy |
1809 blk->status |= BlkDirty; 1810 1811 chatty_assert(!isReadOnly, "Should never see dirty snoop response " 1812 "in read-only cache %s\n", name()); 1813 } 1814 } 1815 1816 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n", --- 37 unchanged lines hidden (view full) --- 1854 PacketPtr pkt = req_pkt; 1855 if (!already_copied) 1856 // do not clear flags, and allocate space for data if the 1857 // packet needs it (the only packets that carry data are read 1858 // responses) 1859 pkt = new Packet(req_pkt, false, req_pkt->isRead()); 1860 1861 assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() || |
1862 pkt->hasSharers()); |
1863 pkt->makeTimingResponse(); 1864 if (pkt->isRead()) { 1865 pkt->setDataFromBlock(blk_data, blkSize); 1866 } 1867 if (pkt->cmd == MemCmd::ReadResp && pending_inval) { 1868 // Assume we defer a response to a read from a far-away cache 1869 // A, then later defer a ReadExcl from a cache B on the same |
1870 // bus as us. We'll assert cacheResponding in both cases, but 1871 // in the latter case cacheResponding will keep the 1872 // invalidation from reaching cache A. This special response 1873 // tells cache A that it gets the block to satisfy its read, 1874 // but must immediately invalidate it. |
1875 pkt->cmd = MemCmd::ReadRespWithInvalidate; 1876 } 1877 // Here we consider forward_time, paying for just forward latency and 1878 // also charging the delay provided by the xbar. 1879 // forward_time is used as send_time in next allocateWriteBuffer(). 1880 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay; 1881 // Here we reset the timing of the packet. 1882 pkt->headerDelay = pkt->payloadDelay = 0; --- 14 unchanged lines hidden (view full) --- 1897 // pending_inval only makes sense on deferred snoops 1898 assert(!(pending_inval && !is_deferred)); 1899 assert(pkt->isRequest()); 1900 1901 // the packet may get modified if we or a forwarded snooper 1902 // responds in atomic mode, so remember a few things about the 1903 // original packet up front 1904 bool invalidate = pkt->isInvalidate(); |
1905 bool M5_VAR_USED needs_writable = pkt->needsWritable(); |
1906 1907 uint32_t snoop_delay = 0; 1908 1909 if (forwardSnoops) { 1910 // first propagate snoop upward to see if anyone above us wants to 1911 // handle it. save & restore packet src since it will get 1912 // rewritten to be relative to cpu-side bus (if any) |
1913 bool alreadyResponded = pkt->cacheResponding(); |
1914 if (is_timing) { 1915 // copy the packet so that we can clear any flags before 1916 // forwarding it upwards, we also allocate data (passing 1917 // the pointer along in case of static data), in case 1918 // there is a snoop hit in upper levels 1919 Packet snoopPkt(pkt, true, true); 1920 snoopPkt.setExpressSnoop(); 1921 // the snoop packet does not need to wait any additional 1922 // time 1923 snoopPkt.headerDelay = snoopPkt.payloadDelay = 0; 1924 cpuSidePort->sendTimingSnoopReq(&snoopPkt); 1925 1926 // add the header delay (including crossbar and snoop 1927 // delays) of the upward snoop to the snoop delay for this 1928 // cache 1929 snoop_delay += snoopPkt.headerDelay; 1930 |
1931 if (snoopPkt.cacheResponding()) { |
1932 // cache-to-cache response from some upper cache 1933 assert(!alreadyResponded); |
1934 pkt->setCacheResponding(); |
1935 } |
1936 // upstream cache has the block, or has an outstanding 1937 // MSHR, pass the flag on 1938 if (snoopPkt.hasSharers()) { 1939 pkt->setHasSharers(); |
1940 } 1941 // If this request is a prefetch or clean evict and an upper level 1942 // signals block present, make sure to propagate the block 1943 // presence to the requester. 1944 if (snoopPkt.isBlockCached()) { 1945 pkt->setBlockCached(); 1946 } 1947 } else { 1948 cpuSidePort->sendAtomicSnoop(pkt); |
1949 if (!alreadyResponded && pkt->cacheResponding()) { |
1950 // cache-to-cache response from some upper cache: 1951 // forward response to original requester 1952 assert(pkt->isResponse()); 1953 } 1954 } 1955 } 1956 1957 if (!blk || !blk->isValid()) { --- 12 unchanged lines hidden (view full) --- 1970 1971 // We may end up modifying both the block state and the packet (if 1972 // we respond in atomic mode), so just figure out what to do now 1973 // and then do it later. If we find dirty data while snooping for 1974 // an invalidate, we don't need to send a response. The 1975 // invalidation itself is taken care of below. 1976 bool respond = blk->isDirty() && pkt->needsResponse() && 1977 pkt->cmd != MemCmd::InvalidateReq; |
1978 bool have_writable = blk->isWritable(); |
1979 1980 // Invalidate any prefetch's from below that would strip write permissions 1981 // MemCmd::HardPFReq is only observed by upstream caches. After missing 1982 // above and in it's own cache, a new MemCmd::ReadReq is created that 1983 // downstream caches observe. 1984 if (pkt->mustCheckAbove()) { 1985 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s from" 1986 " lower cache\n", pkt->getAddr(), pkt->cmdString()); 1987 pkt->setBlockCached(); 1988 return snoop_delay; 1989 } 1990 1991 if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) { |
1992 // reading without requiring the line in a writable state, 1993 // note that we retain the block as Owned if it is Modified 1994 // (dirty data), with the response taken care of below, and 1995 // otherwhise simply downgrade from Exclusive to Shared (or 1996 // remain in Shared) 1997 assert(!needs_writable); 1998 pkt->setHasSharers(); |
1999 blk->status &= ~BlkWritable; 2000 } 2001 2002 if (respond) { 2003 // prevent anyone else from responding, cache as well as 2004 // memory, and also prevent any memory from even seeing the |
2005 // request 2006 pkt->setCacheResponding(); 2007 if (have_writable) { 2008 // inform the cache hierarchy that this cache had the line 2009 // in the Modified state so that we avoid unnecessary 2010 // invalidations (see Packet::setResponderHadWritable) 2011 pkt->setResponderHadWritable(); 2012 |
2013 // in the case of an uncacheable request there is no point |
2014 // in setting the responderHadWritable flag, but since the 2015 // recipient does not care there is no harm in doing so 2016 } else { 2017 // if the packet has needsWritable set we invalidate our 2018 // copy below and all other copies will be invalidates 2019 // through express snoops, and if needsWritable is not set 2020 // we already called setHasSharers above |
2021 } |
2022 |
2023 if (is_timing) { 2024 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval); 2025 } else { 2026 pkt->makeAtomicResponse(); 2027 pkt->setDataFromBlock(blk->data, blkSize); 2028 } 2029 } 2030 --- 94 unchanged lines hidden (view full) --- 2125 // any CleanEvicts from travelling down the memory hierarchy. 2126 pkt->setBlockCached(); 2127 DPRINTF(Cache, "Squashing %s from lower cache on writequeue hit" 2128 " %#x\n", pkt->cmdString(), pkt->getAddr()); 2129 return; 2130 } 2131 2132 if (wb_pkt->cmd == MemCmd::WritebackDirty) { |
2133 // we have dirty data, and so will proceed to respond 2134 pkt->setCacheResponding(); 2135 if (!pkt->needsWritable()) { 2136 // the packet should end up in the Shared state (non 2137 // writable) on the completion of the fill 2138 pkt->setHasSharers(); 2139 // similarly, the writeback is no longer passing 2140 // writeable (the receiving cache should consider the 2141 // block Owned rather than Modified) 2142 wb_pkt->setHasSharers(); |
2143 } else { |
2144 // we need to invalidate our copy. we do that 2145 // below. |
2146 assert(pkt->isInvalidate()); 2147 } 2148 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(), 2149 false, false); 2150 } else { 2151 // on hitting a clean writeback we play it safe and do not 2152 // provide a response, the block may be dirty somewhere 2153 // else 2154 assert(wb_pkt->isCleanEviction()); 2155 // The cache technically holds the block until the 2156 // corresponding message reaches the crossbar 2157 // below. Therefore when a snoop encounters a CleanEvict |
2158 // or WritebackClean message we must call 2159 // setHasSharers (just like when it encounters a 2160 // Writeback) to avoid the snoop filter prematurely 2161 // clearing the holder bit in the crossbar below 2162 if (!pkt->needsWritable()) { 2163 pkt->setHasSharers(); 2164 // the writeback is no longer passing writeable (the 2165 // receiving cache should consider the block Owned 2166 // rather than Modified) 2167 wb_pkt->setHasSharers(); |
2168 } else { 2169 assert(pkt->isInvalidate()); 2170 } 2171 } 2172 2173 if (pkt->isInvalidate()) { 2174 // Invalidation trumps our writeback... discard here 2175 // Note: markInService will remove entry from writeback buffer. --- 140 unchanged lines hidden (view full) --- 2316 snoop_pkt.setExpressSnoop(); 2317 // Assert that packet is either Writeback or CleanEvict and not a 2318 // prefetch request because prefetch requests need an MSHR and may 2319 // generate a snoop response. 2320 assert(pkt->isEviction()); 2321 snoop_pkt.senderState = NULL; 2322 cpuSidePort->sendTimingSnoopReq(&snoop_pkt); 2323 // Writeback/CleanEvict snoops do not generate a snoop response. |
2324 assert(!(snoop_pkt.cacheResponding())); |
2325 return snoop_pkt.isBlockCached(); 2326 } else { 2327 cpuSidePort->sendAtomicSnoop(pkt); 2328 return pkt->isBlockCached(); 2329 } 2330} 2331 2332PacketPtr --- 30 unchanged lines hidden (view full) --- 2363 cpuSidePort->sendTimingSnoopReq(&snoop_pkt); 2364 2365 // Check to see if the prefetch was squashed by an upper cache (to 2366 // prevent us from grabbing the line) or if a Check to see if a 2367 // writeback arrived between the time the prefetch was placed in 2368 // the MSHRs and when it was selected to be sent or if the 2369 // prefetch was squashed by an upper cache. 2370 |
2371 // It is important to check cacheResponding before 2372 // prefetchSquashed. If another cache has committed to 2373 // responding, it will be sending a dirty response which will 2374 // arrive at the MSHR allocated for this request. Checking the 2375 // prefetchSquash first may result in the MSHR being 2376 // prematurely deallocated. 2377 if (snoop_pkt.cacheResponding()) { |
2378 auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req); 2379 assert(r.second); |
2380 2381 // if we are getting a snoop response with no sharers it 2382 // will be allocated as Modified 2383 bool pending_modified_resp = !snoop_pkt.hasSharers(); 2384 markInService(mshr, pending_modified_resp); 2385 |
2386 DPRINTF(Cache, "Upward snoop of prefetch for addr" 2387 " %#x (%s) hit\n", 2388 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns"); 2389 return NULL; 2390 } 2391 2392 if (snoop_pkt.isBlockCached() || blk != NULL) { 2393 DPRINTF(Cache, "Block present, prefetch squashed by cache. " --- 9 unchanged lines hidden (view full) --- 2403 } 2404 } 2405 2406 if (mshr->isForwardNoResponse()) { 2407 // no response expected, just forward packet as it is 2408 assert(tags->findBlock(mshr->blkAddr, mshr->isSecure) == NULL); 2409 pkt = tgt_pkt; 2410 } else { |
2411 pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable()); |
2412 2413 mshr->isForward = (pkt == NULL); 2414 2415 if (mshr->isForward) { 2416 // not a cache block request, but a response is expected 2417 // make copy of current packet to forward, keep current 2418 // copy for response handling 2419 pkt = new Packet(tgt_pkt, false, true); --- 73 unchanged lines hidden (view full) --- 2493 2494bool 2495Cache::CpuSidePort::recvTimingReq(PacketPtr pkt) 2496{ 2497 assert(!cache->system->bypassCaches()); 2498 2499 bool success = false; 2500 |
2501 // always let packets through if an upstream cache has committed 2502 // to responding, even if blocked (we should technically look at 2503 // the isExpressSnoop flag, but it is set by the cache itself, and 2504 // consequently we have to rely on the cacheResponding flag) 2505 if (pkt->cacheResponding()) { |
2506 // do not change the current retry state 2507 bool M5_VAR_USED bypass_success = cache->recvTimingReq(pkt); 2508 assert(bypass_success); 2509 return true; 2510 } else if (blocked || mustSendRetry) { 2511 // either already committed to send a retry, or blocked 2512 success = false; 2513 } else { --- 123 unchanged lines hidden (view full) --- 2637 } 2638 // note that we have now masked any requestBus and 2639 // schedSendEvent (we will wait for a retry before 2640 // doing anything), and this is so even if we do not 2641 // care about this packet and might override it before 2642 // it gets retried 2643 } else { 2644 // As part of the call to sendTimingReq the packet is |
2645 // forwarded to all neighbouring caches (and any caches 2646 // above them) as a snoop. Thus at this point we know if 2647 // any of the neighbouring caches are responding, and if 2648 // so, we know it is dirty, and we can determine if it is 2649 // being passed as Modified, making our MSHR the ordering 2650 // point 2651 bool pending_modified_resp = !pkt->hasSharers() && 2652 pkt->cacheResponding(); |
2653 |
2654 cache.markInService(mshr, pending_modified_resp); |
2655 } 2656 } 2657 2658 // if we succeeded and are not waiting for a retry, schedule the 2659 // next send considering when the next MSHR is ready, note that 2660 // snoop responses have their own packet queue and thus schedule 2661 // their own events 2662 if (!waitingOnRetry) { --- 12 unchanged lines hidden --- |