cache.cc (11194:c3ba89c653a9) cache.cc (11197:f8fdd931e674)
1/*
2 * Copyright (c) 2010-2015 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 54 unchanged lines hidden (view full) ---

63#include "mem/cache/prefetch/base.hh"
64#include "sim/sim_exit.hh"
65
66Cache::Cache(const CacheParams *p)
67 : BaseCache(p, p->system->cacheLineSize()),
68 tags(p->tags),
69 prefetcher(p->prefetcher),
70 doFastWrites(true),
1/*
2 * Copyright (c) 2010-2015 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 54 unchanged lines hidden (view full) ---

63#include "mem/cache/prefetch/base.hh"
64#include "sim/sim_exit.hh"
65
66Cache::Cache(const CacheParams *p)
67 : BaseCache(p, p->system->cacheLineSize()),
68 tags(p->tags),
69 prefetcher(p->prefetcher),
70 doFastWrites(true),
71 prefetchOnAccess(p->prefetch_on_access)
71 prefetchOnAccess(p->prefetch_on_access),
72 clusivity(p->clusivity),
73 tempBlockWriteback(nullptr),
74 writebackTempBlockAtomicEvent(this, false,
75 EventBase::Delayed_Writeback_Pri)
72{
73 tempBlock = new CacheBlk();
74 tempBlock->data = new uint8_t[blkSize];
75
76 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
77 "CpuSidePort");
78 memSidePort = new MemSidePort(p->name + ".mem_side", this,
79 "MemSidePort");

--- 113 unchanged lines hidden (view full) ---

193 assert(pkt->cmd == MemCmd::ReadExReq ||
194 pkt->cmd == MemCmd::SCUpgradeFailReq);
195
196 // if we have a dirty copy, make sure the recipient
197 // keeps it marked dirty
198 if (blk->isDirty()) {
199 pkt->assertMemInhibit();
200 }
76{
77 tempBlock = new CacheBlk();
78 tempBlock->data = new uint8_t[blkSize];
79
80 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
81 "CpuSidePort");
82 memSidePort = new MemSidePort(p->name + ".mem_side", this,
83 "MemSidePort");

--- 113 unchanged lines hidden (view full) ---

197 assert(pkt->cmd == MemCmd::ReadExReq ||
198 pkt->cmd == MemCmd::SCUpgradeFailReq);
199
200 // if we have a dirty copy, make sure the recipient
201 // keeps it marked dirty
202 if (blk->isDirty()) {
203 pkt->assertMemInhibit();
204 }
201 // on ReadExReq we give up our copy unconditionally
202 if (blk != tempBlock)
203 tags->invalidate(blk);
204 blk->invalidate();
205 // on ReadExReq we give up our copy unconditionally,
206 // even if this cache is mostly inclusive, we may want
207 // to revisit this
208 invalidateBlock(blk);
205 } else if (blk->isWritable() && !pending_downgrade &&
206 !pkt->sharedAsserted() &&
207 pkt->cmd != MemCmd::ReadCleanReq) {
208 // we can give the requester an exclusive copy (by not
209 // asserting shared line) on a read request if:
210 // - we have an exclusive copy at this level (& below)
211 // - we don't have a pending snoop from below
212 // signaling another read request
213 // - no other cache above has a copy (otherwise it
214 // would have asseretd shared line on request)
215 // - we are not satisfying an instruction fetch (this
216 // prevents dirty data in the i-cache)
217
218 if (blk->isDirty()) {
219 // special considerations if we're owner:
220 if (!deferred_response) {
221 // if we are responding immediately and can
222 // signal that we're transferring ownership
209 } else if (blk->isWritable() && !pending_downgrade &&
210 !pkt->sharedAsserted() &&
211 pkt->cmd != MemCmd::ReadCleanReq) {
212 // we can give the requester an exclusive copy (by not
213 // asserting shared line) on a read request if:
214 // - we have an exclusive copy at this level (& below)
215 // - we don't have a pending snoop from below
216 // signaling another read request
217 // - no other cache above has a copy (otherwise it
218 // would have asseretd shared line on request)
219 // - we are not satisfying an instruction fetch (this
220 // prevents dirty data in the i-cache)
221
222 if (blk->isDirty()) {
223 // special considerations if we're owner:
224 if (!deferred_response) {
225 // if we are responding immediately and can
226 // signal that we're transferring ownership
223 // along with exclusivity, do so
227 // (inhibit set) along with exclusivity
228 // (shared not set), do so
224 pkt->assertMemInhibit();
229 pkt->assertMemInhibit();
230
231 // if this cache is mostly inclusive, we keep
232 // the block as writable (exclusive), and pass
233 // it upwards as writable and dirty
234 // (modified), hence we have multiple caches
235 // considering the same block writable,
236 // something that we get away with due to the
237 // fact that: 1) this cache has been
238 // considered the ordering points and
239 // responded to all snoops up till now, and 2)
240 // we always snoop upwards before consulting
241 // the local cache, both on a normal request
242 // (snooping done by the crossbar), and on a
243 // snoop
225 blk->status &= ~BlkDirty;
244 blk->status &= ~BlkDirty;
245
246 // if this cache is mostly exclusive with
247 // respect to the cache above, drop the block
248 if (clusivity == Enums::mostly_excl) {
249 invalidateBlock(blk);
250 }
226 } else {
227 // if we're responding after our own miss,
228 // there's a window where the recipient didn't
229 // know it was getting ownership and may not
230 // have responded to snoops correctly, so we
231 // can't pass off ownership *or* exclusivity
232 pkt->assertShared();
233 }
234 }
235 } else {
236 // otherwise only respond with a shared copy
237 pkt->assertShared();
238 }
239 }
240 } else {
241 // Upgrade or Invalidate, since we have it Exclusively (E or
242 // M), we ack then invalidate.
243 assert(pkt->isUpgrade() || pkt->isInvalidate());
251 } else {
252 // if we're responding after our own miss,
253 // there's a window where the recipient didn't
254 // know it was getting ownership and may not
255 // have responded to snoops correctly, so we
256 // can't pass off ownership *or* exclusivity
257 pkt->assertShared();
258 }
259 }
260 } else {
261 // otherwise only respond with a shared copy
262 pkt->assertShared();
263 }
264 }
265 } else {
266 // Upgrade or Invalidate, since we have it Exclusively (E or
267 // M), we ack then invalidate.
268 assert(pkt->isUpgrade() || pkt->isInvalidate());
244 assert(blk != tempBlock);
245 tags->invalidate(blk);
246 blk->invalidate();
269
270 // for invalidations we could be looking at the temp block
271 // (for upgrades we always allocate)
272 invalidateBlock(blk);
247 DPRINTF(Cache, "%s for %s addr %#llx size %d (invalidation)\n",
248 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
249 }
250}
251
252
253/////////////////////////////////////////////////////
254//

--- 501 unchanged lines hidden (view full) ---

756 }
757 // We use forward_time here because it is the same
758 // considering new targets. We have multiple
759 // requests for the same address here. It
760 // specifies the latency to allocate an internal
761 // buffer and to schedule an event to the queued
762 // port and also takes into account the additional
763 // delay of the xbar.
273 DPRINTF(Cache, "%s for %s addr %#llx size %d (invalidation)\n",
274 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
275 }
276}
277
278
279/////////////////////////////////////////////////////
280//

--- 501 unchanged lines hidden (view full) ---

782 }
783 // We use forward_time here because it is the same
784 // considering new targets. We have multiple
785 // requests for the same address here. It
786 // specifies the latency to allocate an internal
787 // buffer and to schedule an event to the queued
788 // port and also takes into account the additional
789 // delay of the xbar.
764 mshr->allocateTarget(pkt, forward_time, order++);
790 mshr->allocateTarget(pkt, forward_time, order++,
791 allocOnFill(pkt->cmd));
765 if (mshr->getNumTargets() == numTarget) {
766 noTargetMSHR = mshr;
767 setBlocked(Blocked_NoTargets);
768 // need to be careful with this... if this mshr isn't
769 // ready yet (i.e. time > curTick()), we don't want to
770 // move it ahead of mshrs that are ready
771 // mshrQueue.moveToFront(mshr);
772 }

--- 249 unchanged lines hidden (view full) ---

1022 // an invalidate request
1023 satisfyCpuSideRequest(pkt, blk);
1024 }
1025 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1026 // note the use of pkt, not bus_pkt here.
1027
1028 // write-line request to the cache that promoted
1029 // the write to a whole line
792 if (mshr->getNumTargets() == numTarget) {
793 noTargetMSHR = mshr;
794 setBlocked(Blocked_NoTargets);
795 // need to be careful with this... if this mshr isn't
796 // ready yet (i.e. time > curTick()), we don't want to
797 // move it ahead of mshrs that are ready
798 // mshrQueue.moveToFront(mshr);
799 }

--- 249 unchanged lines hidden (view full) ---

1049 // an invalidate request
1050 satisfyCpuSideRequest(pkt, blk);
1051 }
1052 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1053 // note the use of pkt, not bus_pkt here.
1054
1055 // write-line request to the cache that promoted
1056 // the write to a whole line
1030 blk = handleFill(pkt, blk, writebacks);
1057 blk = handleFill(pkt, blk, writebacks,
1058 allocOnFill(pkt->cmd));
1031 satisfyCpuSideRequest(pkt, blk);
1032 } else if (bus_pkt->isRead() ||
1033 bus_pkt->cmd == MemCmd::UpgradeResp) {
1034 // we're updating cache state to allow us to
1035 // satisfy the upstream request from the cache
1059 satisfyCpuSideRequest(pkt, blk);
1060 } else if (bus_pkt->isRead() ||
1061 bus_pkt->cmd == MemCmd::UpgradeResp) {
1062 // we're updating cache state to allow us to
1063 // satisfy the upstream request from the cache
1036 blk = handleFill(bus_pkt, blk, writebacks);
1064 blk = handleFill(bus_pkt, blk, writebacks,
1065 allocOnFill(pkt->cmd));
1037 satisfyCpuSideRequest(pkt, blk);
1038 } else {
1039 // we're satisfying the upstream request without
1040 // modifying cache state, e.g., a write-through
1041 pkt->makeAtomicResponse();
1042 }
1043 }
1044 delete bus_pkt;

--- 6 unchanged lines hidden (view full) ---

1051 // rely on bandwidth contention to throttle them; these will tend
1052 // to pollute the cache in atomic mode since there is no bandwidth
1053 // contention. If we ever do want to enable prefetching in atomic
1054 // mode, though, this is the place to do it... see timingAccess()
1055 // for an example (though we'd want to issue the prefetch(es)
1056 // immediately rather than calling requestMemSideBus() as we do
1057 // there).
1058
1066 satisfyCpuSideRequest(pkt, blk);
1067 } else {
1068 // we're satisfying the upstream request without
1069 // modifying cache state, e.g., a write-through
1070 pkt->makeAtomicResponse();
1071 }
1072 }
1073 delete bus_pkt;

--- 6 unchanged lines hidden (view full) ---

1080 // rely on bandwidth contention to throttle them; these will tend
1081 // to pollute the cache in atomic mode since there is no bandwidth
1082 // contention. If we ever do want to enable prefetching in atomic
1083 // mode, though, this is the place to do it... see timingAccess()
1084 // for an example (though we'd want to issue the prefetch(es)
1085 // immediately rather than calling requestMemSideBus() as we do
1086 // there).
1087
1059 // Handle writebacks (from the response handling) if needed
1088 // do any writebacks resulting from the response handling
1060 doWritebacksAtomic(writebacks);
1061
1089 doWritebacksAtomic(writebacks);
1090
1091 // if we used temp block, check to see if its valid and if so
1092 // clear it out, but only do so after the call to recvAtomic is
1093 // finished so that any downstream observers (such as a snoop
1094 // filter), first see the fill, and only then see the eviction
1095 if (blk == tempBlock && tempBlock->isValid()) {
1096 // the atomic CPU calls recvAtomic for fetch and load/store
1097 // sequentuially, and we may already have a tempBlock
1098 // writeback from the fetch that we have not yet sent
1099 if (tempBlockWriteback) {
1100 // if that is the case, write the prevoius one back, and
1101 // do not schedule any new event
1102 writebackTempBlockAtomic();
1103 } else {
1104 // the writeback/clean eviction happens after the call to
1105 // recvAtomic has finished (but before any successive
1106 // calls), so that the response handling from the fill is
1107 // allowed to happen first
1108 schedule(writebackTempBlockAtomicEvent, curTick());
1109 }
1110
1111 tempBlockWriteback = blk->isDirty() ? writebackBlk(blk) :
1112 cleanEvictBlk(blk);
1113 blk->invalidate();
1114 }
1115
1062 if (pkt->needsResponse()) {
1063 pkt->makeAtomicResponse();
1064 }
1065
1066 return lat * clockPeriod();
1067}
1068
1069

--- 139 unchanged lines hidden (view full) ---

1209 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1210
1211 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1212
1213 if (is_fill && !is_error) {
1214 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1215 pkt->getAddr());
1216
1116 if (pkt->needsResponse()) {
1117 pkt->makeAtomicResponse();
1118 }
1119
1120 return lat * clockPeriod();
1121}
1122
1123

--- 139 unchanged lines hidden (view full) ---

1263 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1264
1265 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1266
1267 if (is_fill && !is_error) {
1268 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1269 pkt->getAddr());
1270
1217 blk = handleFill(pkt, blk, writebacks);
1271 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill);
1218 assert(blk != NULL);
1219 }
1220
1221 // allow invalidation responses originating from write-line
1222 // requests to be discarded
1223 bool is_invalidate = pkt->isInvalidate();
1224
1225 // First offset for critical word first calculations

--- 27 unchanged lines hidden (view full) ---

1253 // state. We "catch up" with that logic here, which is duplicated
1254 // from above.
1255 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1256 assert(!is_error);
1257 // we got the block in exclusive state, so promote any
1258 // deferred targets if possible
1259 mshr->promoteExclusive();
1260 // NB: we use the original packet here and not the response!
1272 assert(blk != NULL);
1273 }
1274
1275 // allow invalidation responses originating from write-line
1276 // requests to be discarded
1277 bool is_invalidate = pkt->isInvalidate();
1278
1279 // First offset for critical word first calculations

--- 27 unchanged lines hidden (view full) ---

1307 // state. We "catch up" with that logic here, which is duplicated
1308 // from above.
1309 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1310 assert(!is_error);
1311 // we got the block in exclusive state, so promote any
1312 // deferred targets if possible
1313 mshr->promoteExclusive();
1314 // NB: we use the original packet here and not the response!
1261 blk = handleFill(tgt_pkt, blk, writebacks);
1315 blk = handleFill(tgt_pkt, blk, writebacks, mshr->allocOnFill);
1262 assert(blk != NULL);
1263
1264 // treat as a fill, and discard the invalidation
1265 // response
1266 is_fill = true;
1267 is_invalidate = false;
1268 }
1269

--- 87 unchanged lines hidden (view full) ---

1357 mshr->popTarget();
1358 }
1359
1360 if (blk && blk->isValid()) {
1361 // an invalidate response stemming from a write line request
1362 // should not invalidate the block, so check if the
1363 // invalidation should be discarded
1364 if (is_invalidate || mshr->hasPostInvalidate()) {
1316 assert(blk != NULL);
1317
1318 // treat as a fill, and discard the invalidation
1319 // response
1320 is_fill = true;
1321 is_invalidate = false;
1322 }
1323

--- 87 unchanged lines hidden (view full) ---

1411 mshr->popTarget();
1412 }
1413
1414 if (blk && blk->isValid()) {
1415 // an invalidate response stemming from a write line request
1416 // should not invalidate the block, so check if the
1417 // invalidation should be discarded
1418 if (is_invalidate || mshr->hasPostInvalidate()) {
1365 assert(blk != tempBlock);
1366 tags->invalidate(blk);
1367 blk->invalidate();
1419 invalidateBlock(blk);
1368 } else if (mshr->hasPostDowngrade()) {
1369 blk->status &= ~BlkWritable;
1370 }
1371 }
1372
1373 if (mshr->promoteDeferredTargets()) {
1374 // avoid later read getting stale data while write miss is
1375 // outstanding.. see comment in timingAccess()

--- 207 unchanged lines hidden (view full) ---

1583 writebacks.push_back(cleanEvictBlk(blk));
1584 }
1585 }
1586 }
1587
1588 return blk;
1589}
1590
1420 } else if (mshr->hasPostDowngrade()) {
1421 blk->status &= ~BlkWritable;
1422 }
1423 }
1424
1425 if (mshr->promoteDeferredTargets()) {
1426 // avoid later read getting stale data while write miss is
1427 // outstanding.. see comment in timingAccess()

--- 207 unchanged lines hidden (view full) ---

1635 writebacks.push_back(cleanEvictBlk(blk));
1636 }
1637 }
1638 }
1639
1640 return blk;
1641}
1642
1643void
1644Cache::invalidateBlock(CacheBlk *blk)
1645{
1646 if (blk != tempBlock)
1647 tags->invalidate(blk);
1648 blk->invalidate();
1649}
1591
1592// Note that the reason we return a list of writebacks rather than
1593// inserting them directly in the write buffer is that this function
1594// is called by both atomic and timing-mode accesses, and in atomic
1595// mode we don't mess with the write buffer (we just perform the
1596// writebacks atomically once the original request is complete).
1597CacheBlk*
1650
1651// Note that the reason we return a list of writebacks rather than
1652// inserting them directly in the write buffer is that this function
1653// is called by both atomic and timing-mode accesses, and in atomic
1654// mode we don't mess with the write buffer (we just perform the
1655// writebacks atomically once the original request is complete).
1656CacheBlk*
1598Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks)
1657Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1658 bool allocate)
1599{
1600 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1601 Addr addr = pkt->getAddr();
1602 bool is_secure = pkt->isSecure();
1603#if TRACING_ON
1604 CacheBlk::State old_state = blk ? blk->status : 0;
1605#endif
1606

--- 7 unchanged lines hidden (view full) ---

1614 // better have read new data...
1615 assert(pkt->hasData());
1616
1617 // only read responses and write-line requests have data;
1618 // note that we don't write the data here for write-line - that
1619 // happens in the subsequent satisfyCpuSideRequest.
1620 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1621
1659{
1660 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1661 Addr addr = pkt->getAddr();
1662 bool is_secure = pkt->isSecure();
1663#if TRACING_ON
1664 CacheBlk::State old_state = blk ? blk->status : 0;
1665#endif
1666

--- 7 unchanged lines hidden (view full) ---

1674 // better have read new data...
1675 assert(pkt->hasData());
1676
1677 // only read responses and write-line requests have data;
1678 // note that we don't write the data here for write-line - that
1679 // happens in the subsequent satisfyCpuSideRequest.
1680 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1681
1622 // need to do a replacement
1623 blk = allocateBlock(addr, is_secure, writebacks);
1682 // need to do a replacement if allocating, otherwise we stick
1683 // with the temporary storage
1684 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : NULL;
1685
1624 if (blk == NULL) {
1686 if (blk == NULL) {
1625 // No replaceable block... just use temporary storage to
1626 // complete the current request and then get rid of it
1687 // No replaceable block or a mostly exclusive
1688 // cache... just use temporary storage to complete the
1689 // current request and then get rid of it
1627 assert(!tempBlock->isValid());
1628 blk = tempBlock;
1629 tempBlock->set = tags->extractSet(addr);
1630 tempBlock->tag = tags->extractTag(addr);
1631 // @todo: set security state as well...
1632 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1633 is_secure ? "s" : "ns");
1634 } else {

--- 237 unchanged lines hidden (view full) ---

1872
1873 if (respond) {
1874 // prevent anyone else from responding, cache as well as
1875 // memory, and also prevent any memory from even seeing the
1876 // request (with current inhibited semantics), note that this
1877 // applies both to reads and writes and that for writes it
1878 // works thanks to the fact that we still have dirty data and
1879 // will write it back at a later point
1690 assert(!tempBlock->isValid());
1691 blk = tempBlock;
1692 tempBlock->set = tags->extractSet(addr);
1693 tempBlock->tag = tags->extractTag(addr);
1694 // @todo: set security state as well...
1695 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1696 is_secure ? "s" : "ns");
1697 } else {

--- 237 unchanged lines hidden (view full) ---

1935
1936 if (respond) {
1937 // prevent anyone else from responding, cache as well as
1938 // memory, and also prevent any memory from even seeing the
1939 // request (with current inhibited semantics), note that this
1940 // applies both to reads and writes and that for writes it
1941 // works thanks to the fact that we still have dirty data and
1942 // will write it back at a later point
1943 assert(!pkt->memInhibitAsserted());
1880 pkt->assertMemInhibit();
1881 if (have_exclusive) {
1882 // in the case of an uncacheable request there is no point
1883 // in setting the exclusive flag, but since the recipient
1884 // does not care there is no harm in doing so, in any case
1885 // it is just a hint
1886 pkt->setSupplyExclusive();
1887 }

--- 18 unchanged lines hidden (view full) ---

1906 assert(pkt->needsResponse());
1907 delete pkt->req;
1908 delete pkt;
1909 }
1910
1911 // Do this last in case it deallocates block data or something
1912 // like that
1913 if (invalidate) {
1944 pkt->assertMemInhibit();
1945 if (have_exclusive) {
1946 // in the case of an uncacheable request there is no point
1947 // in setting the exclusive flag, but since the recipient
1948 // does not care there is no harm in doing so, in any case
1949 // it is just a hint
1950 pkt->setSupplyExclusive();
1951 }

--- 18 unchanged lines hidden (view full) ---

1970 assert(pkt->needsResponse());
1971 delete pkt->req;
1972 delete pkt;
1973 }
1974
1975 // Do this last in case it deallocates block data or something
1976 // like that
1977 if (invalidate) {
1914 if (blk != tempBlock)
1915 tags->invalidate(blk);
1916 blk->invalidate();
1978 invalidateBlock(blk);
1917 }
1918
1919 DPRINTF(Cache, "new state is %s\n", blk->print());
1920
1921 return snoop_delay;
1922}
1923
1924

--- 613 unchanged lines hidden --- 		1979 }
1980
1981 DPRINTF(Cache, "new state is %s\n", blk->print());
1982
1983 return snoop_delay;
1984}
1985
1986

--- 613 unchanged lines hidden ---