cache.cc (11600:a38c3f9c82d1) cache.cc (11601:382e0637fae0)
1/*
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 132 unchanged lines hidden (view full) ---

141 if (overwrite_mem) {
142 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
143 blk->status |= BlkDirty;
144 }
145}
146
147
148void
1/*
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 132 unchanged lines hidden (view full) ---

141 if (overwrite_mem) {
142 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
143 blk->status |= BlkDirty;
144 }
145}
146
147
148void
149Cache::satisfyCpuSideRequest(PacketPtr pkt, CacheBlk *blk,
150 bool deferred_response, bool pending_downgrade)
149Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
150 bool deferred_response, bool pending_downgrade)
151{
152 assert(pkt->isRequest());
153
154 assert(blk && blk->isValid());
155 // Occasionally this is not true... if we are a lower-level cache
156 // satisfying a string of Read and ReadEx requests from
157 // upper-level caches, a Read will mark the block as shared but we
158 // can satisfy a following ReadEx anyway since we can rely on the

--- 66 unchanged lines hidden (view full) ---

225 // copy of the line
226 if (blk->isDirty()) {
227 // special considerations if we're owner:
228 if (!deferred_response) {
229 // respond with the line in Modified state
230 // (cacheResponding set, hasSharers not set)
231 pkt->setCacheResponding();
232
151{
152 assert(pkt->isRequest());
153
154 assert(blk && blk->isValid());
155 // Occasionally this is not true... if we are a lower-level cache
156 // satisfying a string of Read and ReadEx requests from
157 // upper-level caches, a Read will mark the block as shared but we
158 // can satisfy a following ReadEx anyway since we can rely on the

--- 66 unchanged lines hidden (view full) ---

225 // copy of the line
226 if (blk->isDirty()) {
227 // special considerations if we're owner:
228 if (!deferred_response) {
229 // respond with the line in Modified state
230 // (cacheResponding set, hasSharers not set)
231 pkt->setCacheResponding();
232
233 if (clusivity == Enums::mostly_excl) {
234 // if this cache is mostly exclusive with
235 // respect to the cache above, drop the
236 // block, no need to first unset the dirty
237 // bit
238 invalidateBlock(blk);
239 } else {
240 // if this cache is mostly inclusive, we
241 // keep the block in the Exclusive state,
242 // and pass it upwards as Modified
243 // (writable and dirty), hence we have
244 // multiple caches, all on the same path
245 // towards memory, all considering the
246 // same block writable, but only one
247 // considering it Modified
233 // if this cache is mostly inclusive, we
234 // keep the block in the Exclusive state,
235 // and pass it upwards as Modified
236 // (writable and dirty), hence we have
237 // multiple caches, all on the same path
238 // towards memory, all considering the
239 // same block writable, but only one
240 // considering it Modified
248
241
249 // we get away with multiple caches (on
250 // the same path to memory) considering
251 // the block writeable as we always enter
252 // the cache hierarchy through a cache,
253 // and first snoop upwards in all other
254 // branches
255 blk->status &= ~BlkDirty;
256 }
242 // we get away with multiple caches (on
243 // the same path to memory) considering
244 // the block writeable as we always enter
245 // the cache hierarchy through a cache,
246 // and first snoop upwards in all other
247 // branches
248 blk->status &= ~BlkDirty;
257 } else {
258 // if we're responding after our own miss,
259 // there's a window where the recipient didn't
260 // know it was getting ownership and may not
261 // have responded to snoops correctly, so we
262 // have to respond with a shared line
263 pkt->setHasSharers();
264 }

--- 163 unchanged lines hidden (view full) ---

428 // it.
429 return true;
430 }
431 // We didn't find the block here, propagate the CleanEvict further
432 // down the memory hierarchy. Returning false will treat the CleanEvict
433 // like a Writeback which could not find a replaceable block so has to
434 // go to next level.
435 return false;
249 } else {
250 // if we're responding after our own miss,
251 // there's a window where the recipient didn't
252 // know it was getting ownership and may not
253 // have responded to snoops correctly, so we
254 // have to respond with a shared line
255 pkt->setHasSharers();
256 }

--- 163 unchanged lines hidden (view full) ---

420 // it.
421 return true;
422 }
423 // We didn't find the block here, propagate the CleanEvict further
424 // down the memory hierarchy. Returning false will treat the CleanEvict
425 // like a Writeback which could not find a replaceable block so has to
426 // go to next level.
427 return false;
436 } else if ((blk != nullptr) &&
437 (pkt->needsWritable() ? blk->isWritable() :
438 blk->isReadable())) {
428 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
429 blk->isReadable())) {
439 // OK to satisfy access
440 incHitCount(pkt);
430 // OK to satisfy access
431 incHitCount(pkt);
441 satisfyCpuSideRequest(pkt, blk);
432 satisfyRequest(pkt, blk);
433 maintainClusivity(pkt->fromCache(), blk);
434
442 return true;
443 }
444
445 // Can't satisfy access normally... either no block (blk == nullptr)
446 // or have block but need writable
447
448 incMissCount(pkt);
449
450 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
451 // complete miss on store conditional... just give up now
452 pkt->req->setExtraData(0);
453 return true;
454 }
455
456 return false;
457}
458
459void
435 return true;
436 }
437
438 // Can't satisfy access normally... either no block (blk == nullptr)
439 // or have block but need writable
440
441 incMissCount(pkt);
442
443 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
444 // complete miss on store conditional... just give up now
445 pkt->req->setExtraData(0);
446 return true;
447 }
448
449 return false;
450}
451
452void
453Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
454{
455 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
456 clusivity == Enums::mostly_excl) {
457 // if we have responded to a cache, and our block is still
458 // valid, but not dirty, and this cache is mostly exclusive
459 // with respect to the cache above, drop the block
460 invalidateBlock(blk);
461 }
462}
463
464void
460Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
461{
462 while (!writebacks.empty()) {
463 PacketPtr wbPkt = writebacks.front();
464 // We use forwardLatency here because we are copying writebacks to
465 // write buffer. Call isCachedAbove for both Writebacks and
466 // CleanEvicts. If isCachedAbove returns true we set BLOCK_CACHED flag
467 // in Writebacks and discard CleanEvicts.

--- 600 unchanged lines hidden (view full) ---

1068 // note the use of pkt, not bus_pkt here.
1069
1070 // write-line request to the cache that promoted
1071 // the write to a whole line
1072 blk = handleFill(pkt, blk, writebacks,
1073 allocOnFill(pkt->cmd));
1074 assert(blk != NULL);
1075 is_invalidate = false;
465Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
466{
467 while (!writebacks.empty()) {
468 PacketPtr wbPkt = writebacks.front();
469 // We use forwardLatency here because we are copying writebacks to
470 // write buffer. Call isCachedAbove for both Writebacks and
471 // CleanEvicts. If isCachedAbove returns true we set BLOCK_CACHED flag
472 // in Writebacks and discard CleanEvicts.

--- 600 unchanged lines hidden (view full) ---

1073 // note the use of pkt, not bus_pkt here.
1074
1075 // write-line request to the cache that promoted
1076 // the write to a whole line
1077 blk = handleFill(pkt, blk, writebacks,
1078 allocOnFill(pkt->cmd));
1079 assert(blk != NULL);
1080 is_invalidate = false;
1076 satisfyCpuSideRequest(pkt, blk);
1081 satisfyRequest(pkt, blk);
1077 } else if (bus_pkt->isRead() ||
1078 bus_pkt->cmd == MemCmd::UpgradeResp) {
1079 // we're updating cache state to allow us to
1080 // satisfy the upstream request from the cache
1081 blk = handleFill(bus_pkt, blk, writebacks,
1082 allocOnFill(pkt->cmd));
1082 } else if (bus_pkt->isRead() ||
1083 bus_pkt->cmd == MemCmd::UpgradeResp) {
1084 // we're updating cache state to allow us to
1085 // satisfy the upstream request from the cache
1086 blk = handleFill(bus_pkt, blk, writebacks,
1087 allocOnFill(pkt->cmd));
1083 satisfyCpuSideRequest(pkt, blk);
1088 satisfyRequest(pkt, blk);
1089 maintainClusivity(pkt->fromCache(), blk);
1084 } else {
1085 // we're satisfying the upstream request without
1086 // modifying cache state, e.g., a write-through
1087 pkt->makeAtomicResponse();
1088 }
1089 }
1090 delete bus_pkt;
1091 }

--- 221 unchanged lines hidden (view full) ---

1313
1314 // allow invalidation responses originating from write-line
1315 // requests to be discarded
1316 bool is_invalidate = pkt->isInvalidate();
1317
1318 // First offset for critical word first calculations
1319 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1320
1090 } else {
1091 // we're satisfying the upstream request without
1092 // modifying cache state, e.g., a write-through
1093 pkt->makeAtomicResponse();
1094 }
1095 }
1096 delete bus_pkt;
1097 }

--- 221 unchanged lines hidden (view full) ---

1319
1320 // allow invalidation responses originating from write-line
1321 // requests to be discarded
1322 bool is_invalidate = pkt->isInvalidate();
1323
1324 // First offset for critical word first calculations
1325 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1326
1327 bool from_cache = false;
1328
1321 while (mshr->hasTargets()) {
1322 MSHR::Target *target = mshr->getTarget();
1323 Packet *tgt_pkt = target->pkt;
1324
1325 switch (target->source) {
1326 case MSHR::Target::FromCPU:
1327 Tick completion_time;
1328 // Here we charge on completion_time the delay of the xbar if the

--- 6 unchanged lines hidden (view full) ---

1335 // immediately with dummy data so the core would be able to
1336 // retire it. This request completes right here, so we
1337 // deallocate it.
1338 delete tgt_pkt->req;
1339 delete tgt_pkt;
1340 break; // skip response
1341 }
1342
1329 while (mshr->hasTargets()) {
1330 MSHR::Target *target = mshr->getTarget();
1331 Packet *tgt_pkt = target->pkt;
1332
1333 switch (target->source) {
1334 case MSHR::Target::FromCPU:
1335 Tick completion_time;
1336 // Here we charge on completion_time the delay of the xbar if the

--- 6 unchanged lines hidden (view full) ---

1343 // immediately with dummy data so the core would be able to
1344 // retire it. This request completes right here, so we
1345 // deallocate it.
1346 delete tgt_pkt->req;
1347 delete tgt_pkt;
1348 break; // skip response
1349 }
1350
1351 // keep track of whether we have responded to another
1352 // cache
1353 from_cache = from_cache || tgt_pkt->fromCache();
1354
1343 // unlike the other packet flows, where data is found in other
1344 // caches or memory and brought back, write-line requests always
1345 // have the data right away, so the above check for "is fill?"
1346 // cannot actually be determined until examining the stored MSHR
1347 // state. We "catch up" with that logic here, which is duplicated
1348 // from above.
1349 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1350 assert(!is_error);

--- 6 unchanged lines hidden (view full) ---

1357
1358 // treat as a fill, and discard the invalidation
1359 // response
1360 is_fill = true;
1361 is_invalidate = false;
1362 }
1363
1364 if (is_fill) {
1355 // unlike the other packet flows, where data is found in other
1356 // caches or memory and brought back, write-line requests always
1357 // have the data right away, so the above check for "is fill?"
1358 // cannot actually be determined until examining the stored MSHR
1359 // state. We "catch up" with that logic here, which is duplicated
1360 // from above.
1361 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1362 assert(!is_error);

--- 6 unchanged lines hidden (view full) ---

1369
1370 // treat as a fill, and discard the invalidation
1371 // response
1372 is_fill = true;
1373 is_invalidate = false;
1374 }
1375
1376 if (is_fill) {
1365 satisfyCpuSideRequest(tgt_pkt, blk,
1366 true, mshr->hasPostDowngrade());
1377 satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1367
1368 // How many bytes past the first request is this one
1369 int transfer_offset =
1370 tgt_pkt->getOffset(blkSize) - initial_offset;
1371 if (transfer_offset < 0) {
1372 transfer_offset += blkSize;
1373 }
1374

--- 71 unchanged lines hidden (view full) ---

1446
1447 default:
1448 panic("Illegal target->source enum %d\n", target->source);
1449 }
1450
1451 mshr->popTarget();
1452 }
1453
1378
1379 // How many bytes past the first request is this one
1380 int transfer_offset =
1381 tgt_pkt->getOffset(blkSize) - initial_offset;
1382 if (transfer_offset < 0) {
1383 transfer_offset += blkSize;
1384 }
1385

--- 71 unchanged lines hidden (view full) ---

1457
1458 default:
1459 panic("Illegal target->source enum %d\n", target->source);
1460 }
1461
1462 mshr->popTarget();
1463 }
1464
1465 maintainClusivity(from_cache, blk);
1466
1454 if (blk && blk->isValid()) {
1455 // an invalidate response stemming from a write line request
1456 // should not invalidate the block, so check if the
1457 // invalidation should be discarded
1458 if (is_invalidate || mshr->hasPostInvalidate()) {
1459 invalidateBlock(blk);
1460 } else if (mshr->hasPostDowngrade()) {
1461 blk->status &= ~BlkWritable;

--- 258 unchanged lines hidden (view full) ---

1720 assert(!writeBuffer.findMatch(addr, is_secure));
1721
1722 if (blk == nullptr) {
1723 // better have read new data...
1724 assert(pkt->hasData());
1725
1726 // only read responses and write-line requests have data;
1727 // note that we don't write the data here for write-line - that
1467 if (blk && blk->isValid()) {
1468 // an invalidate response stemming from a write line request
1469 // should not invalidate the block, so check if the
1470 // invalidation should be discarded
1471 if (is_invalidate || mshr->hasPostInvalidate()) {
1472 invalidateBlock(blk);
1473 } else if (mshr->hasPostDowngrade()) {
1474 blk->status &= ~BlkWritable;

--- 258 unchanged lines hidden (view full) ---

1733 assert(!writeBuffer.findMatch(addr, is_secure));
1734
1735 if (blk == nullptr) {
1736 // better have read new data...
1737 assert(pkt->hasData());
1738
1739 // only read responses and write-line requests have data;
1740 // note that we don't write the data here for write-line - that
1728 // happens in the subsequent satisfyCpuSideRequest.
1741 // happens in the subsequent call to satisfyRequest
1729 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1730
1731 // need to do a replacement if allocating, otherwise we stick
1732 // with the temporary storage
1733 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1734
1735 if (blk == nullptr) {
1736 // No replaceable block or a mostly exclusive

--- 22 unchanged lines hidden (view full) ---

1759 }
1760
1761 if (is_secure)
1762 blk->status |= BlkSecure;
1763 blk->status |= BlkValid | BlkReadable;
1764
1765 // sanity check for whole-line writes, which should always be
1766 // marked as writable as part of the fill, and then later marked
1742 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1743
1744 // need to do a replacement if allocating, otherwise we stick
1745 // with the temporary storage
1746 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1747
1748 if (blk == nullptr) {
1749 // No replaceable block or a mostly exclusive

--- 22 unchanged lines hidden (view full) ---

1772 }
1773
1774 if (is_secure)
1775 blk->status |= BlkSecure;
1776 blk->status |= BlkValid | BlkReadable;
1777
1778 // sanity check for whole-line writes, which should always be
1779 // marked as writable as part of the fill, and then later marked
1767 // dirty as part of satisfyCpuSideRequest
1780 // dirty as part of satisfyRequest
1768 if (pkt->cmd == MemCmd::WriteLineReq) {
1769 assert(!pkt->hasSharers());
1770 // at the moment other caches do not respond to the
1771 // invalidation requests corresponding to a whole-line write
1772 assert(!pkt->cacheResponding());
1773 }
1774
1775 // here we deal with setting the appropriate state of the line,

--- 894 unchanged lines hidden --- 		1781 if (pkt->cmd == MemCmd::WriteLineReq) {
1782 assert(!pkt->hasSharers());
1783 // at the moment other caches do not respond to the
1784 // invalidation requests corresponding to a whole-line write
1785 assert(!pkt->cacheResponding());
1786 }
1787
1788 // here we deal with setting the appropriate state of the line,

--- 894 unchanged lines hidden ---