cache.cc (11352:4e195fb9ec4f) cache.cc (11375:f98df9231cdd)
1/*
1/*
2 * Copyright (c) 2010-2015 ARM Limited
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated

--- 270 unchanged lines hidden (view full) ---

281 // for invalidations we could be looking at the temp block
282 // (for upgrades we always allocate)
283 invalidateBlock(blk);
284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n",
285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
286 }
287}
288
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated

--- 270 unchanged lines hidden (view full) ---

281 // for invalidations we could be looking at the temp block
282 // (for upgrades we always allocate)
283 invalidateBlock(blk);
284 DPRINTF(CacheVerbose, "%s for %s addr %#llx size %d (invalidation)\n",
285 __func__, pkt->cmdString(), pkt->getAddr(), pkt->getSize());
286 }
287}
288
289
290/////////////////////////////////////////////////////
291//
289/////////////////////////////////////////////////////
290//
292// MSHR helper functions
293//
294/////////////////////////////////////////////////////
295
296
297void
298Cache::markInService(MSHR *mshr, bool pending_modified_resp)
299{
300 markInServiceInternal(mshr, pending_modified_resp);
301}
302
303/////////////////////////////////////////////////////
304//
305// Access path: requests coming in from the CPU side
306//
307/////////////////////////////////////////////////////
308
309bool
310Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
311 PacketList &writebacks)
312{

--- 45 unchanged lines hidden (view full) ---

358 // We check for presence of block in above caches before issuing
359 // Writeback or CleanEvict to write buffer. Therefore the only
360 // possible cases can be of a CleanEvict packet coming from above
361 // encountering a Writeback generated in this cache peer cache and
362 // waiting in the write buffer. Cases of upper level peer caches
363 // generating CleanEvict and Writeback or simply CleanEvict and
364 // CleanEvict almost simultaneously will be caught by snoops sent out
365 // by crossbar.
291// Access path: requests coming in from the CPU side
292//
293/////////////////////////////////////////////////////
294
295bool
296Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
297 PacketList &writebacks)
298{

--- 45 unchanged lines hidden (view full) ---

344 // We check for presence of block in above caches before issuing
345 // Writeback or CleanEvict to write buffer. Therefore the only
346 // possible cases can be of a CleanEvict packet coming from above
347 // encountering a Writeback generated in this cache peer cache and
348 // waiting in the write buffer. Cases of upper level peer caches
349 // generating CleanEvict and Writeback or simply CleanEvict and
350 // CleanEvict almost simultaneously will be caught by snoops sent out
351 // by crossbar.
366 std::vector<MSHR *> outgoing;
367 if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(),
368 outgoing)) {
369 assert(outgoing.size() == 1);
370 MSHR *wb_entry = outgoing[0];
352 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
353 pkt->isSecure());
354 if (wb_entry) {
371 assert(wb_entry->getNumTargets() == 1);
372 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
373 assert(wbPkt->isWriteback());
374
375 if (pkt->isCleanEviction()) {
376 // The CleanEvict and WritebackClean snoops into other
377 // peer caches of the same level while traversing the
378 // crossbar. If a copy of the block is found, the

--- 4 unchanged lines hidden (view full) ---

383 // discard the CleanEvict by returning true.
384 wbPkt->clearBlockCached();
385 return true;
386 } else {
387 assert(pkt->cmd == MemCmd::WritebackDirty);
388 // Dirty writeback from above trumps our clean
389 // writeback... discard here
390 // Note: markInService will remove entry from writeback buffer.
355 assert(wb_entry->getNumTargets() == 1);
356 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
357 assert(wbPkt->isWriteback());
358
359 if (pkt->isCleanEviction()) {
360 // The CleanEvict and WritebackClean snoops into other
361 // peer caches of the same level while traversing the
362 // crossbar. If a copy of the block is found, the

--- 4 unchanged lines hidden (view full) ---

367 // discard the CleanEvict by returning true.
368 wbPkt->clearBlockCached();
369 return true;
370 } else {
371 assert(pkt->cmd == MemCmd::WritebackDirty);
372 // Dirty writeback from above trumps our clean
373 // writeback... discard here
374 // Note: markInService will remove entry from writeback buffer.
391 markInService(wb_entry, false);
375 markInService(wb_entry);
392 delete wbPkt;
393 }
394 }
395 }
396
397 // Writeback handling is special case. We can write the block into
398 // the cache without having a writeable copy (or any copy at all).
399 if (pkt->isWriteback()) {

--- 834 unchanged lines hidden (view full) ---

1234/////////////////////////////////////////////////////
1235//
1236// Response handling: responses from the memory side
1237//
1238/////////////////////////////////////////////////////
1239
1240
1241void
376 delete wbPkt;
377 }
378 }
379 }
380
381 // Writeback handling is special case. We can write the block into
382 // the cache without having a writeable copy (or any copy at all).
383 if (pkt->isWriteback()) {

--- 834 unchanged lines hidden (view full) ---

1218/////////////////////////////////////////////////////
1219//
1220// Response handling: responses from the memory side
1221//
1222/////////////////////////////////////////////////////
1223
1224
1225void
1226Cache::handleUncacheableWriteResp(PacketPtr pkt)
1227{
1228 WriteQueueEntry *wq_entry =
1229 dynamic_cast<WriteQueueEntry*>(pkt->senderState);
1230 assert(wq_entry);
1231
1232 WriteQueueEntry::Target *target = wq_entry->getTarget();
1233 Packet *tgt_pkt = target->pkt;
1234
1235 // we send out invalidation reqs and get invalidation
1236 // responses for write-line requests
1237 assert(tgt_pkt->cmd != MemCmd::WriteLineReq);
1238
1239 int stats_cmd_idx = tgt_pkt->cmdToIndex();
1240 Tick miss_latency = curTick() - target->recvTime;
1241 assert(pkt->req->masterId() < system->maxMasters());
1242 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1243 miss_latency;
1244
1245 tgt_pkt->makeTimingResponse();
1246 // if this packet is an error copy that to the new packet
1247 if (pkt->isError())
1248 tgt_pkt->copyError(pkt);
1249 // Reset the bus additional time as it is now accounted for
1250 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1251 Tick completion_time = clockEdge(responseLatency) +
1252 pkt->headerDelay + pkt->payloadDelay;
1253
1254 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1255
1256 wq_entry->popTarget();
1257 assert(!wq_entry->hasTargets());
1258
1259 bool wasFull = writeBuffer.isFull();
1260 writeBuffer.deallocate(wq_entry);
1261
1262 if (wasFull && !writeBuffer.isFull()) {
1263 clearBlocked(Blocked_NoWBBuffers);
1264 }
1265
1266 delete pkt;
1267}
1268
1269void
1242Cache::recvTimingResp(PacketPtr pkt)
1243{
1244 assert(pkt->isResponse());
1245
1246 // all header delay should be paid for by the crossbar, unless
1247 // this is a prefetch response from above
1248 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1249 "%s saw a non-zero packet delay\n", name());
1250
1270Cache::recvTimingResp(PacketPtr pkt)
1271{
1272 assert(pkt->isResponse());
1273
1274 // all header delay should be paid for by the crossbar, unless
1275 // this is a prefetch response from above
1276 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1277 "%s saw a non-zero packet delay\n", name());
1278
1251 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
1252 bool is_error = pkt->isError();
1253
1279 bool is_error = pkt->isError();
1280
1254 assert(mshr);
1255
1256 if (is_error) {
1257 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), "
1258 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns",
1259 pkt->cmdString());
1260 }
1261
1262 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n",
1263 pkt->cmdString(), pkt->getAddr(), pkt->getSize(),
1264 pkt->isSecure() ? "s" : "ns");
1265
1281 if (is_error) {
1282 DPRINTF(Cache, "Cache received packet with error for addr %#llx (%s), "
1283 "cmd: %s\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns",
1284 pkt->cmdString());
1285 }
1286
1287 DPRINTF(Cache, "Handling response %s for addr %#llx size %d (%s)\n",
1288 pkt->cmdString(), pkt->getAddr(), pkt->getSize(),
1289 pkt->isSecure() ? "s" : "ns");
1290
1266 MSHRQueue *mq = mshr->queue;
1267 bool wasFull = mq->isFull();
1291 // if this is a write, we should be looking at an uncacheable
1292 // write
1293 if (pkt->isWrite()) {
1294 assert(pkt->req->isUncacheable());
1295 handleUncacheableWriteResp(pkt);
1296 return;
1297 }
1268
1298
1299 // we have dealt with any (uncacheable) writes above, from here on
1300 // we know we are dealing with an MSHR due to a miss or a prefetch
1301 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
1302 assert(mshr);
1303
1269 if (mshr == noTargetMSHR) {
1270 // we always clear at least one target
1271 clearBlocked(Blocked_NoTargets);
1272 noTargetMSHR = NULL;
1273 }
1274
1275 // Initial target is used just for stats
1276 MSHR::Target *initial_tgt = mshr->getTarget();
1277 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1278 Tick miss_latency = curTick() - initial_tgt->recvTime;
1304 if (mshr == noTargetMSHR) {
1305 // we always clear at least one target
1306 clearBlocked(Blocked_NoTargets);
1307 noTargetMSHR = NULL;
1308 }
1309
1310 // Initial target is used just for stats
1311 MSHR::Target *initial_tgt = mshr->getTarget();
1312 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1313 Tick miss_latency = curTick() - initial_tgt->recvTime;
1279 PacketList writebacks;
1280 // We need forward_time here because we have a call of
1281 // allocateWriteBuffer() that need this parameter to specify the
1282 // time to request the bus. In this case we use forward latency
1283 // because there is a writeback. We pay also here for headerDelay
1284 // that is charged of bus latencies if the packet comes from the
1285 // bus.
1286 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1287
1288 if (pkt->req->isUncacheable()) {
1289 assert(pkt->req->masterId() < system->maxMasters());
1290 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1291 miss_latency;
1292 } else {
1293 assert(pkt->req->masterId() < system->maxMasters());
1294 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1295 miss_latency;
1296 }
1297
1314
1315 if (pkt->req->isUncacheable()) {
1316 assert(pkt->req->masterId() < system->maxMasters());
1317 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1318 miss_latency;
1319 } else {
1320 assert(pkt->req->masterId() < system->maxMasters());
1321 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1322 miss_latency;
1323 }
1324
1325 bool wasFull = mshrQueue.isFull();
1326
1327 PacketList writebacks;
1328
1329 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1330
1298 // upgrade deferred targets if the response has no sharers, and is
1299 // thus passing writable
1300 if (!pkt->hasSharers()) {
1301 mshr->promoteWritable();
1302 }
1303
1304 bool is_fill = !mshr->isForward &&
1305 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);

--- 159 unchanged lines hidden (view full) ---

1465 }
1466
1467 if (mshr->promoteDeferredTargets()) {
1468 // avoid later read getting stale data while write miss is
1469 // outstanding.. see comment in timingAccess()
1470 if (blk) {
1471 blk->status &= ~BlkReadable;
1472 }
1331 // upgrade deferred targets if the response has no sharers, and is
1332 // thus passing writable
1333 if (!pkt->hasSharers()) {
1334 mshr->promoteWritable();
1335 }
1336
1337 bool is_fill = !mshr->isForward &&
1338 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);

--- 159 unchanged lines hidden (view full) ---

1498 }
1499
1500 if (mshr->promoteDeferredTargets()) {
1501 // avoid later read getting stale data while write miss is
1502 // outstanding.. see comment in timingAccess()
1503 if (blk) {
1504 blk->status &= ~BlkReadable;
1505 }
1473 mq = mshr->queue;
1474 mq->markPending(mshr);
1506 mshrQueue.markPending(mshr);
1475 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1476 } else {
1507 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1508 } else {
1477 mq->deallocate(mshr);
1478 if (wasFull && !mq->isFull()) {
1479 clearBlocked((BlockedCause)mq->index);
1509 mshrQueue.deallocate(mshr);
1510 if (wasFull && !mshrQueue.isFull()) {
1511 clearBlocked(Blocked_NoMSHRs);
1480 }
1481
1482 // Request the bus for a prefetch if this deallocation freed enough
1483 // MSHRs for a prefetch to take place
1512 }
1513
1514 // Request the bus for a prefetch if this deallocation freed enough
1515 // MSHRs for a prefetch to take place
1484 if (prefetcher && mq == &mshrQueue && mshrQueue.canPrefetch()) {
1516 if (prefetcher && mshrQueue.canPrefetch()) {
1485 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1486 clockEdge());
1487 if (next_pf_time != MaxTick)
1488 schedMemSideSendEvent(next_pf_time);
1489 }
1490 }
1491 // reset the xbar additional timinig as it is now accounted for
1492 pkt->headerDelay = pkt->payloadDelay = 0;

--- 217 unchanged lines hidden (view full) ---

1710{
1711 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1712 Addr addr = pkt->getAddr();
1713 bool is_secure = pkt->isSecure();
1714#if TRACING_ON
1715 CacheBlk::State old_state = blk ? blk->status : 0;
1716#endif
1717
1517 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1518 clockEdge());
1519 if (next_pf_time != MaxTick)
1520 schedMemSideSendEvent(next_pf_time);
1521 }
1522 }
1523 // reset the xbar additional timinig as it is now accounted for
1524 pkt->headerDelay = pkt->payloadDelay = 0;

--- 217 unchanged lines hidden (view full) ---

1742{
1743 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1744 Addr addr = pkt->getAddr();
1745 bool is_secure = pkt->isSecure();
1746#if TRACING_ON
1747 CacheBlk::State old_state = blk ? blk->status : 0;
1748#endif
1749
1718 // When handling a fill, discard any CleanEvicts for the
1719 // same address in write buffer.
1720 Addr M5_VAR_USED blk_addr = blockAlign(pkt->getAddr());
1721 std::vector<MSHR *> M5_VAR_USED wbs;
1722 assert (!writeBuffer.findMatches(blk_addr, is_secure, wbs));
1750 // When handling a fill, we should have no writes to this line.
1751 assert(addr == blockAlign(addr));
1752 assert(!writeBuffer.findMatch(addr, is_secure));
1723
1724 if (blk == NULL) {
1725 // better have read new data...
1726 assert(pkt->hasData());
1727
1728 // only read responses and write-line requests have data;
1729 // note that we don't write the data here for write-line - that
1730 // happens in the subsequent satisfyCpuSideRequest.

--- 371 unchanged lines hidden (view full) ---

2102 mshr->print());
2103
2104 if (mshr->getNumTargets() > numTarget)
2105 warn("allocating bonus target for snoop"); //handle later
2106 return;
2107 }
2108
2109 //We also need to check the writeback buffers and handle those
1753
1754 if (blk == NULL) {
1755 // better have read new data...
1756 assert(pkt->hasData());
1757
1758 // only read responses and write-line requests have data;
1759 // note that we don't write the data here for write-line - that
1760 // happens in the subsequent satisfyCpuSideRequest.

--- 371 unchanged lines hidden (view full) ---

2132 mshr->print());
2133
2134 if (mshr->getNumTargets() > numTarget)
2135 warn("allocating bonus target for snoop"); //handle later
2136 return;
2137 }
2138
2139 //We also need to check the writeback buffers and handle those
2110 std::vector<MSHR *> writebacks;
2111 if (writeBuffer.findMatches(blk_addr, is_secure, writebacks)) {
2140 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2141 if (wb_entry) {
2112 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2113 pkt->getAddr(), is_secure ? "s" : "ns");
2142 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2143 pkt->getAddr(), is_secure ? "s" : "ns");
2114
2115 // Look through writebacks for any cachable writes.
2116 // We should only ever find a single match
2117 assert(writebacks.size() == 1);
2118 MSHR *wb_entry = writebacks[0];
2119 // Expect to see only Writebacks and/or CleanEvicts here, both of
2120 // which should not be generated for uncacheable data.
2121 assert(!wb_entry->isUncacheable());
2122 // There should only be a single request responsible for generating
2123 // Writebacks/CleanEvicts.
2124 assert(wb_entry->getNumTargets() == 1);
2125 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2126 assert(wb_pkt->isEviction());

--- 34 unchanged lines hidden (view full) ---

2161
2162 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2163 false, false);
2164 }
2165
2166 if (invalidate) {
2167 // Invalidation trumps our writeback... discard here
2168 // Note: markInService will remove entry from writeback buffer.
2144 // Expect to see only Writebacks and/or CleanEvicts here, both of
2145 // which should not be generated for uncacheable data.
2146 assert(!wb_entry->isUncacheable());
2147 // There should only be a single request responsible for generating
2148 // Writebacks/CleanEvicts.
2149 assert(wb_entry->getNumTargets() == 1);
2150 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2151 assert(wb_pkt->isEviction());

--- 34 unchanged lines hidden (view full) ---

2186
2187 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2188 false, false);
2189 }
2190
2191 if (invalidate) {
2192 // Invalidation trumps our writeback... discard here
2193 // Note: markInService will remove entry from writeback buffer.
2169 markInService(wb_entry, false);
2194 markInService(wb_entry);
2170 delete wb_pkt;
2171 }
2172 }
2173
2174 // If this was a shared writeback, there may still be
2175 // other shared copies above that require invalidation.
2176 // We could be more selective and return here if the
2177 // request is non-exclusive or if the writeback is

--- 26 unchanged lines hidden (view full) ---

2204 }
2205
2206 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2207 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2208 return snoop_delay + lookupLatency * clockPeriod();
2209}
2210
2211
2195 delete wb_pkt;
2196 }
2197 }
2198
2199 // If this was a shared writeback, there may still be
2200 // other shared copies above that require invalidation.
2201 // We could be more selective and return here if the
2202 // request is non-exclusive or if the writeback is

--- 26 unchanged lines hidden (view full) ---

2229 }
2230
2231 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2232 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2233 return snoop_delay + lookupLatency * clockPeriod();
2234}
2235
2236
2212MSHR *
2213Cache::getNextMSHR()
2237QueueEntry*
2238Cache::getNextQueueEntry()
2214{
2215 // Check both MSHR queue and write buffer for potential requests,
2216 // note that null does not mean there is no request, it could
2217 // simply be that it is not ready
2239{
2240 // Check both MSHR queue and write buffer for potential requests,
2241 // note that null does not mean there is no request, it could
2242 // simply be that it is not ready
2218 MSHR *miss_mshr = mshrQueue.getNextMSHR();
2219 MSHR *write_mshr = writeBuffer.getNextMSHR();
2243 MSHR *miss_mshr = mshrQueue.getNext();
2244 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2220
2221 // If we got a write buffer request ready, first priority is a
2245
2246 // If we got a write buffer request ready, first priority is a
2222 // full write buffer, otherwhise we favour the miss requests
2223 if (write_mshr &&
2224 ((writeBuffer.isFull() && writeBuffer.inServiceEntries == 0) ||
2247 // full write buffer (but only if we have no uncacheable write
2248 // responses outstanding, possibly revisit this last part),
2249 // otherwhise we favour the miss requests
2250 if (wq_entry &&
2251 ((writeBuffer.isFull() && writeBuffer.numInService() == 0) ||
2225 !miss_mshr)) {
2226 // need to search MSHR queue for conflicting earlier miss.
2227 MSHR *conflict_mshr =
2252 !miss_mshr)) {
2253 // need to search MSHR queue for conflicting earlier miss.
2254 MSHR *conflict_mshr =
2228 mshrQueue.findPending(write_mshr->blkAddr,
2229 write_mshr->isSecure);
2255 mshrQueue.findPending(wq_entry->blkAddr,
2256 wq_entry->isSecure);
2230
2257
2231 if (conflict_mshr && conflict_mshr->order < write_mshr->order) {
2258 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2232 // Service misses in order until conflict is cleared.
2233 return conflict_mshr;
2234
2235 // @todo Note that we ignore the ready time of the conflict here
2236 }
2237
2238 // No conflicts; issue write
2259 // Service misses in order until conflict is cleared.
2260 return conflict_mshr;
2261
2262 // @todo Note that we ignore the ready time of the conflict here
2263 }
2264
2265 // No conflicts; issue write
2239 return write_mshr;
2266 return wq_entry;
2240 } else if (miss_mshr) {
2241 // need to check for conflicting earlier writeback
2267 } else if (miss_mshr) {
2268 // need to check for conflicting earlier writeback
2242 MSHR *conflict_mshr =
2269 WriteQueueEntry *conflict_mshr =
2243 writeBuffer.findPending(miss_mshr->blkAddr,
2244 miss_mshr->isSecure);
2245 if (conflict_mshr) {
2246 // not sure why we don't check order here... it was in the
2247 // original code but commented out.
2248
2249 // The only way this happens is if we are
2250 // doing a write and we didn't have permissions
2251 // then subsequently saw a writeback (owned got evicted)
2252 // We need to make sure to perform the writeback first
2253 // To preserve the dirty data, then we can issue the write
2254
2270 writeBuffer.findPending(miss_mshr->blkAddr,
2271 miss_mshr->isSecure);
2272 if (conflict_mshr) {
2273 // not sure why we don't check order here... it was in the
2274 // original code but commented out.
2275
2276 // The only way this happens is if we are
2277 // doing a write and we didn't have permissions
2278 // then subsequently saw a writeback (owned got evicted)
2279 // We need to make sure to perform the writeback first
2280 // To preserve the dirty data, then we can issue the write
2281
2255 // should we return write_mshr here instead? I.e. do we
2282 // should we return wq_entry here instead? I.e. do we
2256 // have to flush writes in order? I don't think so... not
2257 // for Alpha anyway. Maybe for x86?
2258 return conflict_mshr;
2259
2260 // @todo Note that we ignore the ready time of the conflict here
2261 }
2262
2263 // No conflicts; issue read
2264 return miss_mshr;
2265 }
2266
2267 // fall through... no pending requests. Try a prefetch.
2283 // have to flush writes in order? I don't think so... not
2284 // for Alpha anyway. Maybe for x86?
2285 return conflict_mshr;
2286
2287 // @todo Note that we ignore the ready time of the conflict here
2288 }
2289
2290 // No conflicts; issue read
2291 return miss_mshr;
2292 }
2293
2294 // fall through... no pending requests. Try a prefetch.
2268 assert(!miss_mshr && !write_mshr);
2295 assert(!miss_mshr && !wq_entry);
2269 if (prefetcher && mshrQueue.canPrefetch()) {
2270 // If we have a miss queue slot, we can try a prefetch
2271 PacketPtr pkt = prefetcher->getPacket();
2272 if (pkt) {
2273 Addr pf_addr = blockAlign(pkt->getAddr());
2274 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2275 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2276 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {

--- 9 unchanged lines hidden (view full) ---

2286 } else {
2287 // free the request and packet
2288 delete pkt->req;
2289 delete pkt;
2290 }
2291 }
2292 }
2293
2296 if (prefetcher && mshrQueue.canPrefetch()) {
2297 // If we have a miss queue slot, we can try a prefetch
2298 PacketPtr pkt = prefetcher->getPacket();
2299 if (pkt) {
2300 Addr pf_addr = blockAlign(pkt->getAddr());
2301 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2302 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2303 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {

--- 9 unchanged lines hidden (view full) ---

2313 } else {
2314 // free the request and packet
2315 delete pkt->req;
2316 delete pkt;
2317 }
2318 }
2319 }
2320
2294 return NULL;
2321 return nullptr;
2295}
2296
2297bool
2298Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2299{
2300 if (!forwardSnoops)
2301 return false;
2302 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and

--- 14 unchanged lines hidden (view full) ---

2317 assert(!(snoop_pkt.cacheResponding()));
2318 return snoop_pkt.isBlockCached();
2319 } else {
2320 cpuSidePort->sendAtomicSnoop(pkt);
2321 return pkt->isBlockCached();
2322 }
2323}
2324
2322}
2323
2324bool
2325Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2326{
2327 if (!forwardSnoops)
2328 return false;
2329 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and

--- 14 unchanged lines hidden (view full) ---

2344 assert(!(snoop_pkt.cacheResponding()));
2345 return snoop_pkt.isBlockCached();
2346 } else {
2347 cpuSidePort->sendAtomicSnoop(pkt);
2348 return pkt->isBlockCached();
2349 }
2350}
2351
2325PacketPtr
2326Cache::getTimingPacket()
2352Tick
2353Cache::nextQueueReadyTime() const
2327{
2354{
2328 MSHR *mshr = getNextMSHR();
2355 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2356 writeBuffer.nextReadyTime());
2329
2357
2330 if (mshr == NULL) {
2331 return NULL;
2358 // Don't signal prefetch ready time if no MSHRs available
2359 // Will signal once enoguh MSHRs are deallocated
2360 if (prefetcher && mshrQueue.canPrefetch()) {
2361 nextReady = std::min(nextReady,
2362 prefetcher->nextPrefetchReadyTime());
2332 }
2333
2363 }
2364
2365 return nextReady;
2366}
2367
2368bool
2369Cache::sendMSHRQueuePacket(MSHR* mshr)
2370{
2371 assert(mshr);
2372
2334 // use request from 1st target
2335 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2373 // use request from 1st target
2374 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2336 PacketPtr pkt = NULL;
2337
2375
2338 DPRINTF(CachePort, "%s %s for addr %#llx size %d\n", __func__,
2339 tgt_pkt->cmdString(), tgt_pkt->getAddr(), tgt_pkt->getSize());
2376 DPRINTF(Cache, "%s MSHR %s for addr %#llx size %d\n", __func__,
2377 tgt_pkt->cmdString(), tgt_pkt->getAddr(),
2378 tgt_pkt->getSize());
2340
2341 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2342
2343 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2379
2380 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2381
2382 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2383 // we should never have hardware prefetches to allocated
2384 // blocks
2385 assert(blk == NULL);
2386
2344 // We need to check the caches above us to verify that
2345 // they don't have a copy of this block in the dirty state
2346 // at the moment. Without this check we could get a stale
2347 // copy from memory that might get used in place of the
2348 // dirty one.
2349 Packet snoop_pkt(tgt_pkt, true, false);
2350 snoop_pkt.setExpressSnoop();
2351 // We are sending this packet upwards, but if it hits we will

--- 22 unchanged lines hidden (view full) ---

2374 // if we are getting a snoop response with no sharers it
2375 // will be allocated as Modified
2376 bool pending_modified_resp = !snoop_pkt.hasSharers();
2377 markInService(mshr, pending_modified_resp);
2378
2379 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2380 " %#x (%s) hit\n",
2381 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2387 // We need to check the caches above us to verify that
2388 // they don't have a copy of this block in the dirty state
2389 // at the moment. Without this check we could get a stale
2390 // copy from memory that might get used in place of the
2391 // dirty one.
2392 Packet snoop_pkt(tgt_pkt, true, false);
2393 snoop_pkt.setExpressSnoop();
2394 // We are sending this packet upwards, but if it hits we will

--- 22 unchanged lines hidden (view full) ---

2417 // if we are getting a snoop response with no sharers it
2418 // will be allocated as Modified
2419 bool pending_modified_resp = !snoop_pkt.hasSharers();
2420 markInService(mshr, pending_modified_resp);
2421
2422 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2423 " %#x (%s) hit\n",
2424 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2382 return NULL;
2425 return false;
2383 }
2384
2426 }
2427
2385 if (snoop_pkt.isBlockCached() || blk != NULL) {
2428 if (snoop_pkt.isBlockCached()) {
2386 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2387 "Deallocating mshr target %#x.\n",
2388 mshr->blkAddr);
2429 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2430 "Deallocating mshr target %#x.\n",
2431 mshr->blkAddr);
2432
2389 // Deallocate the mshr target
2433 // Deallocate the mshr target
2390 if (mshr->queue->forceDeallocateTarget(mshr)) {
2434 if (mshrQueue.forceDeallocateTarget(mshr)) {
2391 // Clear block if this deallocation resulted freed an
2392 // mshr when all had previously been utilized
2435 // Clear block if this deallocation resulted freed an
2436 // mshr when all had previously been utilized
2393 clearBlocked((BlockedCause)(mshr->queue->index));
2437 clearBlocked(Blocked_NoMSHRs);
2394 }
2438 }
2395 return NULL;
2439 return false;
2396 }
2397 }
2398
2440 }
2441 }
2442
2399 if (mshr->isForwardNoResponse()) {
2400 // no response expected, just forward packet as it is
2401 assert(tags->findBlock(mshr->blkAddr, mshr->isSecure) == NULL);
2402 pkt = tgt_pkt;
2403 } else {
2404 pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable());
2443 // either a prefetch that is not present upstream, or a normal
2444 // MSHR request, proceed to get the packet to send downstream
2445 PacketPtr pkt = getBusPacket(tgt_pkt, blk, mshr->needsWritable());
2405
2446
2406 mshr->isForward = (pkt == NULL);
2447 mshr->isForward = (pkt == NULL);
2407
2448
2408 if (mshr->isForward) {
2409 // not a cache block request, but a response is expected
2410 // make copy of current packet to forward, keep current
2411 // copy for response handling
2412 pkt = new Packet(tgt_pkt, false, true);
2413 if (pkt->isWrite()) {
2414 pkt->setData(tgt_pkt->getConstPtr<uint8_t>());
2415 }
2416 }
2449 if (mshr->isForward) {
2450 // not a cache block request, but a response is expected
2451 // make copy of current packet to forward, keep current
2452 // copy for response handling
2453 pkt = new Packet(tgt_pkt, false, true);
2454 assert(!pkt->isWrite());
2417 }
2418
2455 }
2456
2419 assert(pkt != NULL);
2420 // play it safe and append (rather than set) the sender state, as
2421 // forwarded packets may already have existing state
2457 // play it safe and append (rather than set) the sender state,
2458 // as forwarded packets may already have existing state
2422 pkt->pushSenderState(mshr);
2459 pkt->pushSenderState(mshr);
2423 return pkt;
2424}
2425
2460
2461 if (!memSidePort->sendTimingReq(pkt)) {
2462 // we are awaiting a retry, but we
2463 // delete the packet and will be creating a new packet
2464 // when we get the opportunity
2465 delete pkt;
2426
2466
2427Tick
2428Cache::nextMSHRReadyTime() const
2467 // note that we have now masked any requestBus and
2468 // schedSendEvent (we will wait for a retry before
2469 // doing anything), and this is so even if we do not
2470 // care about this packet and might override it before
2471 // it gets retried
2472 return true;
2473 } else {
2474 // As part of the call to sendTimingReq the packet is
2475 // forwarded to all neighbouring caches (and any caches
2476 // above them) as a snoop. Thus at this point we know if
2477 // any of the neighbouring caches are responding, and if
2478 // so, we know it is dirty, and we can determine if it is
2479 // being passed as Modified, making our MSHR the ordering
2480 // point
2481 bool pending_modified_resp = !pkt->hasSharers() &&
2482 pkt->cacheResponding();
2483 markInService(mshr, pending_modified_resp);
2484 return false;
2485 }
2486}
2487
2488bool
2489Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2429{
2490{
2430 Tick nextReady = std::min(mshrQueue.nextMSHRReadyTime(),
2431 writeBuffer.nextMSHRReadyTime());
2491 assert(wq_entry);
2432
2492
2433 // Don't signal prefetch ready time if no MSHRs available
2434 // Will signal once enoguh MSHRs are deallocated
2435 if (prefetcher && mshrQueue.canPrefetch()) {
2436 nextReady = std::min(nextReady,
2437 prefetcher->nextPrefetchReadyTime());
2493 // always a single target for write queue entries
2494 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2495
2496 DPRINTF(Cache, "%s write %s for addr %#llx size %d\n", __func__,
2497 tgt_pkt->cmdString(), tgt_pkt->getAddr(),
2498 tgt_pkt->getSize());
2499
2500 PacketPtr pkt = nullptr;
2501 bool delete_pkt = false;
2502
2503 if (tgt_pkt->isEviction()) {
2504 assert(!wq_entry->isUncacheable());
2505 // no response expected, just forward packet as it is
2506 pkt = tgt_pkt;
2507 } else {
2508 // the only thing we deal with besides eviction commands
2509 // are uncacheable writes
2510 assert(tgt_pkt->req->isUncacheable() && tgt_pkt->isWrite());
2511 // not a cache block request, but a response is expected
2512 // make copy of current packet to forward, keep current
2513 // copy for response handling
2514 pkt = new Packet(tgt_pkt, false, true);
2515 pkt->setData(tgt_pkt->getConstPtr<uint8_t>());
2516 delete_pkt = true;
2438 }
2439
2517 }
2518
2440 return nextReady;
2519 pkt->pushSenderState(wq_entry);
2520
2521 if (!memSidePort->sendTimingReq(pkt)) {
2522 if (delete_pkt) {
2523 // we are awaiting a retry, but we
2524 // delete the packet and will be creating a new packet
2525 // when we get the opportunity
2526 delete pkt;
2527 }
2528 // note that we have now masked any requestBus and
2529 // schedSendEvent (we will wait for a retry before
2530 // doing anything), and this is so even if we do not
2531 // care about this packet and might override it before
2532 // it gets retried
2533 return true;
2534 } else {
2535 markInService(wq_entry);
2536 return false;
2537 }
2441}
2442
2443void
2444Cache::serialize(CheckpointOut &cp) const
2445{
2446 bool dirty(isDirty());
2447
2448 if (dirty) {

--- 132 unchanged lines hidden (view full) ---

2581 assert(!waitingOnRetry);
2582
2583 // there should never be any deferred request packets in the
2584 // queue, instead we resly on the cache to provide the packets
2585 // from the MSHR queue or write queue
2586 assert(deferredPacketReadyTime() == MaxTick);
2587
2588 // check for request packets (requests & writebacks)
2538}
2539
2540void
2541Cache::serialize(CheckpointOut &cp) const
2542{
2543 bool dirty(isDirty());
2544
2545 if (dirty) {

--- 132 unchanged lines hidden (view full) ---

2678 assert(!waitingOnRetry);
2679
2680 // there should never be any deferred request packets in the
2681 // queue, instead we resly on the cache to provide the packets
2682 // from the MSHR queue or write queue
2683 assert(deferredPacketReadyTime() == MaxTick);
2684
2685 // check for request packets (requests & writebacks)
2589 PacketPtr pkt = cache.getTimingPacket();
2590 if (pkt == NULL) {
2686 QueueEntry* entry = cache.getNextQueueEntry();
2687
2688 if (!entry) {
2591 // can happen if e.g. we attempt a writeback and fail, but
2592 // before the retry, the writeback is eliminated because
2593 // we snoop another cache's ReadEx.
2594 } else {
2689 // can happen if e.g. we attempt a writeback and fail, but
2690 // before the retry, the writeback is eliminated because
2691 // we snoop another cache's ReadEx.
2692 } else {
2595 MSHR *mshr = dynamic_cast<MSHR*>(pkt->senderState);
2596 // in most cases getTimingPacket allocates a new packet, and
2597 // we must delete it unless it is successfully sent
2598 bool delete_pkt = !mshr->isForwardNoResponse();
2599
2600 // let our snoop responses go first if there are responses to
2693 // let our snoop responses go first if there are responses to
2601 // the same addresses we are about to writeback, note that
2602 // this creates a dependency between requests and snoop
2603 // responses, but that should not be a problem since there is
2604 // a chain already and the key is that the snoop responses can
2605 // sink unconditionally
2606 if (snoopRespQueue.hasAddr(pkt->getAddr())) {
2607 DPRINTF(CachePort, "Waiting for snoop response to be sent\n");
2608 Tick when = snoopRespQueue.deferredPacketReadyTime();
2609 schedSendEvent(when);
2610
2611 if (delete_pkt)
2612 delete pkt;
2613
2694 // the same addresses
2695 if (checkConflictingSnoop(entry->blkAddr)) {
2614 return;
2615 }
2696 return;
2697 }
2616
2617
2618 waitingOnRetry = !masterPort.sendTimingReq(pkt);
2619
2620 if (waitingOnRetry) {
2621 DPRINTF(CachePort, "now waiting on a retry\n");
2622 if (delete_pkt) {
2623 // we are awaiting a retry, but we
2624 // delete the packet and will be creating a new packet
2625 // when we get the opportunity
2626 delete pkt;
2627 }
2628 // note that we have now masked any requestBus and
2629 // schedSendEvent (we will wait for a retry before
2630 // doing anything), and this is so even if we do not
2631 // care about this packet and might override it before
2632 // it gets retried
2633 } else {
2634 // As part of the call to sendTimingReq the packet is
2635 // forwarded to all neighbouring caches (and any caches
2636 // above them) as a snoop. Thus at this point we know if
2637 // any of the neighbouring caches are responding, and if
2638 // so, we know it is dirty, and we can determine if it is
2639 // being passed as Modified, making our MSHR the ordering
2640 // point
2641 bool pending_modified_resp = !pkt->hasSharers() &&
2642 pkt->cacheResponding();
2643
2644 cache.markInService(mshr, pending_modified_resp);
2645 }
2698 waitingOnRetry = entry->sendPacket(cache);
2646 }
2647
2648 // if we succeeded and are not waiting for a retry, schedule the
2699 }
2700
2701 // if we succeeded and are not waiting for a retry, schedule the
2649 // next send considering when the next MSHR is ready, note that
2702 // next send considering when the next queue is ready, note that
2650 // snoop responses have their own packet queue and thus schedule
2651 // their own events
2652 if (!waitingOnRetry) {
2703 // snoop responses have their own packet queue and thus schedule
2704 // their own events
2705 if (!waitingOnRetry) {
2653 schedSendEvent(cache.nextMSHRReadyTime());
2706 schedSendEvent(cache.nextQueueReadyTime());
2654 }
2655}
2656
2657Cache::
2658MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2659 const std::string &_label)
2660 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2661 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2662 _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2663{
2664}
2707 }
2708}
2709
2710Cache::
2711MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2712 const std::string &_label)
2713 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2714 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2715 _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2716{
2717}