46a47
> * Nikos Nikoleris
324a326,328
> if (pkt->req->isCacheMaintenance()) {
> // A cache maintenance operation is always forwarded to the
> // memory below even if the block is found in dirty state.
325a330,335
> // We defer any changes to the state of the block until we
> // create and mark as in service the mshr for the downstream
> // packet.
> return false;
> }
>
652a663,666
> // Cache maintenance operations have to visit all the caches down
> // to the specified xbar (PoC, PoU, etc.). Even if a cache above
> // is responding we forward the packet to the memory below rather
> // than creating an express snoop.
766c780,781
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch()) {
> assert(!pkt->req->isCacheMaintenance());
767a783
> }
857a874,880
> } else if (pkt->cmd == MemCmd::WriteClean) {
> // A WriteClean should never coalesce with any
> // outstanding cache maintenance requests.
>
> // We use forward_time here because there is an
> // uncached memory write, forwarded to WriteBuffer.
> allocateWriteBuffer(pkt, forward_time);
891c914,915
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch() &&
> !pkt->req->isCacheMaintenance())
929,930c953,954
< assert(pkt->needsWritable());
< assert(!blk->isWritable());
---
> assert((pkt->needsWritable() && !blk->isWritable()) ||
> pkt->req->isCacheMaintenance());
941c965,966
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch() &&
> !pkt->req->isCacheMaintenance())
964c989
< cpu_pkt->cmd == MemCmd::InvalidateReq) {
---
> cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
1041c1066,1067
< if (pkt->cacheResponding()) {
---
> if (pkt->cacheResponding() && !pkt->isClean()) {
> assert(!pkt->req->isCacheInvalidate());
1061a1088,1099
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty
> // block. If a dirty block is encountered a WriteClean
> // will update any copies to the path to the memory
> // until the point of reference.
> DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> writebacks.push_back(wb_pkt);
> pkt->setSatisfied();
> }
>
1347c1385,1386
< if (!pkt->hasSharers() && (is_fill || valid_blk)) {
---
> if (!pkt->hasSharers() &&
> (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1362a1402,1407
> // The block was marked as not readable while there was a pending
> // cache maintenance operation, restore its flag.
> if (pkt->isClean() && !is_invalidate && valid_blk) {
> blk->status |= BlkReadable;
> }
>
1505,1507c1550,1552
< // only invalidating, but more specifically an InvalidateResp, the
< // MSHR was created due to an InvalidateReq and a cache above is
< // waiting to satisfy a WriteLineReq. In this case even an
---
> // only invalidating, but more specifically an InvalidateResp and
> // the MSHR was created due to an InvalidateReq then a cache above
> // is waiting to satisfy a WriteLineReq. In this case even an
1512,1514c1557,1559
< assert(!(is_invalidate &&
< pkt->cmd != MemCmd::InvalidateResp &&
< !mshr->hasPostInvalidate()));
---
> assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
> pkt->req->isCacheMaintenance() ||
> mshr->hasPostInvalidate());
2029a2075,2079
> // If the request was satisfied by snooping the cache
> // above, mark the original packet as satisfied too.
> if (snoopPkt.satisfied()) {
> pkt->setSatisfied();
> }
2040c2090,2111
< if (!blk || !blk->isValid()) {
---
> bool respond = false;
> bool blk_valid = blk && blk->isValid();
> if (pkt->isClean()) {
> if (blk_valid && blk->isDirty()) {
> DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> PacketList writebacks;
> writebacks.push_back(wb_pkt);
>
> if (is_timing) {
> // anything that is merely forwarded pays for the forward
> // latency and the delay provided by the crossbar
> Tick forward_time = clockEdge(forwardLatency) +
> pkt->headerDelay;
> doWritebacks(writebacks, forward_time);
> } else {
> doWritebacksAtomic(writebacks);
> }
> pkt->setSatisfied();
> }
> } else if (!blk_valid) {
2059d2129
< }
2061,2063c2131,2138
< chatty_assert(!(isReadOnly && blk->isDirty()),
< "Should never have a dirty block in a read-only cache %s\n",
< name());
---
> // We may end up modifying both the block state and the packet (if
> // we respond in atomic mode), so just figure out what to do now
> // and then do it later. We respond to all snoops that need
> // responses provided we have the block in dirty state. The
> // invalidation itself is taken care of below. We don't respond to
> // cache maintenance operations as this is done by the destination
> // xbar.
> respond = blk->isDirty() && pkt->needsResponse();
2065,2071c2140,2142
< // We may end up modifying both the block state and the packet (if
< // we respond in atomic mode), so just figure out what to do now
< // and then do it later. We respond to all snoops that need
< // responses provided we have the block in dirty state. The
< // invalidation itself is taken care of below.
< bool respond = blk->isDirty() && pkt->needsResponse();
< bool have_writable = blk->isWritable();
---
> chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
> "a dirty block in a read-only cache %s\n", name());
> }
2095a2167
> DPRINTF(Cache, "new state is %s\n", blk->print());
2103c2175
< if (have_writable) {
---
> if (!pkt->isClean() && blk->isWritable()) {
2152c2224
< if (invalidate) {
---
> if (blk_valid && invalidate) {
2153a2226
> DPRINTF(Cache, "new state is %s\n", blk->print());
2156,2157d2228
< DPRINTF(Cache, "new state is %s\n", blk->print());
<
2197a2269,2275
> // Bypass any existing cache maintenance requests if the request
> // has been satisfied already (i.e., the dirty block has been
> // found).
> if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
> return;
> }
>
2262c2340
< if (invalidate) {
---
> if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2530a2609,2615
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty block. Mark
> // the packet so that the destination xbar can determine that
> // there will be a follow-up write packet as well.
> pkt->setSatisfied();
> }
>
2553a2639,2651
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty
> // block. If a dirty block is encountered a WriteClean
> // will update any copies to the path to the memory
> // until the point of reference.
> DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> PacketList writebacks;
> writebacks.push_back(wb_pkt);
> doWritebacks(writebacks, 0);
> }
>
> * Nikos Nikoleris
324a326,328
> if (pkt->req->isCacheMaintenance()) {
> // A cache maintenance operation is always forwarded to the
> // memory below even if the block is found in dirty state.
325a330,335
> // We defer any changes to the state of the block until we
> // create and mark as in service the mshr for the downstream
> // packet.
> return false;
> }
>
652a663,666
> // Cache maintenance operations have to visit all the caches down
> // to the specified xbar (PoC, PoU, etc.). Even if a cache above
> // is responding we forward the packet to the memory below rather
> // than creating an express snoop.
766c780,781
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch()) {
> assert(!pkt->req->isCacheMaintenance());
767a783
> }
857a874,880
> } else if (pkt->cmd == MemCmd::WriteClean) {
> // A WriteClean should never coalesce with any
> // outstanding cache maintenance requests.
>
> // We use forward_time here because there is an
> // uncached memory write, forwarded to WriteBuffer.
> allocateWriteBuffer(pkt, forward_time);
891c914,915
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch() &&
> !pkt->req->isCacheMaintenance())
929,930c953,954
< assert(pkt->needsWritable());
< assert(!blk->isWritable());
---
> assert((pkt->needsWritable() && !blk->isWritable()) ||
> pkt->req->isCacheMaintenance());
941c965,966
< if (!pkt->cmd.isSWPrefetch())
---
> if (!pkt->cmd.isSWPrefetch() &&
> !pkt->req->isCacheMaintenance())
964c989
< cpu_pkt->cmd == MemCmd::InvalidateReq) {
---
> cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
1041c1066,1067
< if (pkt->cacheResponding()) {
---
> if (pkt->cacheResponding() && !pkt->isClean()) {
> assert(!pkt->req->isCacheInvalidate());
1061a1088,1099
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty
> // block. If a dirty block is encountered a WriteClean
> // will update any copies to the path to the memory
> // until the point of reference.
> DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> writebacks.push_back(wb_pkt);
> pkt->setSatisfied();
> }
>
1347c1385,1386
< if (!pkt->hasSharers() && (is_fill || valid_blk)) {
---
> if (!pkt->hasSharers() &&
> (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1362a1402,1407
> // The block was marked as not readable while there was a pending
> // cache maintenance operation, restore its flag.
> if (pkt->isClean() && !is_invalidate && valid_blk) {
> blk->status |= BlkReadable;
> }
>
1505,1507c1550,1552
< // only invalidating, but more specifically an InvalidateResp, the
< // MSHR was created due to an InvalidateReq and a cache above is
< // waiting to satisfy a WriteLineReq. In this case even an
---
> // only invalidating, but more specifically an InvalidateResp and
> // the MSHR was created due to an InvalidateReq then a cache above
> // is waiting to satisfy a WriteLineReq. In this case even an
1512,1514c1557,1559
< assert(!(is_invalidate &&
< pkt->cmd != MemCmd::InvalidateResp &&
< !mshr->hasPostInvalidate()));
---
> assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
> pkt->req->isCacheMaintenance() ||
> mshr->hasPostInvalidate());
2029a2075,2079
> // If the request was satisfied by snooping the cache
> // above, mark the original packet as satisfied too.
> if (snoopPkt.satisfied()) {
> pkt->setSatisfied();
> }
2040c2090,2111
< if (!blk || !blk->isValid()) {
---
> bool respond = false;
> bool blk_valid = blk && blk->isValid();
> if (pkt->isClean()) {
> if (blk_valid && blk->isDirty()) {
> DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> PacketList writebacks;
> writebacks.push_back(wb_pkt);
>
> if (is_timing) {
> // anything that is merely forwarded pays for the forward
> // latency and the delay provided by the crossbar
> Tick forward_time = clockEdge(forwardLatency) +
> pkt->headerDelay;
> doWritebacks(writebacks, forward_time);
> } else {
> doWritebacksAtomic(writebacks);
> }
> pkt->setSatisfied();
> }
> } else if (!blk_valid) {
2059d2129
< }
2061,2063c2131,2138
< chatty_assert(!(isReadOnly && blk->isDirty()),
< "Should never have a dirty block in a read-only cache %s\n",
< name());
---
> // We may end up modifying both the block state and the packet (if
> // we respond in atomic mode), so just figure out what to do now
> // and then do it later. We respond to all snoops that need
> // responses provided we have the block in dirty state. The
> // invalidation itself is taken care of below. We don't respond to
> // cache maintenance operations as this is done by the destination
> // xbar.
> respond = blk->isDirty() && pkt->needsResponse();
2065,2071c2140,2142
< // We may end up modifying both the block state and the packet (if
< // we respond in atomic mode), so just figure out what to do now
< // and then do it later. We respond to all snoops that need
< // responses provided we have the block in dirty state. The
< // invalidation itself is taken care of below.
< bool respond = blk->isDirty() && pkt->needsResponse();
< bool have_writable = blk->isWritable();
---
> chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
> "a dirty block in a read-only cache %s\n", name());
> }
2095a2167
> DPRINTF(Cache, "new state is %s\n", blk->print());
2103c2175
< if (have_writable) {
---
> if (!pkt->isClean() && blk->isWritable()) {
2152c2224
< if (invalidate) {
---
> if (blk_valid && invalidate) {
2153a2226
> DPRINTF(Cache, "new state is %s\n", blk->print());
2156,2157d2228
< DPRINTF(Cache, "new state is %s\n", blk->print());
<
2197a2269,2275
> // Bypass any existing cache maintenance requests if the request
> // has been satisfied already (i.e., the dirty block has been
> // found).
> if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
> return;
> }
>
2262c2340
< if (invalidate) {
---
> if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2530a2609,2615
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty block. Mark
> // the packet so that the destination xbar can determine that
> // there will be a follow-up write packet as well.
> pkt->setSatisfied();
> }
>
2553a2639,2651
> if (pkt->isClean() && blk && blk->isDirty()) {
> // A cache clean opearation is looking for a dirty
> // block. If a dirty block is encountered a WriteClean
> // will update any copies to the path to the memory
> // until the point of reference.
> DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
> __func__, pkt->print(), blk->print());
> PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest());
> PacketList writebacks;
> writebacks.push_back(wb_pkt);
> doWritebacks(writebacks, 0);
> }
>