cache.cc revision 12720:8db2ee0c2cf6
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 *          Dave Greene
43 *          Nathan Binkert
44 *          Steve Reinhardt
45 *          Ron Dreslinski
46 *          Andreas Sandberg
47 *          Nikos Nikoleris
48 */
49
50/**
51 * @file
52 * Cache definitions.
53 */
54
55#include "mem/cache/cache.hh"
56
57#include "base/logging.hh"
58#include "base/types.hh"
59#include "debug/Cache.hh"
60#include "debug/CachePort.hh"
61#include "debug/CacheTags.hh"
62#include "debug/CacheVerbose.hh"
63#include "mem/cache/blk.hh"
64#include "mem/cache/mshr.hh"
65#include "mem/cache/prefetch/base.hh"
66#include "sim/sim_exit.hh"
67
68Cache::Cache(const CacheParams *p)
69    : BaseCache(p, p->system->cacheLineSize()),
70      tags(p->tags),
71      prefetcher(p->prefetcher),
72      doFastWrites(true),
73      prefetchOnAccess(p->prefetch_on_access),
74      clusivity(p->clusivity),
75      writebackClean(p->writeback_clean),
76      tempBlockWriteback(nullptr),
77      writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
78                                    name(), false,
79                                    EventBase::Delayed_Writeback_Pri)
80{
81    tempBlock = new CacheBlk();
82    tempBlock->data = new uint8_t[blkSize];
83
84    cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
85                                  "CpuSidePort");
86    memSidePort = new MemSidePort(p->name + ".mem_side", this,
87                                  "MemSidePort");
88
89    tags->setCache(this);
90    if (prefetcher)
91        prefetcher->setCache(this);
92}
93
94Cache::~Cache()
95{
96    delete [] tempBlock->data;
97    delete tempBlock;
98
99    delete cpuSidePort;
100    delete memSidePort;
101}
102
103void
104Cache::regStats()
105{
106    BaseCache::regStats();
107}
108
109void
110Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
111{
112    assert(pkt->isRequest());
113
114    uint64_t overwrite_val;
115    bool overwrite_mem;
116    uint64_t condition_val64;
117    uint32_t condition_val32;
118
119    int offset = tags->extractBlkOffset(pkt->getAddr());
120    uint8_t *blk_data = blk->data + offset;
121
122    assert(sizeof(uint64_t) >= pkt->getSize());
123
124    overwrite_mem = true;
125    // keep a copy of our possible write value, and copy what is at the
126    // memory address into the packet
127    pkt->writeData((uint8_t *)&overwrite_val);
128    pkt->setData(blk_data);
129
130    if (pkt->req->isCondSwap()) {
131        if (pkt->getSize() == sizeof(uint64_t)) {
132            condition_val64 = pkt->req->getExtraData();
133            overwrite_mem = !std::memcmp(&condition_val64, blk_data,
134                                         sizeof(uint64_t));
135        } else if (pkt->getSize() == sizeof(uint32_t)) {
136            condition_val32 = (uint32_t)pkt->req->getExtraData();
137            overwrite_mem = !std::memcmp(&condition_val32, blk_data,
138                                         sizeof(uint32_t));
139        } else
140            panic("Invalid size for conditional read/write\n");
141    }
142
143    if (overwrite_mem) {
144        std::memcpy(blk_data, &overwrite_val, pkt->getSize());
145        blk->status |= BlkDirty;
146    }
147}
148
149
150void
151Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
152                      bool deferred_response, bool pending_downgrade)
153{
154    assert(pkt->isRequest());
155
156    assert(blk && blk->isValid());
157    // Occasionally this is not true... if we are a lower-level cache
158    // satisfying a string of Read and ReadEx requests from
159    // upper-level caches, a Read will mark the block as shared but we
160    // can satisfy a following ReadEx anyway since we can rely on the
161    // Read requester(s) to have buffered the ReadEx snoop and to
162    // invalidate their blocks after receiving them.
163    // assert(!pkt->needsWritable() || blk->isWritable());
164    assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
165
166    // Check RMW operations first since both isRead() and
167    // isWrite() will be true for them
168    if (pkt->cmd == MemCmd::SwapReq) {
169        cmpAndSwap(blk, pkt);
170    } else if (pkt->isWrite()) {
171        // we have the block in a writable state and can go ahead,
172        // note that the line may be also be considered writable in
173        // downstream caches along the path to memory, but always
174        // Exclusive, and never Modified
175        assert(blk->isWritable());
176        // Write or WriteLine at the first cache with block in writable state
177        if (blk->checkWrite(pkt)) {
178            pkt->writeDataToBlock(blk->data, blkSize);
179        }
180        // Always mark the line as dirty (and thus transition to the
181        // Modified state) even if we are a failed StoreCond so we
182        // supply data to any snoops that have appended themselves to
183        // this cache before knowing the store will fail.
184        blk->status |= BlkDirty;
185        DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
186    } else if (pkt->isRead()) {
187        if (pkt->isLLSC()) {
188            blk->trackLoadLocked(pkt);
189        }
190
191        // all read responses have a data payload
192        assert(pkt->hasRespData());
193        pkt->setDataFromBlock(blk->data, blkSize);
194
195        // determine if this read is from a (coherent) cache or not
196        if (pkt->fromCache()) {
197            assert(pkt->getSize() == blkSize);
198            // special handling for coherent block requests from
199            // upper-level caches
200            if (pkt->needsWritable()) {
201                // sanity check
202                assert(pkt->cmd == MemCmd::ReadExReq ||
203                       pkt->cmd == MemCmd::SCUpgradeFailReq);
204                assert(!pkt->hasSharers());
205
206                // if we have a dirty copy, make sure the recipient
207                // keeps it marked dirty (in the modified state)
208                if (blk->isDirty()) {
209                    pkt->setCacheResponding();
210                    blk->status &= ~BlkDirty;
211                }
212            } else if (blk->isWritable() && !pending_downgrade &&
213                       !pkt->hasSharers() &&
214                       pkt->cmd != MemCmd::ReadCleanReq) {
215                // we can give the requester a writable copy on a read
216                // request if:
217                // - we have a writable copy at this level (& below)
218                // - we don't have a pending snoop from below
219                //   signaling another read request
220                // - no other cache above has a copy (otherwise it
221                //   would have set hasSharers flag when
222                //   snooping the packet)
223                // - the read has explicitly asked for a clean
224                //   copy of the line
225                if (blk->isDirty()) {
226                    // special considerations if we're owner:
227                    if (!deferred_response) {
228                        // respond with the line in Modified state
229                        // (cacheResponding set, hasSharers not set)
230                        pkt->setCacheResponding();
231
232                        // if this cache is mostly inclusive, we
233                        // keep the block in the Exclusive state,
234                        // and pass it upwards as Modified
235                        // (writable and dirty), hence we have
236                        // multiple caches, all on the same path
237                        // towards memory, all considering the
238                        // same block writable, but only one
239                        // considering it Modified
240
241                        // we get away with multiple caches (on
242                        // the same path to memory) considering
243                        // the block writeable as we always enter
244                        // the cache hierarchy through a cache,
245                        // and first snoop upwards in all other
246                        // branches
247                        blk->status &= ~BlkDirty;
248                    } else {
249                        // if we're responding after our own miss,
250                        // there's a window where the recipient didn't
251                        // know it was getting ownership and may not
252                        // have responded to snoops correctly, so we
253                        // have to respond with a shared line
254                        pkt->setHasSharers();
255                    }
256                }
257            } else {
258                // otherwise only respond with a shared copy
259                pkt->setHasSharers();
260            }
261        }
262    } else if (pkt->isUpgrade()) {
263        // sanity check
264        assert(!pkt->hasSharers());
265
266        if (blk->isDirty()) {
267            // we were in the Owned state, and a cache above us that
268            // has the line in Shared state needs to be made aware
269            // that the data it already has is in fact dirty
270            pkt->setCacheResponding();
271            blk->status &= ~BlkDirty;
272        }
273    } else {
274        assert(pkt->isInvalidate());
275        invalidateBlock(blk);
276        DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
277                pkt->print());
278    }
279}
280
281/////////////////////////////////////////////////////
282//
283// Access path: requests coming in from the CPU side
284//
285/////////////////////////////////////////////////////
286
287bool
288Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
289              PacketList &writebacks)
290{
291    // sanity check
292    assert(pkt->isRequest());
293
294    chatty_assert(!(isReadOnly && pkt->isWrite()),
295                  "Should never see a write in a read-only cache %s\n",
296                  name());
297
298    DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
299
300    if (pkt->req->isUncacheable()) {
301        DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
302
303        // flush and invalidate any existing block
304        CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
305        if (old_blk && old_blk->isValid()) {
306            if (old_blk->isDirty() || writebackClean)
307                writebacks.push_back(writebackBlk(old_blk));
308            else
309                writebacks.push_back(cleanEvictBlk(old_blk));
310            invalidateBlock(old_blk);
311        }
312
313        blk = nullptr;
314        // lookupLatency is the latency in case the request is uncacheable.
315        lat = lookupLatency;
316        return false;
317    }
318
319    // Here lat is the value passed as parameter to accessBlock() function
320    // that can modify its value.
321    blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
322
323    DPRINTF(Cache, "%s %s\n", pkt->print(),
324            blk ? "hit " + blk->print() : "miss");
325
326    if (pkt->req->isCacheMaintenance()) {
327        // A cache maintenance operation is always forwarded to the
328        // memory below even if the block is found in dirty state.
329
330        // We defer any changes to the state of the block until we
331        // create and mark as in service the mshr for the downstream
332        // packet.
333        return false;
334    }
335
336    if (pkt->isEviction()) {
337        // We check for presence of block in above caches before issuing
338        // Writeback or CleanEvict to write buffer. Therefore the only
339        // possible cases can be of a CleanEvict packet coming from above
340        // encountering a Writeback generated in this cache peer cache and
341        // waiting in the write buffer. Cases of upper level peer caches
342        // generating CleanEvict and Writeback or simply CleanEvict and
343        // CleanEvict almost simultaneously will be caught by snoops sent out
344        // by crossbar.
345        WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
346                                                          pkt->isSecure());
347        if (wb_entry) {
348            assert(wb_entry->getNumTargets() == 1);
349            PacketPtr wbPkt = wb_entry->getTarget()->pkt;
350            assert(wbPkt->isWriteback());
351
352            if (pkt->isCleanEviction()) {
353                // The CleanEvict and WritebackClean snoops into other
354                // peer caches of the same level while traversing the
355                // crossbar. If a copy of the block is found, the
356                // packet is deleted in the crossbar. Hence, none of
357                // the other upper level caches connected to this
358                // cache have the block, so we can clear the
359                // BLOCK_CACHED flag in the Writeback if set and
360                // discard the CleanEvict by returning true.
361                wbPkt->clearBlockCached();
362                return true;
363            } else {
364                assert(pkt->cmd == MemCmd::WritebackDirty);
365                // Dirty writeback from above trumps our clean
366                // writeback... discard here
367                // Note: markInService will remove entry from writeback buffer.
368                markInService(wb_entry);
369                delete wbPkt;
370            }
371        }
372    }
373
374    // Writeback handling is special case.  We can write the block into
375    // the cache without having a writeable copy (or any copy at all).
376    if (pkt->isWriteback()) {
377        assert(blkSize == pkt->getSize());
378
379        // we could get a clean writeback while we are having
380        // outstanding accesses to a block, do the simple thing for
381        // now and drop the clean writeback so that we do not upset
382        // any ordering/decisions about ownership already taken
383        if (pkt->cmd == MemCmd::WritebackClean &&
384            mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
385            DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
386                    "dropping\n", pkt->getAddr());
387            return true;
388        }
389
390        if (blk == nullptr) {
391            // need to do a replacement
392            blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
393            if (blk == nullptr) {
394                // no replaceable block available: give up, fwd to next level.
395                incMissCount(pkt);
396                return false;
397            }
398            tags->insertBlock(pkt, blk);
399
400            blk->status |= (BlkValid | BlkReadable);
401        }
402        // only mark the block dirty if we got a writeback command,
403        // and leave it as is for a clean writeback
404        if (pkt->cmd == MemCmd::WritebackDirty) {
405            assert(!blk->isDirty());
406            blk->status |= BlkDirty;
407        }
408        // if the packet does not have sharers, it is passing
409        // writable, and we got the writeback in Modified or Exclusive
410        // state, if not we are in the Owned or Shared state
411        if (!pkt->hasSharers()) {
412            blk->status |= BlkWritable;
413        }
414        // nothing else to do; writeback doesn't expect response
415        assert(!pkt->needsResponse());
416        pkt->writeDataToBlock(blk->data, blkSize);
417        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
418        incHitCount(pkt);
419        // populate the time when the block will be ready to access.
420        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
421            pkt->payloadDelay;
422        return true;
423    } else if (pkt->cmd == MemCmd::CleanEvict) {
424        if (blk != nullptr) {
425            // Found the block in the tags, need to stop CleanEvict from
426            // propagating further down the hierarchy. Returning true will
427            // treat the CleanEvict like a satisfied write request and delete
428            // it.
429            return true;
430        }
431        // We didn't find the block here, propagate the CleanEvict further
432        // down the memory hierarchy. Returning false will treat the CleanEvict
433        // like a Writeback which could not find a replaceable block so has to
434        // go to next level.
435        return false;
436    } else if (pkt->cmd == MemCmd::WriteClean) {
437        // WriteClean handling is a special case. We can allocate a
438        // block directly if it doesn't exist and we can update the
439        // block immediately. The WriteClean transfers the ownership
440        // of the block as well.
441        assert(blkSize == pkt->getSize());
442
443        if (!blk) {
444            if (pkt->writeThrough()) {
445                // if this is a write through packet, we don't try to
446                // allocate if the block is not present
447                return false;
448            } else {
449                // a writeback that misses needs to allocate a new block
450                blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
451                                    writebacks);
452                if (!blk) {
453                    // no replaceable block available: give up, fwd to
454                    // next level.
455                    incMissCount(pkt);
456                    return false;
457                }
458                tags->insertBlock(pkt, blk);
459
460                blk->status |= (BlkValid | BlkReadable);
461            }
462        }
463
464        // at this point either this is a writeback or a write-through
465        // write clean operation and the block is already in this
466        // cache, we need to update the data and the block flags
467        assert(blk);
468        assert(!blk->isDirty());
469        if (!pkt->writeThrough()) {
470            blk->status |= BlkDirty;
471        }
472        // nothing else to do; writeback doesn't expect response
473        assert(!pkt->needsResponse());
474        pkt->writeDataToBlock(blk->data, blkSize);
475        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
476
477        incHitCount(pkt);
478        // populate the time when the block will be ready to access.
479        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
480            pkt->payloadDelay;
481        // if this a write-through packet it will be sent to cache
482        // below
483        return !pkt->writeThrough();
484    } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
485                       blk->isReadable())) {
486        // OK to satisfy access
487        incHitCount(pkt);
488        satisfyRequest(pkt, blk);
489        maintainClusivity(pkt->fromCache(), blk);
490
491        return true;
492    }
493
494    // Can't satisfy access normally... either no block (blk == nullptr)
495    // or have block but need writable
496
497    incMissCount(pkt);
498
499    if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
500        // complete miss on store conditional... just give up now
501        pkt->req->setExtraData(0);
502        return true;
503    }
504
505    return false;
506}
507
508void
509Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
510{
511    if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
512        clusivity == Enums::mostly_excl) {
513        // if we have responded to a cache, and our block is still
514        // valid, but not dirty, and this cache is mostly exclusive
515        // with respect to the cache above, drop the block
516        invalidateBlock(blk);
517    }
518}
519
520void
521Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
522{
523    while (!writebacks.empty()) {
524        PacketPtr wbPkt = writebacks.front();
525        // We use forwardLatency here because we are copying writebacks to
526        // write buffer.
527
528        // Call isCachedAbove for Writebacks, CleanEvicts and
529        // WriteCleans to discover if the block is cached above.
530        if (isCachedAbove(wbPkt)) {
531            if (wbPkt->cmd == MemCmd::CleanEvict) {
532                // Delete CleanEvict because cached copies exist above. The
533                // packet destructor will delete the request object because
534                // this is a non-snoop request packet which does not require a
535                // response.
536                delete wbPkt;
537            } else if (wbPkt->cmd == MemCmd::WritebackClean) {
538                // clean writeback, do not send since the block is
539                // still cached above
540                assert(writebackClean);
541                delete wbPkt;
542            } else {
543                assert(wbPkt->cmd == MemCmd::WritebackDirty ||
544                       wbPkt->cmd == MemCmd::WriteClean);
545                // Set BLOCK_CACHED flag in Writeback and send below, so that
546                // the Writeback does not reset the bit corresponding to this
547                // address in the snoop filter below.
548                wbPkt->setBlockCached();
549                allocateWriteBuffer(wbPkt, forward_time);
550            }
551        } else {
552            // If the block is not cached above, send packet below. Both
553            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
554            // reset the bit corresponding to this address in the snoop filter
555            // below.
556            allocateWriteBuffer(wbPkt, forward_time);
557        }
558        writebacks.pop_front();
559    }
560}
561
562void
563Cache::doWritebacksAtomic(PacketList& writebacks)
564{
565    while (!writebacks.empty()) {
566        PacketPtr wbPkt = writebacks.front();
567        // Call isCachedAbove for both Writebacks and CleanEvicts. If
568        // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
569        // and discard CleanEvicts.
570        if (isCachedAbove(wbPkt, false)) {
571            if (wbPkt->cmd == MemCmd::WritebackDirty ||
572                wbPkt->cmd == MemCmd::WriteClean) {
573                // Set BLOCK_CACHED flag in Writeback and send below,
574                // so that the Writeback does not reset the bit
575                // corresponding to this address in the snoop filter
576                // below. We can discard CleanEvicts because cached
577                // copies exist above. Atomic mode isCachedAbove
578                // modifies packet to set BLOCK_CACHED flag
579                memSidePort->sendAtomic(wbPkt);
580            }
581        } else {
582            // If the block is not cached above, send packet below. Both
583            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
584            // reset the bit corresponding to this address in the snoop filter
585            // below.
586            memSidePort->sendAtomic(wbPkt);
587        }
588        writebacks.pop_front();
589        // In case of CleanEvicts, the packet destructor will delete the
590        // request object because this is a non-snoop request packet which
591        // does not require a response.
592        delete wbPkt;
593    }
594}
595
596
597void
598Cache::recvTimingSnoopResp(PacketPtr pkt)
599{
600    DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
601
602    assert(pkt->isResponse());
603    assert(!system->bypassCaches());
604
605    // determine if the response is from a snoop request we created
606    // (in which case it should be in the outstandingSnoop), or if we
607    // merely forwarded someone else's snoop request
608    const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
609        outstandingSnoop.end();
610
611    if (!forwardAsSnoop) {
612        // the packet came from this cache, so sink it here and do not
613        // forward it
614        assert(pkt->cmd == MemCmd::HardPFResp);
615
616        outstandingSnoop.erase(pkt->req);
617
618        DPRINTF(Cache, "Got prefetch response from above for addr "
619                "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
620        recvTimingResp(pkt);
621        return;
622    }
623
624    // forwardLatency is set here because there is a response from an
625    // upper level cache.
626    // To pay the delay that occurs if the packet comes from the bus,
627    // we charge also headerDelay.
628    Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
629    // Reset the timing of the packet.
630    pkt->headerDelay = pkt->payloadDelay = 0;
631    memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
632}
633
634void
635Cache::promoteWholeLineWrites(PacketPtr pkt)
636{
637    // Cache line clearing instructions
638    if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
639        (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
640        pkt->cmd = MemCmd::WriteLineReq;
641        DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
642    }
643}
644
645void
646Cache::handleTimingReqHit(PacketPtr pkt, CacheBlk *blk, Tick request_time)
647{
648    // should never be satisfying an uncacheable access as we
649    // flush and invalidate any existing block as part of the
650    // lookup
651    assert(!pkt->req->isUncacheable());
652
653    if (pkt->needsResponse()) {
654        pkt->makeTimingResponse();
655        // @todo: Make someone pay for this
656        pkt->headerDelay = pkt->payloadDelay = 0;
657
658        // In this case we are considering request_time that takes
659        // into account the delay of the xbar, if any, and just
660        // lat, neglecting responseLatency, modelling hit latency
661        // just as lookupLatency or or the value of lat overriden
662        // by access(), that calls accessBlock() function.
663        cpuSidePort->schedTimingResp(pkt, request_time, true);
664    } else {
665        DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
666                pkt->print());
667
668        // queue the packet for deletion, as the sending cache is
669        // still relying on it; if the block is found in access(),
670        // CleanEvict and Writeback messages will be deleted
671        // here as well
672        pendingDelete.reset(pkt);
673    }
674}
675
676void
677Cache::handleTimingReqMiss(PacketPtr pkt, CacheBlk *blk, Tick forward_time,
678                           Tick request_time)
679{
680    Addr blk_addr = pkt->getBlockAddr(blkSize);
681
682    // ignore any existing MSHR if we are dealing with an
683    // uncacheable request
684    MSHR *mshr = pkt->req->isUncacheable() ? nullptr :
685        mshrQueue.findMatch(blk_addr, pkt->isSecure());
686
687    // Software prefetch handling:
688    // To keep the core from waiting on data it won't look at
689    // anyway, send back a response with dummy data. Miss handling
690    // will continue asynchronously. Unfortunately, the core will
691    // insist upon freeing original Packet/Request, so we have to
692    // create a new pair with a different lifecycle. Note that this
693    // processing happens before any MSHR munging on the behalf of
694    // this request because this new Request will be the one stored
695    // into the MSHRs, not the original.
696    if (pkt->cmd.isSWPrefetch()) {
697        assert(pkt->needsResponse());
698        assert(pkt->req->hasPaddr());
699        assert(!pkt->req->isUncacheable());
700
701        // There's no reason to add a prefetch as an additional target
702        // to an existing MSHR. If an outstanding request is already
703        // in progress, there is nothing for the prefetch to do.
704        // If this is the case, we don't even create a request at all.
705        PacketPtr pf = nullptr;
706
707        if (!mshr) {
708            // copy the request and create a new SoftPFReq packet
709            RequestPtr req = new Request(pkt->req->getPaddr(),
710                                         pkt->req->getSize(),
711                                         pkt->req->getFlags(),
712                                         pkt->req->masterId());
713            pf = new Packet(req, pkt->cmd);
714            pf->allocate();
715            assert(pf->getAddr() == pkt->getAddr());
716            assert(pf->getSize() == pkt->getSize());
717        }
718
719        pkt->makeTimingResponse();
720
721        // request_time is used here, taking into account lat and the delay
722        // charged if the packet comes from the xbar.
723        cpuSidePort->schedTimingResp(pkt, request_time, true);
724
725        // If an outstanding request is in progress (we found an
726        // MSHR) this is set to null
727        pkt = pf;
728    }
729
730    if (mshr) {
731        /// MSHR hit
732        /// @note writebacks will be checked in getNextMSHR()
733        /// for any conflicting requests to the same block
734
735        //@todo remove hw_pf here
736
737        // Coalesce unless it was a software prefetch (see above).
738        if (pkt) {
739            assert(!pkt->isWriteback());
740            // CleanEvicts corresponding to blocks which have
741            // outstanding requests in MSHRs are simply sunk here
742            if (pkt->cmd == MemCmd::CleanEvict) {
743                pendingDelete.reset(pkt);
744            } else if (pkt->cmd == MemCmd::WriteClean) {
745                // A WriteClean should never coalesce with any
746                // outstanding cache maintenance requests.
747
748                // We use forward_time here because there is an
749                // uncached memory write, forwarded to WriteBuffer.
750                allocateWriteBuffer(pkt, forward_time);
751            } else {
752                DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
753                        pkt->print());
754
755                assert(pkt->req->masterId() < system->maxMasters());
756                mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
757
758                // uncacheable accesses always allocate a new
759                // MSHR, and cacheable accesses ignore any
760                // uncacheable MSHRs, thus we should never have
761                // targets addded if originally allocated
762                // uncacheable
763                assert(!mshr->isUncacheable());
764
765                // We use forward_time here because it is the same
766                // considering new targets. We have multiple
767                // requests for the same address here. It
768                // specifies the latency to allocate an internal
769                // buffer and to schedule an event to the queued
770                // port and also takes into account the additional
771                // delay of the xbar.
772                mshr->allocateTarget(pkt, forward_time, order++,
773                                     allocOnFill(pkt->cmd));
774                if (mshr->getNumTargets() == numTarget) {
775                    noTargetMSHR = mshr;
776                    setBlocked(Blocked_NoTargets);
777                    // need to be careful with this... if this mshr isn't
778                    // ready yet (i.e. time > curTick()), we don't want to
779                    // move it ahead of mshrs that are ready
780                    // mshrQueue.moveToFront(mshr);
781                }
782            }
783        }
784    } else {
785        // no MSHR
786        assert(pkt->req->masterId() < system->maxMasters());
787        if (pkt->req->isUncacheable()) {
788            mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
789        } else {
790            mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
791        }
792
793        if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
794            (pkt->req->isUncacheable() && pkt->isWrite())) {
795            // We use forward_time here because there is an
796            // uncached memory write, forwarded to WriteBuffer.
797            allocateWriteBuffer(pkt, forward_time);
798        } else {
799            if (blk && blk->isValid()) {
800                // should have flushed and have no valid block
801                assert(!pkt->req->isUncacheable());
802
803                // If we have a write miss to a valid block, we
804                // need to mark the block non-readable.  Otherwise
805                // if we allow reads while there's an outstanding
806                // write miss, the read could return stale data
807                // out of the cache block... a more aggressive
808                // system could detect the overlap (if any) and
809                // forward data out of the MSHRs, but we don't do
810                // that yet.  Note that we do need to leave the
811                // block valid so that it stays in the cache, in
812                // case we get an upgrade response (and hence no
813                // new data) when the write miss completes.
814                // As long as CPUs do proper store/load forwarding
815                // internally, and have a sufficiently weak memory
816                // model, this is probably unnecessary, but at some
817                // point it must have seemed like we needed it...
818                assert((pkt->needsWritable() && !blk->isWritable()) ||
819                       pkt->req->isCacheMaintenance());
820                blk->status &= ~BlkReadable;
821            }
822            // Here we are using forward_time, modelling the latency of
823            // a miss (outbound) just as forwardLatency, neglecting the
824            // lookupLatency component.
825            allocateMissBuffer(pkt, forward_time);
826        }
827    }
828}
829
830void
831Cache::recvTimingReq(PacketPtr pkt)
832{
833    DPRINTF(CacheTags, "%s tags:\n%s\n", __func__, tags->print());
834
835    assert(pkt->isRequest());
836
837    // Just forward the packet if caches are disabled.
838    if (system->bypassCaches()) {
839        // @todo This should really enqueue the packet rather
840        bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
841        assert(success);
842        return;
843    }
844
845    promoteWholeLineWrites(pkt);
846
847    // Cache maintenance operations have to visit all the caches down
848    // to the specified xbar (PoC, PoU, etc.). Even if a cache above
849    // is responding we forward the packet to the memory below rather
850    // than creating an express snoop.
851    if (pkt->cacheResponding()) {
852        // a cache above us (but not where the packet came from) is
853        // responding to the request, in other words it has the line
854        // in Modified or Owned state
855        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
856                pkt->print());
857
858        // if the packet needs the block to be writable, and the cache
859        // that has promised to respond (setting the cache responding
860        // flag) is not providing writable (it is in Owned rather than
861        // the Modified state), we know that there may be other Shared
862        // copies in the system; go out and invalidate them all
863        assert(pkt->needsWritable() && !pkt->responderHadWritable());
864
865        // an upstream cache that had the line in Owned state
866        // (dirty, but not writable), is responding and thus
867        // transferring the dirty line from one branch of the
868        // cache hierarchy to another
869
870        // send out an express snoop and invalidate all other
871        // copies (snooping a packet that needs writable is the
872        // same as an invalidation), thus turning the Owned line
873        // into a Modified line, note that we don't invalidate the
874        // block in the current cache or any other cache on the
875        // path to memory
876
877        // create a downstream express snoop with cleared packet
878        // flags, there is no need to allocate any data as the
879        // packet is merely used to co-ordinate state transitions
880        Packet *snoop_pkt = new Packet(pkt, true, false);
881
882        // also reset the bus time that the original packet has
883        // not yet paid for
884        snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
885
886        // make this an instantaneous express snoop, and let the
887        // other caches in the system know that the another cache
888        // is responding, because we have found the authorative
889        // copy (Modified or Owned) that will supply the right
890        // data
891        snoop_pkt->setExpressSnoop();
892        snoop_pkt->setCacheResponding();
893
894        // this express snoop travels towards the memory, and at
895        // every crossbar it is snooped upwards thus reaching
896        // every cache in the system
897        bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt);
898        // express snoops always succeed
899        assert(success);
900
901        // main memory will delete the snoop packet
902
903        // queue for deletion, as opposed to immediate deletion, as
904        // the sending cache is still relying on the packet
905        pendingDelete.reset(pkt);
906
907        // no need to take any further action in this particular cache
908        // as an upstram cache has already committed to responding,
909        // and we have already sent out any express snoops in the
910        // section above to ensure all other copies in the system are
911        // invalidated
912        return;
913    }
914
915    // anything that is merely forwarded pays for the forward latency and
916    // the delay provided by the crossbar
917    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
918
919    // We use lookupLatency here because it is used to specify the latency
920    // to access.
921    Cycles lat = lookupLatency;
922    CacheBlk *blk = nullptr;
923    bool satisfied = false;
924    {
925        PacketList writebacks;
926        // Note that lat is passed by reference here. The function
927        // access() calls accessBlock() which can modify lat value.
928        satisfied = access(pkt, blk, lat, writebacks);
929
930        // copy writebacks to write buffer here to ensure they logically
931        // proceed anything happening below
932        doWritebacks(writebacks, forward_time);
933    }
934
935    // Here we charge the headerDelay that takes into account the latencies
936    // of the bus, if the packet comes from it.
937    // The latency charged it is just lat that is the value of lookupLatency
938    // modified by access() function, or if not just lookupLatency.
939    // In case of a hit we are neglecting response latency.
940    // In case of a miss we are neglecting forward latency.
941    Tick request_time = clockEdge(lat) + pkt->headerDelay;
942    // Here we reset the timing of the packet.
943    pkt->headerDelay = pkt->payloadDelay = 0;
944
945    // track time of availability of next prefetch, if any
946    Tick next_pf_time = MaxTick;
947
948    if (satisfied) {
949        // if need to notify the prefetcher we need to do it anything
950        // else, handleTimingReqHit might turn the packet into a
951        // response
952        if (prefetcher &&
953            (prefetchOnAccess || (blk && blk->wasPrefetched()))) {
954            if (blk)
955                blk->status &= ~BlkHWPrefetched;
956
957            // Don't notify on SWPrefetch
958            if (!pkt->cmd.isSWPrefetch()) {
959                assert(!pkt->req->isCacheMaintenance());
960                next_pf_time = prefetcher->notify(pkt);
961            }
962        }
963
964        handleTimingReqHit(pkt, blk, request_time);
965    } else {
966        handleTimingReqMiss(pkt, blk, forward_time, request_time);
967
968        // We should call the prefetcher reguardless if the request is
969        // satisfied or not, reguardless if the request is in the MSHR
970        // or not.  The request could be a ReadReq hit, but still not
971        // satisfied (potentially because of a prior write to the same
972        // cache line.  So, even when not satisfied, there is an MSHR
973        // already allocated for this, we need to let the prefetcher
974        // know about the request
975        if (prefetcher && pkt &&
976            !pkt->cmd.isSWPrefetch() &&
977            !pkt->req->isCacheMaintenance()) {
978            next_pf_time = prefetcher->notify(pkt);
979        }
980    }
981
982    if (next_pf_time != MaxTick)
983        schedMemSideSendEvent(next_pf_time);
984}
985
986PacketPtr
987Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
988                        bool needsWritable) const
989{
990    // should never see evictions here
991    assert(!cpu_pkt->isEviction());
992
993    bool blkValid = blk && blk->isValid();
994
995    if (cpu_pkt->req->isUncacheable() ||
996        (!blkValid && cpu_pkt->isUpgrade()) ||
997        cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
998        // uncacheable requests and upgrades from upper-level caches
999        // that missed completely just go through as is
1000        return nullptr;
1001    }
1002
1003    assert(cpu_pkt->needsResponse());
1004
1005    MemCmd cmd;
1006    // @TODO make useUpgrades a parameter.
1007    // Note that ownership protocols require upgrade, otherwise a
1008    // write miss on a shared owned block will generate a ReadExcl,
1009    // which will clobber the owned copy.
1010    const bool useUpgrades = true;
1011    if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
1012        assert(!blkValid || !blk->isWritable());
1013        // forward as invalidate to all other caches, this gives us
1014        // the line in Exclusive state, and invalidates all other
1015        // copies
1016        cmd = MemCmd::InvalidateReq;
1017    } else if (blkValid && useUpgrades) {
1018        // only reason to be here is that blk is read only and we need
1019        // it to be writable
1020        assert(needsWritable);
1021        assert(!blk->isWritable());
1022        cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
1023    } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
1024               cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
1025        // Even though this SC will fail, we still need to send out the
1026        // request and get the data to supply it to other snoopers in the case
1027        // where the determination the StoreCond fails is delayed due to
1028        // all caches not being on the same local bus.
1029        cmd = MemCmd::SCUpgradeFailReq;
1030    } else {
1031        // block is invalid
1032
1033        // If the request does not need a writable there are two cases
1034        // where we need to ensure the response will not fetch the
1035        // block in dirty state:
1036        // * this cache is read only and it does not perform
1037        //   writebacks,
1038        // * this cache is mostly exclusive and will not fill (since
1039        //   it does not fill it will have to writeback the dirty data
1040        //   immediately which generates uneccesary writebacks).
1041        bool force_clean_rsp = isReadOnly || clusivity == Enums::mostly_excl;
1042        cmd = needsWritable ? MemCmd::ReadExReq :
1043            (force_clean_rsp ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1044    }
1045    PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1046
1047    // if there are upstream caches that have already marked the
1048    // packet as having sharers (not passing writable), pass that info
1049    // downstream
1050    if (cpu_pkt->hasSharers() && !needsWritable) {
1051        // note that cpu_pkt may have spent a considerable time in the
1052        // MSHR queue and that the information could possibly be out
1053        // of date, however, there is no harm in conservatively
1054        // assuming the block has sharers
1055        pkt->setHasSharers();
1056        DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1057                __func__, cpu_pkt->print(), pkt->print());
1058    }
1059
1060    // the packet should be block aligned
1061    assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1062
1063    pkt->allocate();
1064    DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1065            cpu_pkt->print());
1066    return pkt;
1067}
1068
1069
1070Tick
1071Cache::recvAtomic(PacketPtr pkt)
1072{
1073    // We are in atomic mode so we pay just for lookupLatency here.
1074    Cycles lat = lookupLatency;
1075
1076    // Forward the request if the system is in cache bypass mode.
1077    if (system->bypassCaches())
1078        return ticksToCycles(memSidePort->sendAtomic(pkt));
1079
1080    promoteWholeLineWrites(pkt);
1081
1082    // follow the same flow as in recvTimingReq, and check if a cache
1083    // above us is responding
1084    if (pkt->cacheResponding() && !pkt->isClean()) {
1085        assert(!pkt->req->isCacheInvalidate());
1086        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1087                pkt->print());
1088
1089        // if a cache is responding, and it had the line in Owned
1090        // rather than Modified state, we need to invalidate any
1091        // copies that are not on the same path to memory
1092        assert(pkt->needsWritable() && !pkt->responderHadWritable());
1093        lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1094
1095        return lat * clockPeriod();
1096    }
1097
1098    // should assert here that there are no outstanding MSHRs or
1099    // writebacks... that would mean that someone used an atomic
1100    // access in timing mode
1101
1102    CacheBlk *blk = nullptr;
1103    PacketList writebacks;
1104    bool satisfied = access(pkt, blk, lat, writebacks);
1105
1106    if (pkt->isClean() && blk && blk->isDirty()) {
1107        // A cache clean opearation is looking for a dirty
1108        // block. If a dirty block is encountered a WriteClean
1109        // will update any copies to the path to the memory
1110        // until the point of reference.
1111        DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1112                __func__, pkt->print(), blk->print());
1113        PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
1114        writebacks.push_back(wb_pkt);
1115        pkt->setSatisfied();
1116    }
1117
1118    // handle writebacks resulting from the access here to ensure they
1119    // logically proceed anything happening below
1120    doWritebacksAtomic(writebacks);
1121
1122    if (!satisfied) {
1123        // MISS
1124
1125        // deal with the packets that go through the write path of
1126        // the cache, i.e. any evictions and writes
1127        if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1128            (pkt->req->isUncacheable() && pkt->isWrite())) {
1129            lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1130            return lat * clockPeriod();
1131        }
1132        // only misses left
1133
1134        PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1135
1136        bool is_forward = (bus_pkt == nullptr);
1137
1138        if (is_forward) {
1139            // just forwarding the same request to the next level
1140            // no local cache operation involved
1141            bus_pkt = pkt;
1142        }
1143
1144        DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1145                bus_pkt->print());
1146
1147#if TRACING_ON
1148        CacheBlk::State old_state = blk ? blk->status : 0;
1149#endif
1150
1151        lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1152
1153        bool is_invalidate = bus_pkt->isInvalidate();
1154
1155        // We are now dealing with the response handling
1156        DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1157                bus_pkt->print(), old_state);
1158
1159        // If packet was a forward, the response (if any) is already
1160        // in place in the bus_pkt == pkt structure, so we don't need
1161        // to do anything.  Otherwise, use the separate bus_pkt to
1162        // generate response to pkt and then delete it.
1163        if (!is_forward) {
1164            if (pkt->needsResponse()) {
1165                assert(bus_pkt->isResponse());
1166                if (bus_pkt->isError()) {
1167                    pkt->makeAtomicResponse();
1168                    pkt->copyError(bus_pkt);
1169                } else if (pkt->cmd == MemCmd::WriteLineReq) {
1170                    // note the use of pkt, not bus_pkt here.
1171
1172                    // write-line request to the cache that promoted
1173                    // the write to a whole line
1174                    blk = handleFill(pkt, blk, writebacks,
1175                                     allocOnFill(pkt->cmd));
1176                    assert(blk != NULL);
1177                    is_invalidate = false;
1178                    satisfyRequest(pkt, blk);
1179                } else if (bus_pkt->isRead() ||
1180                           bus_pkt->cmd == MemCmd::UpgradeResp) {
1181                    // we're updating cache state to allow us to
1182                    // satisfy the upstream request from the cache
1183                    blk = handleFill(bus_pkt, blk, writebacks,
1184                                     allocOnFill(pkt->cmd));
1185                    satisfyRequest(pkt, blk);
1186                    maintainClusivity(pkt->fromCache(), blk);
1187                } else {
1188                    // we're satisfying the upstream request without
1189                    // modifying cache state, e.g., a write-through
1190                    pkt->makeAtomicResponse();
1191                }
1192            }
1193            delete bus_pkt;
1194        }
1195
1196        if (is_invalidate && blk && blk->isValid()) {
1197            invalidateBlock(blk);
1198        }
1199    }
1200
1201    // Note that we don't invoke the prefetcher at all in atomic mode.
1202    // It's not clear how to do it properly, particularly for
1203    // prefetchers that aggressively generate prefetch candidates and
1204    // rely on bandwidth contention to throttle them; these will tend
1205    // to pollute the cache in atomic mode since there is no bandwidth
1206    // contention.  If we ever do want to enable prefetching in atomic
1207    // mode, though, this is the place to do it... see timingAccess()
1208    // for an example (though we'd want to issue the prefetch(es)
1209    // immediately rather than calling requestMemSideBus() as we do
1210    // there).
1211
1212    // do any writebacks resulting from the response handling
1213    doWritebacksAtomic(writebacks);
1214
1215    // if we used temp block, check to see if its valid and if so
1216    // clear it out, but only do so after the call to recvAtomic is
1217    // finished so that any downstream observers (such as a snoop
1218    // filter), first see the fill, and only then see the eviction
1219    if (blk == tempBlock && tempBlock->isValid()) {
1220        // the atomic CPU calls recvAtomic for fetch and load/store
1221        // sequentuially, and we may already have a tempBlock
1222        // writeback from the fetch that we have not yet sent
1223        if (tempBlockWriteback) {
1224            // if that is the case, write the prevoius one back, and
1225            // do not schedule any new event
1226            writebackTempBlockAtomic();
1227        } else {
1228            // the writeback/clean eviction happens after the call to
1229            // recvAtomic has finished (but before any successive
1230            // calls), so that the response handling from the fill is
1231            // allowed to happen first
1232            schedule(writebackTempBlockAtomicEvent, curTick());
1233        }
1234
1235        tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1236            writebackBlk(blk) : cleanEvictBlk(blk);
1237        invalidateBlock(blk);
1238    }
1239
1240    if (pkt->needsResponse()) {
1241        pkt->makeAtomicResponse();
1242    }
1243
1244    return lat * clockPeriod();
1245}
1246
1247
1248void
1249Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1250{
1251    if (system->bypassCaches()) {
1252        // Packets from the memory side are snoop request and
1253        // shouldn't happen in bypass mode.
1254        assert(fromCpuSide);
1255
1256        // The cache should be flushed if we are in cache bypass mode,
1257        // so we don't need to check if we need to update anything.
1258        memSidePort->sendFunctional(pkt);
1259        return;
1260    }
1261
1262    Addr blk_addr = pkt->getBlockAddr(blkSize);
1263    bool is_secure = pkt->isSecure();
1264    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1265    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1266
1267    pkt->pushLabel(name());
1268
1269    CacheBlkPrintWrapper cbpw(blk);
1270
1271    // Note that just because an L2/L3 has valid data doesn't mean an
1272    // L1 doesn't have a more up-to-date modified copy that still
1273    // needs to be found.  As a result we always update the request if
1274    // we have it, but only declare it satisfied if we are the owner.
1275
1276    // see if we have data at all (owned or otherwise)
1277    bool have_data = blk && blk->isValid()
1278        && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1279                                blk->data);
1280
1281    // data we have is dirty if marked as such or if we have an
1282    // in-service MSHR that is pending a modified line
1283    bool have_dirty =
1284        have_data && (blk->isDirty() ||
1285                      (mshr && mshr->inService && mshr->isPendingModified()));
1286
1287    bool done = have_dirty
1288        || cpuSidePort->checkFunctional(pkt)
1289        || mshrQueue.checkFunctional(pkt, blk_addr)
1290        || writeBuffer.checkFunctional(pkt, blk_addr)
1291        || memSidePort->checkFunctional(pkt);
1292
1293    DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__,  pkt->print(),
1294            (blk && blk->isValid()) ? "valid " : "",
1295            have_data ? "data " : "", done ? "done " : "");
1296
1297    // We're leaving the cache, so pop cache->name() label
1298    pkt->popLabel();
1299
1300    if (done) {
1301        pkt->makeResponse();
1302    } else {
1303        // if it came as a request from the CPU side then make sure it
1304        // continues towards the memory side
1305        if (fromCpuSide) {
1306            memSidePort->sendFunctional(pkt);
1307        } else if (cpuSidePort->isSnooping()) {
1308            // if it came from the memory side, it must be a snoop request
1309            // and we should only forward it if we are forwarding snoops
1310            cpuSidePort->sendFunctionalSnoop(pkt);
1311        }
1312    }
1313}
1314
1315
1316/////////////////////////////////////////////////////
1317//
1318// Response handling: responses from the memory side
1319//
1320/////////////////////////////////////////////////////
1321
1322
1323void
1324Cache::handleUncacheableWriteResp(PacketPtr pkt)
1325{
1326    Tick completion_time = clockEdge(responseLatency) +
1327        pkt->headerDelay + pkt->payloadDelay;
1328
1329    // Reset the bus additional time as it is now accounted for
1330    pkt->headerDelay = pkt->payloadDelay = 0;
1331
1332    cpuSidePort->schedTimingResp(pkt, completion_time, true);
1333}
1334
1335void
1336Cache::serviceMSHRTargets(MSHR *mshr, const PacketPtr pkt, CacheBlk *blk,
1337                          PacketList &writebacks)
1338{
1339    MSHR::Target *initial_tgt = mshr->getTarget();
1340    // First offset for critical word first calculations
1341    const int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1342
1343    const bool is_error = pkt->isError();
1344    bool is_fill = !mshr->isForward &&
1345        (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1346    // allow invalidation responses originating from write-line
1347    // requests to be discarded
1348    bool is_invalidate = pkt->isInvalidate();
1349
1350    MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1351    for (auto &target: targets) {
1352        Packet *tgt_pkt = target.pkt;
1353        switch (target.source) {
1354          case MSHR::Target::FromCPU:
1355            Tick completion_time;
1356            // Here we charge on completion_time the delay of the xbar if the
1357            // packet comes from it, charged on headerDelay.
1358            completion_time = pkt->headerDelay;
1359
1360            // Software prefetch handling for cache closest to core
1361            if (tgt_pkt->cmd.isSWPrefetch()) {
1362                // a software prefetch would have already been ack'd
1363                // immediately with dummy data so the core would be able to
1364                // retire it. This request completes right here, so we
1365                // deallocate it.
1366                delete tgt_pkt->req;
1367                delete tgt_pkt;
1368                break; // skip response
1369            }
1370
1371            // unlike the other packet flows, where data is found in other
1372            // caches or memory and brought back, write-line requests always
1373            // have the data right away, so the above check for "is fill?"
1374            // cannot actually be determined until examining the stored MSHR
1375            // state. We "catch up" with that logic here, which is duplicated
1376            // from above.
1377            if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1378                assert(!is_error);
1379                // we got the block in a writable state, so promote
1380                // any deferred targets if possible
1381                mshr->promoteWritable();
1382                // NB: we use the original packet here and not the response!
1383                blk = handleFill(tgt_pkt, blk, writebacks,
1384                                 targets.allocOnFill);
1385                assert(blk);
1386
1387                // treat as a fill, and discard the invalidation
1388                // response
1389                is_fill = true;
1390                is_invalidate = false;
1391            }
1392
1393            if (is_fill) {
1394                satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1395
1396                // How many bytes past the first request is this one
1397                int transfer_offset =
1398                    tgt_pkt->getOffset(blkSize) - initial_offset;
1399                if (transfer_offset < 0) {
1400                    transfer_offset += blkSize;
1401                }
1402
1403                // If not critical word (offset) return payloadDelay.
1404                // responseLatency is the latency of the return path
1405                // from lower level caches/memory to an upper level cache or
1406                // the core.
1407                completion_time += clockEdge(responseLatency) +
1408                    (transfer_offset ? pkt->payloadDelay : 0);
1409
1410                assert(!tgt_pkt->req->isUncacheable());
1411
1412                assert(tgt_pkt->req->masterId() < system->maxMasters());
1413                missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1414                    completion_time - target.recvTime;
1415            } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1416                // failed StoreCond upgrade
1417                assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1418                       tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1419                       tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1420                // responseLatency is the latency of the return path
1421                // from lower level caches/memory to an upper level cache or
1422                // the core.
1423                completion_time += clockEdge(responseLatency) +
1424                    pkt->payloadDelay;
1425                tgt_pkt->req->setExtraData(0);
1426            } else {
1427                // We are about to send a response to a cache above
1428                // that asked for an invalidation; we need to
1429                // invalidate our copy immediately as the most
1430                // up-to-date copy of the block will now be in the
1431                // cache above. It will also prevent this cache from
1432                // responding (if the block was previously dirty) to
1433                // snoops as they should snoop the caches above where
1434                // they will get the response from.
1435                if (is_invalidate && blk && blk->isValid()) {
1436                    invalidateBlock(blk);
1437                }
1438                // not a cache fill, just forwarding response
1439                // responseLatency is the latency of the return path
1440                // from lower level cahces/memory to the core.
1441                completion_time += clockEdge(responseLatency) +
1442                    pkt->payloadDelay;
1443                if (pkt->isRead() && !is_error) {
1444                    // sanity check
1445                    assert(pkt->getAddr() == tgt_pkt->getAddr());
1446                    assert(pkt->getSize() >= tgt_pkt->getSize());
1447
1448                    tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1449                }
1450            }
1451            tgt_pkt->makeTimingResponse();
1452            // if this packet is an error copy that to the new packet
1453            if (is_error)
1454                tgt_pkt->copyError(pkt);
1455            if (tgt_pkt->cmd == MemCmd::ReadResp &&
1456                (is_invalidate || mshr->hasPostInvalidate())) {
1457                // If intermediate cache got ReadRespWithInvalidate,
1458                // propagate that.  Response should not have
1459                // isInvalidate() set otherwise.
1460                tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1461                DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1462                        tgt_pkt->print());
1463            }
1464            // Reset the bus additional time as it is now accounted for
1465            tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1466            cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1467            break;
1468
1469          case MSHR::Target::FromPrefetcher:
1470            assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1471            if (blk)
1472                blk->status |= BlkHWPrefetched;
1473            delete tgt_pkt->req;
1474            delete tgt_pkt;
1475            break;
1476
1477          case MSHR::Target::FromSnoop:
1478            // I don't believe that a snoop can be in an error state
1479            assert(!is_error);
1480            // response to snoop request
1481            DPRINTF(Cache, "processing deferred snoop...\n");
1482            // If the response is invalidating, a snooping target can
1483            // be satisfied if it is also invalidating. If the reponse is, not
1484            // only invalidating, but more specifically an InvalidateResp and
1485            // the MSHR was created due to an InvalidateReq then a cache above
1486            // is waiting to satisfy a WriteLineReq. In this case even an
1487            // non-invalidating snoop is added as a target here since this is
1488            // the ordering point. When the InvalidateResp reaches this cache,
1489            // the snooping target will snoop further the cache above with the
1490            // WriteLineReq.
1491            assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
1492                   pkt->req->isCacheMaintenance() ||
1493                   mshr->hasPostInvalidate());
1494            handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1495            break;
1496
1497          default:
1498            panic("Illegal target->source enum %d\n", target.source);
1499        }
1500    }
1501
1502    maintainClusivity(targets.hasFromCache, blk);
1503
1504    if (blk && blk->isValid()) {
1505        // an invalidate response stemming from a write line request
1506        // should not invalidate the block, so check if the
1507        // invalidation should be discarded
1508        if (is_invalidate || mshr->hasPostInvalidate()) {
1509            invalidateBlock(blk);
1510        } else if (mshr->hasPostDowngrade()) {
1511            blk->status &= ~BlkWritable;
1512        }
1513    }
1514}
1515
1516void
1517Cache::recvTimingResp(PacketPtr pkt)
1518{
1519    assert(pkt->isResponse());
1520
1521    // all header delay should be paid for by the crossbar, unless
1522    // this is a prefetch response from above
1523    panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1524             "%s saw a non-zero packet delay\n", name());
1525
1526    const bool is_error = pkt->isError();
1527
1528    if (is_error) {
1529        DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1530                pkt->print());
1531    }
1532
1533    DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1534            pkt->print());
1535
1536    // if this is a write, we should be looking at an uncacheable
1537    // write
1538    if (pkt->isWrite()) {
1539        assert(pkt->req->isUncacheable());
1540        handleUncacheableWriteResp(pkt);
1541        return;
1542    }
1543
1544    // we have dealt with any (uncacheable) writes above, from here on
1545    // we know we are dealing with an MSHR due to a miss or a prefetch
1546    MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1547    assert(mshr);
1548
1549    if (mshr == noTargetMSHR) {
1550        // we always clear at least one target
1551        clearBlocked(Blocked_NoTargets);
1552        noTargetMSHR = nullptr;
1553    }
1554
1555    // Initial target is used just for stats
1556    MSHR::Target *initial_tgt = mshr->getTarget();
1557    int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1558    Tick miss_latency = curTick() - initial_tgt->recvTime;
1559
1560    if (pkt->req->isUncacheable()) {
1561        assert(pkt->req->masterId() < system->maxMasters());
1562        mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1563            miss_latency;
1564    } else {
1565        assert(pkt->req->masterId() < system->maxMasters());
1566        mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1567            miss_latency;
1568    }
1569
1570    PacketList writebacks;
1571
1572    bool is_fill = !mshr->isForward &&
1573        (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1574
1575    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1576
1577    if (is_fill && !is_error) {
1578        DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1579                pkt->getAddr());
1580
1581        blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1582        assert(blk != nullptr);
1583    }
1584
1585    if (blk && blk->isValid() && pkt->isClean() && !pkt->isInvalidate()) {
1586        // The block was marked not readable while there was a pending
1587        // cache maintenance operation, restore its flag.
1588        blk->status |= BlkReadable;
1589    }
1590
1591    if (blk && blk->isWritable() && !pkt->req->isCacheInvalidate()) {
1592        // If at this point the referenced block is writable and the
1593        // response is not a cache invalidate, we promote targets that
1594        // were deferred as we couldn't guarrantee a writable copy
1595        mshr->promoteWritable();
1596    }
1597
1598    serviceMSHRTargets(mshr, pkt, blk, writebacks);
1599
1600    if (mshr->promoteDeferredTargets()) {
1601        // avoid later read getting stale data while write miss is
1602        // outstanding.. see comment in timingAccess()
1603        if (blk) {
1604            blk->status &= ~BlkReadable;
1605        }
1606        mshrQueue.markPending(mshr);
1607        schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1608    } else {
1609        // while we deallocate an mshr from the queue we still have to
1610        // check the isFull condition before and after as we might
1611        // have been using the reserved entries already
1612        const bool was_full = mshrQueue.isFull();
1613        mshrQueue.deallocate(mshr);
1614        if (was_full && !mshrQueue.isFull()) {
1615            clearBlocked(Blocked_NoMSHRs);
1616        }
1617
1618        // Request the bus for a prefetch if this deallocation freed enough
1619        // MSHRs for a prefetch to take place
1620        if (prefetcher && mshrQueue.canPrefetch()) {
1621            Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1622                                         clockEdge());
1623            if (next_pf_time != MaxTick)
1624                schedMemSideSendEvent(next_pf_time);
1625        }
1626    }
1627
1628    // if we used temp block, check to see if its valid and then clear it out
1629    if (blk == tempBlock && tempBlock->isValid()) {
1630        PacketPtr wb_pkt = tempBlock->isDirty() || writebackClean ?
1631            writebackBlk(blk) : cleanEvictBlk(blk);
1632        writebacks.push_back(wb_pkt);
1633        invalidateBlock(tempBlock);
1634    }
1635
1636    const Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1637    // copy writebacks to write buffer
1638    doWritebacks(writebacks, forward_time);
1639
1640    DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1641    delete pkt;
1642}
1643
1644PacketPtr
1645Cache::writebackBlk(CacheBlk *blk)
1646{
1647    chatty_assert(!isReadOnly || writebackClean,
1648                  "Writeback from read-only cache");
1649    assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1650
1651    writebacks[Request::wbMasterId]++;
1652
1653    Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1654                               Request::wbMasterId);
1655    if (blk->isSecure())
1656        req->setFlags(Request::SECURE);
1657
1658    req->taskId(blk->task_id);
1659
1660    PacketPtr pkt =
1661        new Packet(req, blk->isDirty() ?
1662                   MemCmd::WritebackDirty : MemCmd::WritebackClean);
1663
1664    DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1665            pkt->print(), blk->isWritable(), blk->isDirty());
1666
1667    if (blk->isWritable()) {
1668        // not asserting shared means we pass the block in modified
1669        // state, mark our own block non-writeable
1670        blk->status &= ~BlkWritable;
1671    } else {
1672        // we are in the Owned state, tell the receiver
1673        pkt->setHasSharers();
1674    }
1675
1676    // make sure the block is not marked dirty
1677    blk->status &= ~BlkDirty;
1678
1679    pkt->allocate();
1680    pkt->setDataFromBlock(blk->data, blkSize);
1681
1682    return pkt;
1683}
1684
1685PacketPtr
1686Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1687{
1688    Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1689                               Request::wbMasterId);
1690    if (blk->isSecure()) {
1691        req->setFlags(Request::SECURE);
1692    }
1693    req->taskId(blk->task_id);
1694
1695    PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1696
1697    if (dest) {
1698        req->setFlags(dest);
1699        pkt->setWriteThrough();
1700    }
1701
1702    DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1703            blk->isWritable(), blk->isDirty());
1704
1705    if (blk->isWritable()) {
1706        // not asserting shared means we pass the block in modified
1707        // state, mark our own block non-writeable
1708        blk->status &= ~BlkWritable;
1709    } else {
1710        // we are in the Owned state, tell the receiver
1711        pkt->setHasSharers();
1712    }
1713
1714    // make sure the block is not marked dirty
1715    blk->status &= ~BlkDirty;
1716
1717    pkt->allocate();
1718    pkt->setDataFromBlock(blk->data, blkSize);
1719
1720    return pkt;
1721}
1722
1723
1724PacketPtr
1725Cache::cleanEvictBlk(CacheBlk *blk)
1726{
1727    assert(!writebackClean);
1728    assert(blk && blk->isValid() && !blk->isDirty());
1729    // Creating a zero sized write, a message to the snoop filter
1730    Request *req =
1731        new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1732                    Request::wbMasterId);
1733    if (blk->isSecure())
1734        req->setFlags(Request::SECURE);
1735
1736    req->taskId(blk->task_id);
1737
1738    PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1739    pkt->allocate();
1740    DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1741
1742    return pkt;
1743}
1744
1745void
1746Cache::memWriteback()
1747{
1748    CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1749    tags->forEachBlk(visitor);
1750}
1751
1752void
1753Cache::memInvalidate()
1754{
1755    CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1756    tags->forEachBlk(visitor);
1757}
1758
1759bool
1760Cache::isDirty() const
1761{
1762    CacheBlkIsDirtyVisitor visitor;
1763    tags->forEachBlk(visitor);
1764
1765    return visitor.isDirty();
1766}
1767
1768bool
1769Cache::writebackVisitor(CacheBlk &blk)
1770{
1771    if (blk.isDirty()) {
1772        assert(blk.isValid());
1773
1774        Request request(tags->regenerateBlkAddr(&blk), blkSize, 0,
1775                        Request::funcMasterId);
1776        request.taskId(blk.task_id);
1777        if (blk.isSecure()) {
1778            request.setFlags(Request::SECURE);
1779        }
1780
1781        Packet packet(&request, MemCmd::WriteReq);
1782        packet.dataStatic(blk.data);
1783
1784        memSidePort->sendFunctional(&packet);
1785
1786        blk.status &= ~BlkDirty;
1787    }
1788
1789    return true;
1790}
1791
1792bool
1793Cache::invalidateVisitor(CacheBlk &blk)
1794{
1795
1796    if (blk.isDirty())
1797        warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1798
1799    if (blk.isValid()) {
1800        assert(!blk.isDirty());
1801        invalidateBlock(&blk);
1802    }
1803
1804    return true;
1805}
1806
1807CacheBlk*
1808Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1809{
1810    // Find replacement victim
1811    CacheBlk *blk = tags->findVictim(addr);
1812
1813    // It is valid to return nullptr if there is no victim
1814    if (!blk)
1815        return nullptr;
1816
1817    if (blk->isValid()) {
1818        Addr repl_addr = tags->regenerateBlkAddr(blk);
1819        MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1820        if (repl_mshr) {
1821            // must be an outstanding upgrade or clean request
1822            // on a block we're about to replace...
1823            assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
1824                   repl_mshr->isCleaning());
1825            // too hard to replace block with transient state
1826            // allocation failed, block not inserted
1827            return nullptr;
1828        } else {
1829            DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1830                    "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1831                    addr, is_secure ? "s" : "ns",
1832                    blk->isDirty() ? "writeback" : "clean");
1833
1834            if (blk->wasPrefetched()) {
1835                unusedPrefetches++;
1836            }
1837            // Will send up Writeback/CleanEvict snoops via isCachedAbove
1838            // when pushing this writeback list into the write buffer.
1839            if (blk->isDirty() || writebackClean) {
1840                // Save writeback packet for handling by caller
1841                writebacks.push_back(writebackBlk(blk));
1842            } else {
1843                writebacks.push_back(cleanEvictBlk(blk));
1844            }
1845            replacements++;
1846        }
1847    }
1848
1849    return blk;
1850}
1851
1852void
1853Cache::invalidateBlock(CacheBlk *blk)
1854{
1855    if (blk != tempBlock)
1856        tags->invalidate(blk);
1857    blk->invalidate();
1858}
1859
1860// Note that the reason we return a list of writebacks rather than
1861// inserting them directly in the write buffer is that this function
1862// is called by both atomic and timing-mode accesses, and in atomic
1863// mode we don't mess with the write buffer (we just perform the
1864// writebacks atomically once the original request is complete).
1865CacheBlk*
1866Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1867                  bool allocate)
1868{
1869    assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1870    Addr addr = pkt->getAddr();
1871    bool is_secure = pkt->isSecure();
1872#if TRACING_ON
1873    CacheBlk::State old_state = blk ? blk->status : 0;
1874#endif
1875
1876    // When handling a fill, we should have no writes to this line.
1877    assert(addr == pkt->getBlockAddr(blkSize));
1878    assert(!writeBuffer.findMatch(addr, is_secure));
1879
1880    if (blk == nullptr) {
1881        // better have read new data...
1882        assert(pkt->hasData());
1883
1884        // only read responses and write-line requests have data;
1885        // note that we don't write the data here for write-line - that
1886        // happens in the subsequent call to satisfyRequest
1887        assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1888
1889        // need to do a replacement if allocating, otherwise we stick
1890        // with the temporary storage
1891        blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1892
1893        if (blk == nullptr) {
1894            // No replaceable block or a mostly exclusive
1895            // cache... just use temporary storage to complete the
1896            // current request and then get rid of it
1897            assert(!tempBlock->isValid());
1898            blk = tempBlock;
1899            tempBlock->set = tags->extractSet(addr);
1900            tempBlock->tag = tags->extractTag(addr);
1901            if (is_secure) {
1902                tempBlock->status |= BlkSecure;
1903            }
1904            DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1905                    is_secure ? "s" : "ns");
1906        } else {
1907            tags->insertBlock(pkt, blk);
1908        }
1909
1910        // we should never be overwriting a valid block
1911        assert(!blk->isValid());
1912    } else {
1913        // existing block... probably an upgrade
1914        assert(blk->tag == tags->extractTag(addr));
1915        // either we're getting new data or the block should already be valid
1916        assert(pkt->hasData() || blk->isValid());
1917        // don't clear block status... if block is already dirty we
1918        // don't want to lose that
1919    }
1920
1921    if (is_secure)
1922        blk->status |= BlkSecure;
1923    blk->status |= BlkValid | BlkReadable;
1924
1925    // sanity check for whole-line writes, which should always be
1926    // marked as writable as part of the fill, and then later marked
1927    // dirty as part of satisfyRequest
1928    if (pkt->cmd == MemCmd::WriteLineReq) {
1929        assert(!pkt->hasSharers());
1930    }
1931
1932    // here we deal with setting the appropriate state of the line,
1933    // and we start by looking at the hasSharers flag, and ignore the
1934    // cacheResponding flag (normally signalling dirty data) if the
1935    // packet has sharers, thus the line is never allocated as Owned
1936    // (dirty but not writable), and always ends up being either
1937    // Shared, Exclusive or Modified, see Packet::setCacheResponding
1938    // for more details
1939    if (!pkt->hasSharers()) {
1940        // we could get a writable line from memory (rather than a
1941        // cache) even in a read-only cache, note that we set this bit
1942        // even for a read-only cache, possibly revisit this decision
1943        blk->status |= BlkWritable;
1944
1945        // check if we got this via cache-to-cache transfer (i.e., from a
1946        // cache that had the block in Modified or Owned state)
1947        if (pkt->cacheResponding()) {
1948            // we got the block in Modified state, and invalidated the
1949            // owners copy
1950            blk->status |= BlkDirty;
1951
1952            chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1953                          "in read-only cache %s\n", name());
1954        }
1955    }
1956
1957    DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1958            addr, is_secure ? "s" : "ns", old_state, blk->print());
1959
1960    // if we got new data, copy it in (checking for a read response
1961    // and a response that has data is the same in the end)
1962    if (pkt->isRead()) {
1963        // sanity checks
1964        assert(pkt->hasData());
1965        assert(pkt->getSize() == blkSize);
1966
1967        pkt->writeDataToBlock(blk->data, blkSize);
1968    }
1969    // We pay for fillLatency here.
1970    blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1971        pkt->payloadDelay;
1972
1973    return blk;
1974}
1975
1976
1977/////////////////////////////////////////////////////
1978//
1979// Snoop path: requests coming in from the memory side
1980//
1981/////////////////////////////////////////////////////
1982
1983void
1984Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1985                              bool already_copied, bool pending_inval)
1986{
1987    // sanity check
1988    assert(req_pkt->isRequest());
1989    assert(req_pkt->needsResponse());
1990
1991    DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1992    // timing-mode snoop responses require a new packet, unless we
1993    // already made a copy...
1994    PacketPtr pkt = req_pkt;
1995    if (!already_copied)
1996        // do not clear flags, and allocate space for data if the
1997        // packet needs it (the only packets that carry data are read
1998        // responses)
1999        pkt = new Packet(req_pkt, false, req_pkt->isRead());
2000
2001    assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
2002           pkt->hasSharers());
2003    pkt->makeTimingResponse();
2004    if (pkt->isRead()) {
2005        pkt->setDataFromBlock(blk_data, blkSize);
2006    }
2007    if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
2008        // Assume we defer a response to a read from a far-away cache
2009        // A, then later defer a ReadExcl from a cache B on the same
2010        // bus as us. We'll assert cacheResponding in both cases, but
2011        // in the latter case cacheResponding will keep the
2012        // invalidation from reaching cache A. This special response
2013        // tells cache A that it gets the block to satisfy its read,
2014        // but must immediately invalidate it.
2015        pkt->cmd = MemCmd::ReadRespWithInvalidate;
2016    }
2017    // Here we consider forward_time, paying for just forward latency and
2018    // also charging the delay provided by the xbar.
2019    // forward_time is used as send_time in next allocateWriteBuffer().
2020    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
2021    // Here we reset the timing of the packet.
2022    pkt->headerDelay = pkt->payloadDelay = 0;
2023    DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
2024            pkt->print(), forward_time);
2025    memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
2026}
2027
2028uint32_t
2029Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
2030                   bool is_deferred, bool pending_inval)
2031{
2032    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2033    // deferred snoops can only happen in timing mode
2034    assert(!(is_deferred && !is_timing));
2035    // pending_inval only makes sense on deferred snoops
2036    assert(!(pending_inval && !is_deferred));
2037    assert(pkt->isRequest());
2038
2039    // the packet may get modified if we or a forwarded snooper
2040    // responds in atomic mode, so remember a few things about the
2041    // original packet up front
2042    bool invalidate = pkt->isInvalidate();
2043    bool M5_VAR_USED needs_writable = pkt->needsWritable();
2044
2045    // at the moment we could get an uncacheable write which does not
2046    // have the invalidate flag, and we need a suitable way of dealing
2047    // with this case
2048    panic_if(invalidate && pkt->req->isUncacheable(),
2049             "%s got an invalidating uncacheable snoop request %s",
2050             name(), pkt->print());
2051
2052    uint32_t snoop_delay = 0;
2053
2054    if (forwardSnoops) {
2055        // first propagate snoop upward to see if anyone above us wants to
2056        // handle it.  save & restore packet src since it will get
2057        // rewritten to be relative to cpu-side bus (if any)
2058        bool alreadyResponded = pkt->cacheResponding();
2059        if (is_timing) {
2060            // copy the packet so that we can clear any flags before
2061            // forwarding it upwards, we also allocate data (passing
2062            // the pointer along in case of static data), in case
2063            // there is a snoop hit in upper levels
2064            Packet snoopPkt(pkt, true, true);
2065            snoopPkt.setExpressSnoop();
2066            // the snoop packet does not need to wait any additional
2067            // time
2068            snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2069            cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2070
2071            // add the header delay (including crossbar and snoop
2072            // delays) of the upward snoop to the snoop delay for this
2073            // cache
2074            snoop_delay += snoopPkt.headerDelay;
2075
2076            if (snoopPkt.cacheResponding()) {
2077                // cache-to-cache response from some upper cache
2078                assert(!alreadyResponded);
2079                pkt->setCacheResponding();
2080            }
2081            // upstream cache has the block, or has an outstanding
2082            // MSHR, pass the flag on
2083            if (snoopPkt.hasSharers()) {
2084                pkt->setHasSharers();
2085            }
2086            // If this request is a prefetch or clean evict and an upper level
2087            // signals block present, make sure to propagate the block
2088            // presence to the requester.
2089            if (snoopPkt.isBlockCached()) {
2090                pkt->setBlockCached();
2091            }
2092            // If the request was satisfied by snooping the cache
2093            // above, mark the original packet as satisfied too.
2094            if (snoopPkt.satisfied()) {
2095                pkt->setSatisfied();
2096            }
2097        } else {
2098            cpuSidePort->sendAtomicSnoop(pkt);
2099            if (!alreadyResponded && pkt->cacheResponding()) {
2100                // cache-to-cache response from some upper cache:
2101                // forward response to original requester
2102                assert(pkt->isResponse());
2103            }
2104        }
2105    }
2106
2107    bool respond = false;
2108    bool blk_valid = blk && blk->isValid();
2109    if (pkt->isClean()) {
2110        if (blk_valid && blk->isDirty()) {
2111            DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
2112                    __func__, pkt->print(), blk->print());
2113            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
2114            PacketList writebacks;
2115            writebacks.push_back(wb_pkt);
2116
2117            if (is_timing) {
2118                // anything that is merely forwarded pays for the forward
2119                // latency and the delay provided by the crossbar
2120                Tick forward_time = clockEdge(forwardLatency) +
2121                    pkt->headerDelay;
2122                doWritebacks(writebacks, forward_time);
2123            } else {
2124                doWritebacksAtomic(writebacks);
2125            }
2126            pkt->setSatisfied();
2127        }
2128    } else if (!blk_valid) {
2129        DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2130                pkt->print());
2131        if (is_deferred) {
2132            // we no longer have the block, and will not respond, but a
2133            // packet was allocated in MSHR::handleSnoop and we have
2134            // to delete it
2135            assert(pkt->needsResponse());
2136
2137            // we have passed the block to a cache upstream, that
2138            // cache should be responding
2139            assert(pkt->cacheResponding());
2140
2141            delete pkt;
2142        }
2143        return snoop_delay;
2144    } else {
2145        DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2146                pkt->print(), blk->print());
2147
2148        // We may end up modifying both the block state and the packet (if
2149        // we respond in atomic mode), so just figure out what to do now
2150        // and then do it later. We respond to all snoops that need
2151        // responses provided we have the block in dirty state. The
2152        // invalidation itself is taken care of below. We don't respond to
2153        // cache maintenance operations as this is done by the destination
2154        // xbar.
2155        respond = blk->isDirty() && pkt->needsResponse();
2156
2157        chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
2158                      "a dirty block in a read-only cache %s\n", name());
2159    }
2160
2161    // Invalidate any prefetch's from below that would strip write permissions
2162    // MemCmd::HardPFReq is only observed by upstream caches.  After missing
2163    // above and in it's own cache, a new MemCmd::ReadReq is created that
2164    // downstream caches observe.
2165    if (pkt->mustCheckAbove()) {
2166        DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2167                "from lower cache\n", pkt->getAddr(), pkt->print());
2168        pkt->setBlockCached();
2169        return snoop_delay;
2170    }
2171
2172    if (pkt->isRead() && !invalidate) {
2173        // reading without requiring the line in a writable state
2174        assert(!needs_writable);
2175        pkt->setHasSharers();
2176
2177        // if the requesting packet is uncacheable, retain the line in
2178        // the current state, otherwhise unset the writable flag,
2179        // which means we go from Modified to Owned (and will respond
2180        // below), remain in Owned (and will respond below), from
2181        // Exclusive to Shared, or remain in Shared
2182        if (!pkt->req->isUncacheable())
2183            blk->status &= ~BlkWritable;
2184        DPRINTF(Cache, "new state is %s\n", blk->print());
2185    }
2186
2187    if (respond) {
2188        // prevent anyone else from responding, cache as well as
2189        // memory, and also prevent any memory from even seeing the
2190        // request
2191        pkt->setCacheResponding();
2192        if (!pkt->isClean() && blk->isWritable()) {
2193            // inform the cache hierarchy that this cache had the line
2194            // in the Modified state so that we avoid unnecessary
2195            // invalidations (see Packet::setResponderHadWritable)
2196            pkt->setResponderHadWritable();
2197
2198            // in the case of an uncacheable request there is no point
2199            // in setting the responderHadWritable flag, but since the
2200            // recipient does not care there is no harm in doing so
2201        } else {
2202            // if the packet has needsWritable set we invalidate our
2203            // copy below and all other copies will be invalidates
2204            // through express snoops, and if needsWritable is not set
2205            // we already called setHasSharers above
2206        }
2207
2208        // if we are returning a writable and dirty (Modified) line,
2209        // we should be invalidating the line
2210        panic_if(!invalidate && !pkt->hasSharers(),
2211                 "%s is passing a Modified line through %s, "
2212                 "but keeping the block", name(), pkt->print());
2213
2214        if (is_timing) {
2215            doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2216        } else {
2217            pkt->makeAtomicResponse();
2218            // packets such as upgrades do not actually have any data
2219            // payload
2220            if (pkt->hasData())
2221                pkt->setDataFromBlock(blk->data, blkSize);
2222        }
2223    }
2224
2225    if (!respond && is_deferred) {
2226        assert(pkt->needsResponse());
2227
2228        // if we copied the deferred packet with the intention to
2229        // respond, but are not responding, then a cache above us must
2230        // be, and we can use this as the indication of whether this
2231        // is a packet where we created a copy of the request or not
2232        if (!pkt->cacheResponding()) {
2233            delete pkt->req;
2234        }
2235
2236        delete pkt;
2237    }
2238
2239    // Do this last in case it deallocates block data or something
2240    // like that
2241    if (blk_valid && invalidate) {
2242        invalidateBlock(blk);
2243        DPRINTF(Cache, "new state is %s\n", blk->print());
2244    }
2245
2246    return snoop_delay;
2247}
2248
2249
2250void
2251Cache::recvTimingSnoopReq(PacketPtr pkt)
2252{
2253    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2254
2255    // Snoops shouldn't happen when bypassing caches
2256    assert(!system->bypassCaches());
2257
2258    // no need to snoop requests that are not in range
2259    if (!inRange(pkt->getAddr())) {
2260        return;
2261    }
2262
2263    bool is_secure = pkt->isSecure();
2264    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2265
2266    Addr blk_addr = pkt->getBlockAddr(blkSize);
2267    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2268
2269    // Update the latency cost of the snoop so that the crossbar can
2270    // account for it. Do not overwrite what other neighbouring caches
2271    // have already done, rather take the maximum. The update is
2272    // tentative, for cases where we return before an upward snoop
2273    // happens below.
2274    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2275                                         lookupLatency * clockPeriod());
2276
2277    // Inform request(Prefetch, CleanEvict or Writeback) from below of
2278    // MSHR hit, set setBlockCached.
2279    if (mshr && pkt->mustCheckAbove()) {
2280        DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2281                "mshr hit\n", pkt->print());
2282        pkt->setBlockCached();
2283        return;
2284    }
2285
2286    // Bypass any existing cache maintenance requests if the request
2287    // has been satisfied already (i.e., the dirty block has been
2288    // found).
2289    if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
2290        return;
2291    }
2292
2293    // Let the MSHR itself track the snoop and decide whether we want
2294    // to go ahead and do the regular cache snoop
2295    if (mshr && mshr->handleSnoop(pkt, order++)) {
2296        DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2297                "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2298                mshr->print());
2299
2300        if (mshr->getNumTargets() > numTarget)
2301            warn("allocating bonus target for snoop"); //handle later
2302        return;
2303    }
2304
2305    //We also need to check the writeback buffers and handle those
2306    WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2307    if (wb_entry) {
2308        DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2309                pkt->getAddr(), is_secure ? "s" : "ns");
2310        // Expect to see only Writebacks and/or CleanEvicts here, both of
2311        // which should not be generated for uncacheable data.
2312        assert(!wb_entry->isUncacheable());
2313        // There should only be a single request responsible for generating
2314        // Writebacks/CleanEvicts.
2315        assert(wb_entry->getNumTargets() == 1);
2316        PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2317        assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2318
2319        if (pkt->isEviction()) {
2320            // if the block is found in the write queue, set the BLOCK_CACHED
2321            // flag for Writeback/CleanEvict snoop. On return the snoop will
2322            // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2323            // any CleanEvicts from travelling down the memory hierarchy.
2324            pkt->setBlockCached();
2325            DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2326                    "hit\n", __func__, pkt->print());
2327            return;
2328        }
2329
2330        // conceptually writebacks are no different to other blocks in
2331        // this cache, so the behaviour is modelled after handleSnoop,
2332        // the difference being that instead of querying the block
2333        // state to determine if it is dirty and writable, we use the
2334        // command and fields of the writeback packet
2335        bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2336            pkt->needsResponse();
2337        bool have_writable = !wb_pkt->hasSharers();
2338        bool invalidate = pkt->isInvalidate();
2339
2340        if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2341            assert(!pkt->needsWritable());
2342            pkt->setHasSharers();
2343            wb_pkt->setHasSharers();
2344        }
2345
2346        if (respond) {
2347            pkt->setCacheResponding();
2348
2349            if (have_writable) {
2350                pkt->setResponderHadWritable();
2351            }
2352
2353            doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2354                                   false, false);
2355        }
2356
2357        if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2358            // Invalidation trumps our writeback... discard here
2359            // Note: markInService will remove entry from writeback buffer.
2360            markInService(wb_entry);
2361            delete wb_pkt;
2362        }
2363    }
2364
2365    // If this was a shared writeback, there may still be
2366    // other shared copies above that require invalidation.
2367    // We could be more selective and return here if the
2368    // request is non-exclusive or if the writeback is
2369    // exclusive.
2370    uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2371
2372    // Override what we did when we first saw the snoop, as we now
2373    // also have the cost of the upwards snoops to account for
2374    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2375                                         lookupLatency * clockPeriod());
2376}
2377
2378bool
2379Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2380{
2381    // Express snoop responses from master to slave, e.g., from L1 to L2
2382    cache->recvTimingSnoopResp(pkt);
2383    return true;
2384}
2385
2386Tick
2387Cache::recvAtomicSnoop(PacketPtr pkt)
2388{
2389    // Snoops shouldn't happen when bypassing caches
2390    assert(!system->bypassCaches());
2391
2392    // no need to snoop requests that are not in range.
2393    if (!inRange(pkt->getAddr())) {
2394        return 0;
2395    }
2396
2397    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2398    uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2399    return snoop_delay + lookupLatency * clockPeriod();
2400}
2401
2402
2403QueueEntry*
2404Cache::getNextQueueEntry()
2405{
2406    // Check both MSHR queue and write buffer for potential requests,
2407    // note that null does not mean there is no request, it could
2408    // simply be that it is not ready
2409    MSHR *miss_mshr  = mshrQueue.getNext();
2410    WriteQueueEntry *wq_entry = writeBuffer.getNext();
2411
2412    // If we got a write buffer request ready, first priority is a
2413    // full write buffer, otherwise we favour the miss requests
2414    if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2415        // need to search MSHR queue for conflicting earlier miss.
2416        MSHR *conflict_mshr =
2417            mshrQueue.findPending(wq_entry->blkAddr,
2418                                  wq_entry->isSecure);
2419
2420        if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2421            // Service misses in order until conflict is cleared.
2422            return conflict_mshr;
2423
2424            // @todo Note that we ignore the ready time of the conflict here
2425        }
2426
2427        // No conflicts; issue write
2428        return wq_entry;
2429    } else if (miss_mshr) {
2430        // need to check for conflicting earlier writeback
2431        WriteQueueEntry *conflict_mshr =
2432            writeBuffer.findPending(miss_mshr->blkAddr,
2433                                    miss_mshr->isSecure);
2434        if (conflict_mshr) {
2435            // not sure why we don't check order here... it was in the
2436            // original code but commented out.
2437
2438            // The only way this happens is if we are
2439            // doing a write and we didn't have permissions
2440            // then subsequently saw a writeback (owned got evicted)
2441            // We need to make sure to perform the writeback first
2442            // To preserve the dirty data, then we can issue the write
2443
2444            // should we return wq_entry here instead?  I.e. do we
2445            // have to flush writes in order?  I don't think so... not
2446            // for Alpha anyway.  Maybe for x86?
2447            return conflict_mshr;
2448
2449            // @todo Note that we ignore the ready time of the conflict here
2450        }
2451
2452        // No conflicts; issue read
2453        return miss_mshr;
2454    }
2455
2456    // fall through... no pending requests.  Try a prefetch.
2457    assert(!miss_mshr && !wq_entry);
2458    if (prefetcher && mshrQueue.canPrefetch()) {
2459        // If we have a miss queue slot, we can try a prefetch
2460        PacketPtr pkt = prefetcher->getPacket();
2461        if (pkt) {
2462            Addr pf_addr = pkt->getBlockAddr(blkSize);
2463            if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2464                !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2465                !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2466                // Update statistic on number of prefetches issued
2467                // (hwpf_mshr_misses)
2468                assert(pkt->req->masterId() < system->maxMasters());
2469                mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2470
2471                // allocate an MSHR and return it, note
2472                // that we send the packet straight away, so do not
2473                // schedule the send
2474                return allocateMissBuffer(pkt, curTick(), false);
2475            } else {
2476                // free the request and packet
2477                delete pkt->req;
2478                delete pkt;
2479            }
2480        }
2481    }
2482
2483    return nullptr;
2484}
2485
2486bool
2487Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2488{
2489    if (!forwardSnoops)
2490        return false;
2491    // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2492    // Writeback snoops into upper level caches to check for copies of the
2493    // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2494    // packet, the cache can inform the crossbar below of presence or absence
2495    // of the block.
2496    if (is_timing) {
2497        Packet snoop_pkt(pkt, true, false);
2498        snoop_pkt.setExpressSnoop();
2499        // Assert that packet is either Writeback or CleanEvict and not a
2500        // prefetch request because prefetch requests need an MSHR and may
2501        // generate a snoop response.
2502        assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2503        snoop_pkt.senderState = nullptr;
2504        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2505        // Writeback/CleanEvict snoops do not generate a snoop response.
2506        assert(!(snoop_pkt.cacheResponding()));
2507        return snoop_pkt.isBlockCached();
2508    } else {
2509        cpuSidePort->sendAtomicSnoop(pkt);
2510        return pkt->isBlockCached();
2511    }
2512}
2513
2514Tick
2515Cache::nextQueueReadyTime() const
2516{
2517    Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2518                              writeBuffer.nextReadyTime());
2519
2520    // Don't signal prefetch ready time if no MSHRs available
2521    // Will signal once enoguh MSHRs are deallocated
2522    if (prefetcher && mshrQueue.canPrefetch()) {
2523        nextReady = std::min(nextReady,
2524                             prefetcher->nextPrefetchReadyTime());
2525    }
2526
2527    return nextReady;
2528}
2529
2530bool
2531Cache::sendMSHRQueuePacket(MSHR* mshr)
2532{
2533    assert(mshr);
2534
2535    // use request from 1st target
2536    PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2537
2538    DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2539
2540    CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2541
2542    if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2543        // we should never have hardware prefetches to allocated
2544        // blocks
2545        assert(blk == nullptr);
2546
2547        // We need to check the caches above us to verify that
2548        // they don't have a copy of this block in the dirty state
2549        // at the moment. Without this check we could get a stale
2550        // copy from memory that might get used in place of the
2551        // dirty one.
2552        Packet snoop_pkt(tgt_pkt, true, false);
2553        snoop_pkt.setExpressSnoop();
2554        // We are sending this packet upwards, but if it hits we will
2555        // get a snoop response that we end up treating just like a
2556        // normal response, hence it needs the MSHR as its sender
2557        // state
2558        snoop_pkt.senderState = mshr;
2559        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2560
2561        // Check to see if the prefetch was squashed by an upper cache (to
2562        // prevent us from grabbing the line) or if a Check to see if a
2563        // writeback arrived between the time the prefetch was placed in
2564        // the MSHRs and when it was selected to be sent or if the
2565        // prefetch was squashed by an upper cache.
2566
2567        // It is important to check cacheResponding before
2568        // prefetchSquashed. If another cache has committed to
2569        // responding, it will be sending a dirty response which will
2570        // arrive at the MSHR allocated for this request. Checking the
2571        // prefetchSquash first may result in the MSHR being
2572        // prematurely deallocated.
2573        if (snoop_pkt.cacheResponding()) {
2574            auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2575            assert(r.second);
2576
2577            // if we are getting a snoop response with no sharers it
2578            // will be allocated as Modified
2579            bool pending_modified_resp = !snoop_pkt.hasSharers();
2580            markInService(mshr, pending_modified_resp);
2581
2582            DPRINTF(Cache, "Upward snoop of prefetch for addr"
2583                    " %#x (%s) hit\n",
2584                    tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2585            return false;
2586        }
2587
2588        if (snoop_pkt.isBlockCached()) {
2589            DPRINTF(Cache, "Block present, prefetch squashed by cache.  "
2590                    "Deallocating mshr target %#x.\n",
2591                    mshr->blkAddr);
2592
2593            // Deallocate the mshr target
2594            if (mshrQueue.forceDeallocateTarget(mshr)) {
2595                // Clear block if this deallocation resulted freed an
2596                // mshr when all had previously been utilized
2597                clearBlocked(Blocked_NoMSHRs);
2598            }
2599
2600            // given that no response is expected, delete Request and Packet
2601            delete tgt_pkt->req;
2602            delete tgt_pkt;
2603
2604            return false;
2605        }
2606    }
2607
2608    // either a prefetch that is not present upstream, or a normal
2609    // MSHR request, proceed to get the packet to send downstream
2610    PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2611
2612    mshr->isForward = (pkt == nullptr);
2613
2614    if (mshr->isForward) {
2615        // not a cache block request, but a response is expected
2616        // make copy of current packet to forward, keep current
2617        // copy for response handling
2618        pkt = new Packet(tgt_pkt, false, true);
2619        assert(!pkt->isWrite());
2620    }
2621
2622    // play it safe and append (rather than set) the sender state,
2623    // as forwarded packets may already have existing state
2624    pkt->pushSenderState(mshr);
2625
2626    if (pkt->isClean() && blk && blk->isDirty()) {
2627        // A cache clean opearation is looking for a dirty block. Mark
2628        // the packet so that the destination xbar can determine that
2629        // there will be a follow-up write packet as well.
2630        pkt->setSatisfied();
2631    }
2632
2633    if (!memSidePort->sendTimingReq(pkt)) {
2634        // we are awaiting a retry, but we
2635        // delete the packet and will be creating a new packet
2636        // when we get the opportunity
2637        delete pkt;
2638
2639        // note that we have now masked any requestBus and
2640        // schedSendEvent (we will wait for a retry before
2641        // doing anything), and this is so even if we do not
2642        // care about this packet and might override it before
2643        // it gets retried
2644        return true;
2645    } else {
2646        // As part of the call to sendTimingReq the packet is
2647        // forwarded to all neighbouring caches (and any caches
2648        // above them) as a snoop. Thus at this point we know if
2649        // any of the neighbouring caches are responding, and if
2650        // so, we know it is dirty, and we can determine if it is
2651        // being passed as Modified, making our MSHR the ordering
2652        // point
2653        bool pending_modified_resp = !pkt->hasSharers() &&
2654            pkt->cacheResponding();
2655        markInService(mshr, pending_modified_resp);
2656        if (pkt->isClean() && blk && blk->isDirty()) {
2657            // A cache clean opearation is looking for a dirty
2658            // block. If a dirty block is encountered a WriteClean
2659            // will update any copies to the path to the memory
2660            // until the point of reference.
2661            DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
2662                    __func__, pkt->print(), blk->print());
2663            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
2664                                             pkt->id);
2665            PacketList writebacks;
2666            writebacks.push_back(wb_pkt);
2667            doWritebacks(writebacks, 0);
2668        }
2669
2670        return false;
2671    }
2672}
2673
2674bool
2675Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2676{
2677    assert(wq_entry);
2678
2679    // always a single target for write queue entries
2680    PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2681
2682    DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2683
2684    // forward as is, both for evictions and uncacheable writes
2685    if (!memSidePort->sendTimingReq(tgt_pkt)) {
2686        // note that we have now masked any requestBus and
2687        // schedSendEvent (we will wait for a retry before
2688        // doing anything), and this is so even if we do not
2689        // care about this packet and might override it before
2690        // it gets retried
2691        return true;
2692    } else {
2693        markInService(wq_entry);
2694        return false;
2695    }
2696}
2697
2698void
2699Cache::serialize(CheckpointOut &cp) const
2700{
2701    bool dirty(isDirty());
2702
2703    if (dirty) {
2704        warn("*** The cache still contains dirty data. ***\n");
2705        warn("    Make sure to drain the system using the correct flags.\n");
2706        warn("    This checkpoint will not restore correctly and dirty data "
2707             "    in the cache will be lost!\n");
2708    }
2709
2710    // Since we don't checkpoint the data in the cache, any dirty data
2711    // will be lost when restoring from a checkpoint of a system that
2712    // wasn't drained properly. Flag the checkpoint as invalid if the
2713    // cache contains dirty data.
2714    bool bad_checkpoint(dirty);
2715    SERIALIZE_SCALAR(bad_checkpoint);
2716}
2717
2718void
2719Cache::unserialize(CheckpointIn &cp)
2720{
2721    bool bad_checkpoint;
2722    UNSERIALIZE_SCALAR(bad_checkpoint);
2723    if (bad_checkpoint) {
2724        fatal("Restoring from checkpoints with dirty caches is not supported "
2725              "in the classic memory system. Please remove any caches or "
2726              " drain them properly before taking checkpoints.\n");
2727    }
2728}
2729
2730///////////////
2731//
2732// CpuSidePort
2733//
2734///////////////
2735
2736AddrRangeList
2737Cache::CpuSidePort::getAddrRanges() const
2738{
2739    return cache->getAddrRanges();
2740}
2741
2742bool
2743Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2744{
2745    assert(!cache->system->bypassCaches());
2746
2747    // always let express snoop packets through if even if blocked
2748    if (pkt->isExpressSnoop()) {
2749        return true;
2750    } else if (isBlocked() || mustSendRetry) {
2751        // either already committed to send a retry, or blocked
2752        mustSendRetry = true;
2753        return false;
2754    }
2755    mustSendRetry = false;
2756    return true;
2757}
2758
2759bool
2760Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2761{
2762    assert(!cache->system->bypassCaches());
2763
2764    // always let express snoop packets through if even if blocked
2765    if (pkt->isExpressSnoop() || tryTiming(pkt)) {
2766        cache->recvTimingReq(pkt);
2767        return true;
2768    }
2769    return false;
2770}
2771
2772Tick
2773Cache::CpuSidePort::recvAtomic(PacketPtr pkt)
2774{
2775    return cache->recvAtomic(pkt);
2776}
2777
2778void
2779Cache::CpuSidePort::recvFunctional(PacketPtr pkt)
2780{
2781    // functional request
2782    cache->functionalAccess(pkt, true);
2783}
2784
2785Cache::
2786CpuSidePort::CpuSidePort(const std::string &_name, Cache *_cache,
2787                         const std::string &_label)
2788    : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2789{
2790}
2791
2792Cache*
2793CacheParams::create()
2794{
2795    assert(tags);
2796    assert(replacement_policy);
2797
2798    return new Cache(this);
2799}
2800///////////////
2801//
2802// MemSidePort
2803//
2804///////////////
2805
2806bool
2807Cache::MemSidePort::recvTimingResp(PacketPtr pkt)
2808{
2809    cache->recvTimingResp(pkt);
2810    return true;
2811}
2812
2813// Express snooping requests to memside port
2814void
2815Cache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2816{
2817    // handle snooping requests
2818    cache->recvTimingSnoopReq(pkt);
2819}
2820
2821Tick
2822Cache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2823{
2824    return cache->recvAtomicSnoop(pkt);
2825}
2826
2827void
2828Cache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2829{
2830    // functional snoop (note that in contrast to atomic we don't have
2831    // a specific functionalSnoop method, as they have the same
2832    // behaviour regardless)
2833    cache->functionalAccess(pkt, false);
2834}
2835
2836void
2837Cache::CacheReqPacketQueue::sendDeferredPacket()
2838{
2839    // sanity check
2840    assert(!waitingOnRetry);
2841
2842    // there should never be any deferred request packets in the
2843    // queue, instead we resly on the cache to provide the packets
2844    // from the MSHR queue or write queue
2845    assert(deferredPacketReadyTime() == MaxTick);
2846
2847    // check for request packets (requests & writebacks)
2848    QueueEntry* entry = cache.getNextQueueEntry();
2849
2850    if (!entry) {
2851        // can happen if e.g. we attempt a writeback and fail, but
2852        // before the retry, the writeback is eliminated because
2853        // we snoop another cache's ReadEx.
2854    } else {
2855        // let our snoop responses go first if there are responses to
2856        // the same addresses
2857        if (checkConflictingSnoop(entry->blkAddr)) {
2858            return;
2859        }
2860        waitingOnRetry = entry->sendPacket(cache);
2861    }
2862
2863    // if we succeeded and are not waiting for a retry, schedule the
2864    // next send considering when the next queue is ready, note that
2865    // snoop responses have their own packet queue and thus schedule
2866    // their own events
2867    if (!waitingOnRetry) {
2868        schedSendEvent(cache.nextQueueReadyTime());
2869    }
2870}
2871
2872Cache::
2873MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2874                         const std::string &_label)
2875    : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2876      _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2877      _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2878{
2879}
2880