cache.cc revision 12556
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 *          Dave Greene
43 *          Nathan Binkert
44 *          Steve Reinhardt
45 *          Ron Dreslinski
46 *          Andreas Sandberg
47 *          Nikos Nikoleris
48 */
49
50/**
51 * @file
52 * Cache definitions.
53 */
54
55#include "mem/cache/cache.hh"
56
57#include "base/logging.hh"
58#include "base/types.hh"
59#include "debug/Cache.hh"
60#include "debug/CachePort.hh"
61#include "debug/CacheTags.hh"
62#include "debug/CacheVerbose.hh"
63#include "mem/cache/blk.hh"
64#include "mem/cache/mshr.hh"
65#include "mem/cache/prefetch/base.hh"
66#include "sim/sim_exit.hh"
67
68Cache::Cache(const CacheParams *p)
69    : BaseCache(p, p->system->cacheLineSize()),
70      tags(p->tags),
71      prefetcher(p->prefetcher),
72      doFastWrites(true),
73      prefetchOnAccess(p->prefetch_on_access),
74      clusivity(p->clusivity),
75      writebackClean(p->writeback_clean),
76      tempBlockWriteback(nullptr),
77      writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
78                                    name(), false,
79                                    EventBase::Delayed_Writeback_Pri)
80{
81    tempBlock = new CacheBlk();
82    tempBlock->data = new uint8_t[blkSize];
83
84    cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
85                                  "CpuSidePort");
86    memSidePort = new MemSidePort(p->name + ".mem_side", this,
87                                  "MemSidePort");
88
89    tags->setCache(this);
90    if (prefetcher)
91        prefetcher->setCache(this);
92}
93
94Cache::~Cache()
95{
96    delete [] tempBlock->data;
97    delete tempBlock;
98
99    delete cpuSidePort;
100    delete memSidePort;
101}
102
103void
104Cache::regStats()
105{
106    BaseCache::regStats();
107}
108
109void
110Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
111{
112    assert(pkt->isRequest());
113
114    uint64_t overwrite_val;
115    bool overwrite_mem;
116    uint64_t condition_val64;
117    uint32_t condition_val32;
118
119    int offset = tags->extractBlkOffset(pkt->getAddr());
120    uint8_t *blk_data = blk->data + offset;
121
122    assert(sizeof(uint64_t) >= pkt->getSize());
123
124    overwrite_mem = true;
125    // keep a copy of our possible write value, and copy what is at the
126    // memory address into the packet
127    pkt->writeData((uint8_t *)&overwrite_val);
128    pkt->setData(blk_data);
129
130    if (pkt->req->isCondSwap()) {
131        if (pkt->getSize() == sizeof(uint64_t)) {
132            condition_val64 = pkt->req->getExtraData();
133            overwrite_mem = !std::memcmp(&condition_val64, blk_data,
134                                         sizeof(uint64_t));
135        } else if (pkt->getSize() == sizeof(uint32_t)) {
136            condition_val32 = (uint32_t)pkt->req->getExtraData();
137            overwrite_mem = !std::memcmp(&condition_val32, blk_data,
138                                         sizeof(uint32_t));
139        } else
140            panic("Invalid size for conditional read/write\n");
141    }
142
143    if (overwrite_mem) {
144        std::memcpy(blk_data, &overwrite_val, pkt->getSize());
145        blk->status |= BlkDirty;
146    }
147}
148
149
150void
151Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
152                      bool deferred_response, bool pending_downgrade)
153{
154    assert(pkt->isRequest());
155
156    assert(blk && blk->isValid());
157    // Occasionally this is not true... if we are a lower-level cache
158    // satisfying a string of Read and ReadEx requests from
159    // upper-level caches, a Read will mark the block as shared but we
160    // can satisfy a following ReadEx anyway since we can rely on the
161    // Read requester(s) to have buffered the ReadEx snoop and to
162    // invalidate their blocks after receiving them.
163    // assert(!pkt->needsWritable() || blk->isWritable());
164    assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
165
166    // Check RMW operations first since both isRead() and
167    // isWrite() will be true for them
168    if (pkt->cmd == MemCmd::SwapReq) {
169        cmpAndSwap(blk, pkt);
170    } else if (pkt->isWrite()) {
171        // we have the block in a writable state and can go ahead,
172        // note that the line may be also be considered writable in
173        // downstream caches along the path to memory, but always
174        // Exclusive, and never Modified
175        assert(blk->isWritable());
176        // Write or WriteLine at the first cache with block in writable state
177        if (blk->checkWrite(pkt)) {
178            pkt->writeDataToBlock(blk->data, blkSize);
179        }
180        // Always mark the line as dirty (and thus transition to the
181        // Modified state) even if we are a failed StoreCond so we
182        // supply data to any snoops that have appended themselves to
183        // this cache before knowing the store will fail.
184        blk->status |= BlkDirty;
185        DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
186    } else if (pkt->isRead()) {
187        if (pkt->isLLSC()) {
188            blk->trackLoadLocked(pkt);
189        }
190
191        // all read responses have a data payload
192        assert(pkt->hasRespData());
193        pkt->setDataFromBlock(blk->data, blkSize);
194
195        // determine if this read is from a (coherent) cache or not
196        if (pkt->fromCache()) {
197            assert(pkt->getSize() == blkSize);
198            // special handling for coherent block requests from
199            // upper-level caches
200            if (pkt->needsWritable()) {
201                // sanity check
202                assert(pkt->cmd == MemCmd::ReadExReq ||
203                       pkt->cmd == MemCmd::SCUpgradeFailReq);
204                assert(!pkt->hasSharers());
205
206                // if we have a dirty copy, make sure the recipient
207                // keeps it marked dirty (in the modified state)
208                if (blk->isDirty()) {
209                    pkt->setCacheResponding();
210                    blk->status &= ~BlkDirty;
211                }
212            } else if (blk->isWritable() && !pending_downgrade &&
213                       !pkt->hasSharers() &&
214                       pkt->cmd != MemCmd::ReadCleanReq) {
215                // we can give the requester a writable copy on a read
216                // request if:
217                // - we have a writable copy at this level (& below)
218                // - we don't have a pending snoop from below
219                //   signaling another read request
220                // - no other cache above has a copy (otherwise it
221                //   would have set hasSharers flag when
222                //   snooping the packet)
223                // - the read has explicitly asked for a clean
224                //   copy of the line
225                if (blk->isDirty()) {
226                    // special considerations if we're owner:
227                    if (!deferred_response) {
228                        // respond with the line in Modified state
229                        // (cacheResponding set, hasSharers not set)
230                        pkt->setCacheResponding();
231
232                        // if this cache is mostly inclusive, we
233                        // keep the block in the Exclusive state,
234                        // and pass it upwards as Modified
235                        // (writable and dirty), hence we have
236                        // multiple caches, all on the same path
237                        // towards memory, all considering the
238                        // same block writable, but only one
239                        // considering it Modified
240
241                        // we get away with multiple caches (on
242                        // the same path to memory) considering
243                        // the block writeable as we always enter
244                        // the cache hierarchy through a cache,
245                        // and first snoop upwards in all other
246                        // branches
247                        blk->status &= ~BlkDirty;
248                    } else {
249                        // if we're responding after our own miss,
250                        // there's a window where the recipient didn't
251                        // know it was getting ownership and may not
252                        // have responded to snoops correctly, so we
253                        // have to respond with a shared line
254                        pkt->setHasSharers();
255                    }
256                }
257            } else {
258                // otherwise only respond with a shared copy
259                pkt->setHasSharers();
260            }
261        }
262    } else if (pkt->isUpgrade()) {
263        // sanity check
264        assert(!pkt->hasSharers());
265
266        if (blk->isDirty()) {
267            // we were in the Owned state, and a cache above us that
268            // has the line in Shared state needs to be made aware
269            // that the data it already has is in fact dirty
270            pkt->setCacheResponding();
271            blk->status &= ~BlkDirty;
272        }
273    } else {
274        assert(pkt->isInvalidate());
275        invalidateBlock(blk);
276        DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
277                pkt->print());
278    }
279}
280
281/////////////////////////////////////////////////////
282//
283// Access path: requests coming in from the CPU side
284//
285/////////////////////////////////////////////////////
286
287bool
288Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
289              PacketList &writebacks)
290{
291    // sanity check
292    assert(pkt->isRequest());
293
294    chatty_assert(!(isReadOnly && pkt->isWrite()),
295                  "Should never see a write in a read-only cache %s\n",
296                  name());
297
298    DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
299
300    if (pkt->req->isUncacheable()) {
301        DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
302
303        // flush and invalidate any existing block
304        CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
305        if (old_blk && old_blk->isValid()) {
306            if (old_blk->isDirty() || writebackClean)
307                writebacks.push_back(writebackBlk(old_blk));
308            else
309                writebacks.push_back(cleanEvictBlk(old_blk));
310            invalidateBlock(old_blk);
311        }
312
313        blk = nullptr;
314        // lookupLatency is the latency in case the request is uncacheable.
315        lat = lookupLatency;
316        return false;
317    }
318
319    // Here lat is the value passed as parameter to accessBlock() function
320    // that can modify its value.
321    blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
322
323    DPRINTF(Cache, "%s %s\n", pkt->print(),
324            blk ? "hit " + blk->print() : "miss");
325
326    if (pkt->req->isCacheMaintenance()) {
327        // A cache maintenance operation is always forwarded to the
328        // memory below even if the block is found in dirty state.
329
330        // We defer any changes to the state of the block until we
331        // create and mark as in service the mshr for the downstream
332        // packet.
333        return false;
334    }
335
336    if (pkt->isEviction()) {
337        // We check for presence of block in above caches before issuing
338        // Writeback or CleanEvict to write buffer. Therefore the only
339        // possible cases can be of a CleanEvict packet coming from above
340        // encountering a Writeback generated in this cache peer cache and
341        // waiting in the write buffer. Cases of upper level peer caches
342        // generating CleanEvict and Writeback or simply CleanEvict and
343        // CleanEvict almost simultaneously will be caught by snoops sent out
344        // by crossbar.
345        WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
346                                                          pkt->isSecure());
347        if (wb_entry) {
348            assert(wb_entry->getNumTargets() == 1);
349            PacketPtr wbPkt = wb_entry->getTarget()->pkt;
350            assert(wbPkt->isWriteback());
351
352            if (pkt->isCleanEviction()) {
353                // The CleanEvict and WritebackClean snoops into other
354                // peer caches of the same level while traversing the
355                // crossbar. If a copy of the block is found, the
356                // packet is deleted in the crossbar. Hence, none of
357                // the other upper level caches connected to this
358                // cache have the block, so we can clear the
359                // BLOCK_CACHED flag in the Writeback if set and
360                // discard the CleanEvict by returning true.
361                wbPkt->clearBlockCached();
362                return true;
363            } else {
364                assert(pkt->cmd == MemCmd::WritebackDirty);
365                // Dirty writeback from above trumps our clean
366                // writeback... discard here
367                // Note: markInService will remove entry from writeback buffer.
368                markInService(wb_entry);
369                delete wbPkt;
370            }
371        }
372    }
373
374    // Writeback handling is special case.  We can write the block into
375    // the cache without having a writeable copy (or any copy at all).
376    if (pkt->isWriteback()) {
377        assert(blkSize == pkt->getSize());
378
379        // we could get a clean writeback while we are having
380        // outstanding accesses to a block, do the simple thing for
381        // now and drop the clean writeback so that we do not upset
382        // any ordering/decisions about ownership already taken
383        if (pkt->cmd == MemCmd::WritebackClean &&
384            mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
385            DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
386                    "dropping\n", pkt->getAddr());
387            return true;
388        }
389
390        if (blk == nullptr) {
391            // need to do a replacement
392            blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
393            if (blk == nullptr) {
394                // no replaceable block available: give up, fwd to next level.
395                incMissCount(pkt);
396                return false;
397            }
398            tags->insertBlock(pkt, blk);
399
400            blk->status = (BlkValid | BlkReadable);
401            if (pkt->isSecure()) {
402                blk->status |= BlkSecure;
403            }
404        }
405        // only mark the block dirty if we got a writeback command,
406        // and leave it as is for a clean writeback
407        if (pkt->cmd == MemCmd::WritebackDirty) {
408            assert(!blk->isDirty());
409            blk->status |= BlkDirty;
410        }
411        // if the packet does not have sharers, it is passing
412        // writable, and we got the writeback in Modified or Exclusive
413        // state, if not we are in the Owned or Shared state
414        if (!pkt->hasSharers()) {
415            blk->status |= BlkWritable;
416        }
417        // nothing else to do; writeback doesn't expect response
418        assert(!pkt->needsResponse());
419        std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
420        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
421        incHitCount(pkt);
422        // populate the time when the block will be ready to access.
423        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
424            pkt->payloadDelay;
425        return true;
426    } else if (pkt->cmd == MemCmd::CleanEvict) {
427        if (blk != nullptr) {
428            // Found the block in the tags, need to stop CleanEvict from
429            // propagating further down the hierarchy. Returning true will
430            // treat the CleanEvict like a satisfied write request and delete
431            // it.
432            return true;
433        }
434        // We didn't find the block here, propagate the CleanEvict further
435        // down the memory hierarchy. Returning false will treat the CleanEvict
436        // like a Writeback which could not find a replaceable block so has to
437        // go to next level.
438        return false;
439    } else if (pkt->cmd == MemCmd::WriteClean) {
440        // WriteClean handling is a special case. We can allocate a
441        // block directly if it doesn't exist and we can update the
442        // block immediately. The WriteClean transfers the ownership
443        // of the block as well.
444        assert(blkSize == pkt->getSize());
445
446        if (!blk) {
447            if (pkt->writeThrough()) {
448                // if this is a write through packet, we don't try to
449                // allocate if the block is not present
450                return false;
451            } else {
452                // a writeback that misses needs to allocate a new block
453                blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
454                                    writebacks);
455                if (!blk) {
456                    // no replaceable block available: give up, fwd to
457                    // next level.
458                    incMissCount(pkt);
459                    return false;
460                }
461                tags->insertBlock(pkt, blk);
462
463                blk->status = (BlkValid | BlkReadable);
464                if (pkt->isSecure()) {
465                    blk->status |= BlkSecure;
466                }
467            }
468        }
469
470        // at this point either this is a writeback or a write-through
471        // write clean operation and the block is already in this
472        // cache, we need to update the data and the block flags
473        assert(blk);
474        assert(!blk->isDirty());
475        if (!pkt->writeThrough()) {
476            blk->status |= BlkDirty;
477        }
478        // nothing else to do; writeback doesn't expect response
479        assert(!pkt->needsResponse());
480        std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
481        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
482
483        incHitCount(pkt);
484        // populate the time when the block will be ready to access.
485        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
486            pkt->payloadDelay;
487        // if this a write-through packet it will be sent to cache
488        // below
489        return !pkt->writeThrough();
490    } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
491                       blk->isReadable())) {
492        // OK to satisfy access
493        incHitCount(pkt);
494        satisfyRequest(pkt, blk);
495        maintainClusivity(pkt->fromCache(), blk);
496
497        return true;
498    }
499
500    // Can't satisfy access normally... either no block (blk == nullptr)
501    // or have block but need writable
502
503    incMissCount(pkt);
504
505    if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
506        // complete miss on store conditional... just give up now
507        pkt->req->setExtraData(0);
508        return true;
509    }
510
511    return false;
512}
513
514void
515Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
516{
517    if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
518        clusivity == Enums::mostly_excl) {
519        // if we have responded to a cache, and our block is still
520        // valid, but not dirty, and this cache is mostly exclusive
521        // with respect to the cache above, drop the block
522        invalidateBlock(blk);
523    }
524}
525
526void
527Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
528{
529    while (!writebacks.empty()) {
530        PacketPtr wbPkt = writebacks.front();
531        // We use forwardLatency here because we are copying writebacks to
532        // write buffer.
533
534        // Call isCachedAbove for Writebacks, CleanEvicts and
535        // WriteCleans to discover if the block is cached above.
536        if (isCachedAbove(wbPkt)) {
537            if (wbPkt->cmd == MemCmd::CleanEvict) {
538                // Delete CleanEvict because cached copies exist above. The
539                // packet destructor will delete the request object because
540                // this is a non-snoop request packet which does not require a
541                // response.
542                delete wbPkt;
543            } else if (wbPkt->cmd == MemCmd::WritebackClean) {
544                // clean writeback, do not send since the block is
545                // still cached above
546                assert(writebackClean);
547                delete wbPkt;
548            } else {
549                assert(wbPkt->cmd == MemCmd::WritebackDirty ||
550                       wbPkt->cmd == MemCmd::WriteClean);
551                // Set BLOCK_CACHED flag in Writeback and send below, so that
552                // the Writeback does not reset the bit corresponding to this
553                // address in the snoop filter below.
554                wbPkt->setBlockCached();
555                allocateWriteBuffer(wbPkt, forward_time);
556            }
557        } else {
558            // If the block is not cached above, send packet below. Both
559            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
560            // reset the bit corresponding to this address in the snoop filter
561            // below.
562            allocateWriteBuffer(wbPkt, forward_time);
563        }
564        writebacks.pop_front();
565    }
566}
567
568void
569Cache::doWritebacksAtomic(PacketList& writebacks)
570{
571    while (!writebacks.empty()) {
572        PacketPtr wbPkt = writebacks.front();
573        // Call isCachedAbove for both Writebacks and CleanEvicts. If
574        // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
575        // and discard CleanEvicts.
576        if (isCachedAbove(wbPkt, false)) {
577            if (wbPkt->cmd == MemCmd::WritebackDirty ||
578                wbPkt->cmd == MemCmd::WriteClean) {
579                // Set BLOCK_CACHED flag in Writeback and send below,
580                // so that the Writeback does not reset the bit
581                // corresponding to this address in the snoop filter
582                // below. We can discard CleanEvicts because cached
583                // copies exist above. Atomic mode isCachedAbove
584                // modifies packet to set BLOCK_CACHED flag
585                memSidePort->sendAtomic(wbPkt);
586            }
587        } else {
588            // If the block is not cached above, send packet below. Both
589            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
590            // reset the bit corresponding to this address in the snoop filter
591            // below.
592            memSidePort->sendAtomic(wbPkt);
593        }
594        writebacks.pop_front();
595        // In case of CleanEvicts, the packet destructor will delete the
596        // request object because this is a non-snoop request packet which
597        // does not require a response.
598        delete wbPkt;
599    }
600}
601
602
603void
604Cache::recvTimingSnoopResp(PacketPtr pkt)
605{
606    DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
607
608    assert(pkt->isResponse());
609    assert(!system->bypassCaches());
610
611    // determine if the response is from a snoop request we created
612    // (in which case it should be in the outstandingSnoop), or if we
613    // merely forwarded someone else's snoop request
614    const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
615        outstandingSnoop.end();
616
617    if (!forwardAsSnoop) {
618        // the packet came from this cache, so sink it here and do not
619        // forward it
620        assert(pkt->cmd == MemCmd::HardPFResp);
621
622        outstandingSnoop.erase(pkt->req);
623
624        DPRINTF(Cache, "Got prefetch response from above for addr "
625                "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
626        recvTimingResp(pkt);
627        return;
628    }
629
630    // forwardLatency is set here because there is a response from an
631    // upper level cache.
632    // To pay the delay that occurs if the packet comes from the bus,
633    // we charge also headerDelay.
634    Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
635    // Reset the timing of the packet.
636    pkt->headerDelay = pkt->payloadDelay = 0;
637    memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
638}
639
640void
641Cache::promoteWholeLineWrites(PacketPtr pkt)
642{
643    // Cache line clearing instructions
644    if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
645        (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
646        pkt->cmd = MemCmd::WriteLineReq;
647        DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
648    }
649}
650
651bool
652Cache::recvTimingReq(PacketPtr pkt)
653{
654    DPRINTF(CacheTags, "%s tags:\n%s\n", __func__, tags->print());
655
656    assert(pkt->isRequest());
657
658    // Just forward the packet if caches are disabled.
659    if (system->bypassCaches()) {
660        // @todo This should really enqueue the packet rather
661        bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
662        assert(success);
663        return true;
664    }
665
666    promoteWholeLineWrites(pkt);
667
668    // Cache maintenance operations have to visit all the caches down
669    // to the specified xbar (PoC, PoU, etc.). Even if a cache above
670    // is responding we forward the packet to the memory below rather
671    // than creating an express snoop.
672    if (pkt->cacheResponding()) {
673        // a cache above us (but not where the packet came from) is
674        // responding to the request, in other words it has the line
675        // in Modified or Owned state
676        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
677                pkt->print());
678
679        // if the packet needs the block to be writable, and the cache
680        // that has promised to respond (setting the cache responding
681        // flag) is not providing writable (it is in Owned rather than
682        // the Modified state), we know that there may be other Shared
683        // copies in the system; go out and invalidate them all
684        assert(pkt->needsWritable() && !pkt->responderHadWritable());
685
686        // an upstream cache that had the line in Owned state
687        // (dirty, but not writable), is responding and thus
688        // transferring the dirty line from one branch of the
689        // cache hierarchy to another
690
691        // send out an express snoop and invalidate all other
692        // copies (snooping a packet that needs writable is the
693        // same as an invalidation), thus turning the Owned line
694        // into a Modified line, note that we don't invalidate the
695        // block in the current cache or any other cache on the
696        // path to memory
697
698        // create a downstream express snoop with cleared packet
699        // flags, there is no need to allocate any data as the
700        // packet is merely used to co-ordinate state transitions
701        Packet *snoop_pkt = new Packet(pkt, true, false);
702
703        // also reset the bus time that the original packet has
704        // not yet paid for
705        snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
706
707        // make this an instantaneous express snoop, and let the
708        // other caches in the system know that the another cache
709        // is responding, because we have found the authorative
710        // copy (Modified or Owned) that will supply the right
711        // data
712        snoop_pkt->setExpressSnoop();
713        snoop_pkt->setCacheResponding();
714
715        // this express snoop travels towards the memory, and at
716        // every crossbar it is snooped upwards thus reaching
717        // every cache in the system
718        bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt);
719        // express snoops always succeed
720        assert(success);
721
722        // main memory will delete the snoop packet
723
724        // queue for deletion, as opposed to immediate deletion, as
725        // the sending cache is still relying on the packet
726        pendingDelete.reset(pkt);
727
728        // no need to take any further action in this particular cache
729        // as an upstram cache has already committed to responding,
730        // and we have already sent out any express snoops in the
731        // section above to ensure all other copies in the system are
732        // invalidated
733        return true;
734    }
735
736    // anything that is merely forwarded pays for the forward latency and
737    // the delay provided by the crossbar
738    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
739
740    // We use lookupLatency here because it is used to specify the latency
741    // to access.
742    Cycles lat = lookupLatency;
743    CacheBlk *blk = nullptr;
744    bool satisfied = false;
745    {
746        PacketList writebacks;
747        // Note that lat is passed by reference here. The function
748        // access() calls accessBlock() which can modify lat value.
749        satisfied = access(pkt, blk, lat, writebacks);
750
751        // copy writebacks to write buffer here to ensure they logically
752        // proceed anything happening below
753        doWritebacks(writebacks, forward_time);
754    }
755
756    // Here we charge the headerDelay that takes into account the latencies
757    // of the bus, if the packet comes from it.
758    // The latency charged it is just lat that is the value of lookupLatency
759    // modified by access() function, or if not just lookupLatency.
760    // In case of a hit we are neglecting response latency.
761    // In case of a miss we are neglecting forward latency.
762    Tick request_time = clockEdge(lat) + pkt->headerDelay;
763    // Here we reset the timing of the packet.
764    pkt->headerDelay = pkt->payloadDelay = 0;
765
766    // track time of availability of next prefetch, if any
767    Tick next_pf_time = MaxTick;
768
769    bool needsResponse = pkt->needsResponse();
770
771    if (satisfied) {
772        // should never be satisfying an uncacheable access as we
773        // flush and invalidate any existing block as part of the
774        // lookup
775        assert(!pkt->req->isUncacheable());
776
777        // hit (for all other request types)
778
779        if (prefetcher && (prefetchOnAccess ||
780                           (blk && blk->wasPrefetched()))) {
781            if (blk)
782                blk->status &= ~BlkHWPrefetched;
783
784            // Don't notify on SWPrefetch
785            if (!pkt->cmd.isSWPrefetch()) {
786                assert(!pkt->req->isCacheMaintenance());
787                next_pf_time = prefetcher->notify(pkt);
788            }
789        }
790
791        if (needsResponse) {
792            pkt->makeTimingResponse();
793            // @todo: Make someone pay for this
794            pkt->headerDelay = pkt->payloadDelay = 0;
795
796            // In this case we are considering request_time that takes
797            // into account the delay of the xbar, if any, and just
798            // lat, neglecting responseLatency, modelling hit latency
799            // just as lookupLatency or or the value of lat overriden
800            // by access(), that calls accessBlock() function.
801            cpuSidePort->schedTimingResp(pkt, request_time, true);
802        } else {
803            DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
804                    pkt->print());
805
806            // queue the packet for deletion, as the sending cache is
807            // still relying on it; if the block is found in access(),
808            // CleanEvict and Writeback messages will be deleted
809            // here as well
810            pendingDelete.reset(pkt);
811        }
812    } else {
813        // miss
814
815        Addr blk_addr = pkt->getBlockAddr(blkSize);
816
817        // ignore any existing MSHR if we are dealing with an
818        // uncacheable request
819        MSHR *mshr = pkt->req->isUncacheable() ? nullptr :
820            mshrQueue.findMatch(blk_addr, pkt->isSecure());
821
822        // Software prefetch handling:
823        // To keep the core from waiting on data it won't look at
824        // anyway, send back a response with dummy data. Miss handling
825        // will continue asynchronously. Unfortunately, the core will
826        // insist upon freeing original Packet/Request, so we have to
827        // create a new pair with a different lifecycle. Note that this
828        // processing happens before any MSHR munging on the behalf of
829        // this request because this new Request will be the one stored
830        // into the MSHRs, not the original.
831        if (pkt->cmd.isSWPrefetch()) {
832            assert(needsResponse);
833            assert(pkt->req->hasPaddr());
834            assert(!pkt->req->isUncacheable());
835
836            // There's no reason to add a prefetch as an additional target
837            // to an existing MSHR. If an outstanding request is already
838            // in progress, there is nothing for the prefetch to do.
839            // If this is the case, we don't even create a request at all.
840            PacketPtr pf = nullptr;
841
842            if (!mshr) {
843                // copy the request and create a new SoftPFReq packet
844                RequestPtr req = new Request(pkt->req->getPaddr(),
845                                             pkt->req->getSize(),
846                                             pkt->req->getFlags(),
847                                             pkt->req->masterId());
848                pf = new Packet(req, pkt->cmd);
849                pf->allocate();
850                assert(pf->getAddr() == pkt->getAddr());
851                assert(pf->getSize() == pkt->getSize());
852            }
853
854            pkt->makeTimingResponse();
855
856            // request_time is used here, taking into account lat and the delay
857            // charged if the packet comes from the xbar.
858            cpuSidePort->schedTimingResp(pkt, request_time, true);
859
860            // If an outstanding request is in progress (we found an
861            // MSHR) this is set to null
862            pkt = pf;
863        }
864
865        if (mshr) {
866            /// MSHR hit
867            /// @note writebacks will be checked in getNextMSHR()
868            /// for any conflicting requests to the same block
869
870            //@todo remove hw_pf here
871
872            // Coalesce unless it was a software prefetch (see above).
873            if (pkt) {
874                assert(!pkt->isWriteback());
875                // CleanEvicts corresponding to blocks which have
876                // outstanding requests in MSHRs are simply sunk here
877                if (pkt->cmd == MemCmd::CleanEvict) {
878                    pendingDelete.reset(pkt);
879                } else if (pkt->cmd == MemCmd::WriteClean) {
880                    // A WriteClean should never coalesce with any
881                    // outstanding cache maintenance requests.
882
883                    // We use forward_time here because there is an
884                    // uncached memory write, forwarded to WriteBuffer.
885                    allocateWriteBuffer(pkt, forward_time);
886                } else {
887                    DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
888                            pkt->print());
889
890                    assert(pkt->req->masterId() < system->maxMasters());
891                    mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
892                    // We use forward_time here because it is the same
893                    // considering new targets. We have multiple
894                    // requests for the same address here. It
895                    // specifies the latency to allocate an internal
896                    // buffer and to schedule an event to the queued
897                    // port and also takes into account the additional
898                    // delay of the xbar.
899                    mshr->allocateTarget(pkt, forward_time, order++,
900                                         allocOnFill(pkt->cmd));
901                    if (mshr->getNumTargets() == numTarget) {
902                        noTargetMSHR = mshr;
903                        setBlocked(Blocked_NoTargets);
904                        // need to be careful with this... if this mshr isn't
905                        // ready yet (i.e. time > curTick()), we don't want to
906                        // move it ahead of mshrs that are ready
907                        // mshrQueue.moveToFront(mshr);
908                    }
909                }
910                // We should call the prefetcher reguardless if the request is
911                // satisfied or not, reguardless if the request is in the MSHR
912                // or not.  The request could be a ReadReq hit, but still not
913                // satisfied (potentially because of a prior write to the same
914                // cache line.  So, even when not satisfied, tehre is an MSHR
915                // already allocated for this, we need to let the prefetcher
916                // know about the request
917                if (prefetcher) {
918                    // Don't notify on SWPrefetch
919                    if (!pkt->cmd.isSWPrefetch() &&
920                        !pkt->req->isCacheMaintenance())
921                        next_pf_time = prefetcher->notify(pkt);
922                }
923            }
924        } else {
925            // no MSHR
926            assert(pkt->req->masterId() < system->maxMasters());
927            if (pkt->req->isUncacheable()) {
928                mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
929            } else {
930                mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
931            }
932
933            if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
934                (pkt->req->isUncacheable() && pkt->isWrite())) {
935                // We use forward_time here because there is an
936                // uncached memory write, forwarded to WriteBuffer.
937                allocateWriteBuffer(pkt, forward_time);
938            } else {
939                if (blk && blk->isValid()) {
940                    // should have flushed and have no valid block
941                    assert(!pkt->req->isUncacheable());
942
943                    // If we have a write miss to a valid block, we
944                    // need to mark the block non-readable.  Otherwise
945                    // if we allow reads while there's an outstanding
946                    // write miss, the read could return stale data
947                    // out of the cache block... a more aggressive
948                    // system could detect the overlap (if any) and
949                    // forward data out of the MSHRs, but we don't do
950                    // that yet.  Note that we do need to leave the
951                    // block valid so that it stays in the cache, in
952                    // case we get an upgrade response (and hence no
953                    // new data) when the write miss completes.
954                    // As long as CPUs do proper store/load forwarding
955                    // internally, and have a sufficiently weak memory
956                    // model, this is probably unnecessary, but at some
957                    // point it must have seemed like we needed it...
958                    assert((pkt->needsWritable() && !blk->isWritable()) ||
959                           pkt->req->isCacheMaintenance());
960                    blk->status &= ~BlkReadable;
961                }
962                // Here we are using forward_time, modelling the latency of
963                // a miss (outbound) just as forwardLatency, neglecting the
964                // lookupLatency component.
965                allocateMissBuffer(pkt, forward_time);
966            }
967
968            if (prefetcher) {
969                // Don't notify on SWPrefetch
970                if (!pkt->cmd.isSWPrefetch() &&
971                    !pkt->req->isCacheMaintenance())
972                    next_pf_time = prefetcher->notify(pkt);
973            }
974        }
975    }
976
977    if (next_pf_time != MaxTick)
978        schedMemSideSendEvent(next_pf_time);
979
980    return true;
981}
982
983PacketPtr
984Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
985                        bool needsWritable) const
986{
987    // should never see evictions here
988    assert(!cpu_pkt->isEviction());
989
990    bool blkValid = blk && blk->isValid();
991
992    if (cpu_pkt->req->isUncacheable() ||
993        (!blkValid && cpu_pkt->isUpgrade()) ||
994        cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
995        // uncacheable requests and upgrades from upper-level caches
996        // that missed completely just go through as is
997        return nullptr;
998    }
999
1000    assert(cpu_pkt->needsResponse());
1001
1002    MemCmd cmd;
1003    // @TODO make useUpgrades a parameter.
1004    // Note that ownership protocols require upgrade, otherwise a
1005    // write miss on a shared owned block will generate a ReadExcl,
1006    // which will clobber the owned copy.
1007    const bool useUpgrades = true;
1008    if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
1009        assert(!blkValid || !blk->isWritable());
1010        // forward as invalidate to all other caches, this gives us
1011        // the line in Exclusive state, and invalidates all other
1012        // copies
1013        cmd = MemCmd::InvalidateReq;
1014    } else if (blkValid && useUpgrades) {
1015        // only reason to be here is that blk is read only and we need
1016        // it to be writable
1017        assert(needsWritable);
1018        assert(!blk->isWritable());
1019        cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
1020    } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
1021               cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
1022        // Even though this SC will fail, we still need to send out the
1023        // request and get the data to supply it to other snoopers in the case
1024        // where the determination the StoreCond fails is delayed due to
1025        // all caches not being on the same local bus.
1026        cmd = MemCmd::SCUpgradeFailReq;
1027    } else {
1028        // block is invalid
1029
1030        // If the request does not need a writable there are two cases
1031        // where we need to ensure the response will not fetch the
1032        // block in dirty state:
1033        // * this cache is read only and it does not perform
1034        //   writebacks,
1035        // * this cache is mostly exclusive and will not fill (since
1036        //   it does not fill it will have to writeback the dirty data
1037        //   immediately which generates uneccesary writebacks).
1038        bool force_clean_rsp = isReadOnly || clusivity == Enums::mostly_excl;
1039        cmd = needsWritable ? MemCmd::ReadExReq :
1040            (force_clean_rsp ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1041    }
1042    PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1043
1044    // if there are upstream caches that have already marked the
1045    // packet as having sharers (not passing writable), pass that info
1046    // downstream
1047    if (cpu_pkt->hasSharers() && !needsWritable) {
1048        // note that cpu_pkt may have spent a considerable time in the
1049        // MSHR queue and that the information could possibly be out
1050        // of date, however, there is no harm in conservatively
1051        // assuming the block has sharers
1052        pkt->setHasSharers();
1053        DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1054                __func__, cpu_pkt->print(), pkt->print());
1055    }
1056
1057    // the packet should be block aligned
1058    assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1059
1060    pkt->allocate();
1061    DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1062            cpu_pkt->print());
1063    return pkt;
1064}
1065
1066
1067Tick
1068Cache::recvAtomic(PacketPtr pkt)
1069{
1070    // We are in atomic mode so we pay just for lookupLatency here.
1071    Cycles lat = lookupLatency;
1072
1073    // Forward the request if the system is in cache bypass mode.
1074    if (system->bypassCaches())
1075        return ticksToCycles(memSidePort->sendAtomic(pkt));
1076
1077    promoteWholeLineWrites(pkt);
1078
1079    // follow the same flow as in recvTimingReq, and check if a cache
1080    // above us is responding
1081    if (pkt->cacheResponding() && !pkt->isClean()) {
1082        assert(!pkt->req->isCacheInvalidate());
1083        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1084                pkt->print());
1085
1086        // if a cache is responding, and it had the line in Owned
1087        // rather than Modified state, we need to invalidate any
1088        // copies that are not on the same path to memory
1089        assert(pkt->needsWritable() && !pkt->responderHadWritable());
1090        lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1091
1092        return lat * clockPeriod();
1093    }
1094
1095    // should assert here that there are no outstanding MSHRs or
1096    // writebacks... that would mean that someone used an atomic
1097    // access in timing mode
1098
1099    CacheBlk *blk = nullptr;
1100    PacketList writebacks;
1101    bool satisfied = access(pkt, blk, lat, writebacks);
1102
1103    if (pkt->isClean() && blk && blk->isDirty()) {
1104        // A cache clean opearation is looking for a dirty
1105        // block. If a dirty block is encountered a WriteClean
1106        // will update any copies to the path to the memory
1107        // until the point of reference.
1108        DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1109                __func__, pkt->print(), blk->print());
1110        PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
1111        writebacks.push_back(wb_pkt);
1112        pkt->setSatisfied();
1113    }
1114
1115    // handle writebacks resulting from the access here to ensure they
1116    // logically proceed anything happening below
1117    doWritebacksAtomic(writebacks);
1118
1119    if (!satisfied) {
1120        // MISS
1121
1122        // deal with the packets that go through the write path of
1123        // the cache, i.e. any evictions and writes
1124        if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1125            (pkt->req->isUncacheable() && pkt->isWrite())) {
1126            lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1127            return lat * clockPeriod();
1128        }
1129        // only misses left
1130
1131        PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1132
1133        bool is_forward = (bus_pkt == nullptr);
1134
1135        if (is_forward) {
1136            // just forwarding the same request to the next level
1137            // no local cache operation involved
1138            bus_pkt = pkt;
1139        }
1140
1141        DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1142                bus_pkt->print());
1143
1144#if TRACING_ON
1145        CacheBlk::State old_state = blk ? blk->status : 0;
1146#endif
1147
1148        lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1149
1150        bool is_invalidate = bus_pkt->isInvalidate();
1151
1152        // We are now dealing with the response handling
1153        DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1154                bus_pkt->print(), old_state);
1155
1156        // If packet was a forward, the response (if any) is already
1157        // in place in the bus_pkt == pkt structure, so we don't need
1158        // to do anything.  Otherwise, use the separate bus_pkt to
1159        // generate response to pkt and then delete it.
1160        if (!is_forward) {
1161            if (pkt->needsResponse()) {
1162                assert(bus_pkt->isResponse());
1163                if (bus_pkt->isError()) {
1164                    pkt->makeAtomicResponse();
1165                    pkt->copyError(bus_pkt);
1166                } else if (pkt->cmd == MemCmd::WriteLineReq) {
1167                    // note the use of pkt, not bus_pkt here.
1168
1169                    // write-line request to the cache that promoted
1170                    // the write to a whole line
1171                    blk = handleFill(pkt, blk, writebacks,
1172                                     allocOnFill(pkt->cmd));
1173                    assert(blk != NULL);
1174                    is_invalidate = false;
1175                    satisfyRequest(pkt, blk);
1176                } else if (bus_pkt->isRead() ||
1177                           bus_pkt->cmd == MemCmd::UpgradeResp) {
1178                    // we're updating cache state to allow us to
1179                    // satisfy the upstream request from the cache
1180                    blk = handleFill(bus_pkt, blk, writebacks,
1181                                     allocOnFill(pkt->cmd));
1182                    satisfyRequest(pkt, blk);
1183                    maintainClusivity(pkt->fromCache(), blk);
1184                } else {
1185                    // we're satisfying the upstream request without
1186                    // modifying cache state, e.g., a write-through
1187                    pkt->makeAtomicResponse();
1188                }
1189            }
1190            delete bus_pkt;
1191        }
1192
1193        if (is_invalidate && blk && blk->isValid()) {
1194            invalidateBlock(blk);
1195        }
1196    }
1197
1198    // Note that we don't invoke the prefetcher at all in atomic mode.
1199    // It's not clear how to do it properly, particularly for
1200    // prefetchers that aggressively generate prefetch candidates and
1201    // rely on bandwidth contention to throttle them; these will tend
1202    // to pollute the cache in atomic mode since there is no bandwidth
1203    // contention.  If we ever do want to enable prefetching in atomic
1204    // mode, though, this is the place to do it... see timingAccess()
1205    // for an example (though we'd want to issue the prefetch(es)
1206    // immediately rather than calling requestMemSideBus() as we do
1207    // there).
1208
1209    // do any writebacks resulting from the response handling
1210    doWritebacksAtomic(writebacks);
1211
1212    // if we used temp block, check to see if its valid and if so
1213    // clear it out, but only do so after the call to recvAtomic is
1214    // finished so that any downstream observers (such as a snoop
1215    // filter), first see the fill, and only then see the eviction
1216    if (blk == tempBlock && tempBlock->isValid()) {
1217        // the atomic CPU calls recvAtomic for fetch and load/store
1218        // sequentuially, and we may already have a tempBlock
1219        // writeback from the fetch that we have not yet sent
1220        if (tempBlockWriteback) {
1221            // if that is the case, write the prevoius one back, and
1222            // do not schedule any new event
1223            writebackTempBlockAtomic();
1224        } else {
1225            // the writeback/clean eviction happens after the call to
1226            // recvAtomic has finished (but before any successive
1227            // calls), so that the response handling from the fill is
1228            // allowed to happen first
1229            schedule(writebackTempBlockAtomicEvent, curTick());
1230        }
1231
1232        tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1233            writebackBlk(blk) : cleanEvictBlk(blk);
1234        invalidateBlock(blk);
1235    }
1236
1237    if (pkt->needsResponse()) {
1238        pkt->makeAtomicResponse();
1239    }
1240
1241    return lat * clockPeriod();
1242}
1243
1244
1245void
1246Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1247{
1248    if (system->bypassCaches()) {
1249        // Packets from the memory side are snoop request and
1250        // shouldn't happen in bypass mode.
1251        assert(fromCpuSide);
1252
1253        // The cache should be flushed if we are in cache bypass mode,
1254        // so we don't need to check if we need to update anything.
1255        memSidePort->sendFunctional(pkt);
1256        return;
1257    }
1258
1259    Addr blk_addr = pkt->getBlockAddr(blkSize);
1260    bool is_secure = pkt->isSecure();
1261    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1262    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1263
1264    pkt->pushLabel(name());
1265
1266    CacheBlkPrintWrapper cbpw(blk);
1267
1268    // Note that just because an L2/L3 has valid data doesn't mean an
1269    // L1 doesn't have a more up-to-date modified copy that still
1270    // needs to be found.  As a result we always update the request if
1271    // we have it, but only declare it satisfied if we are the owner.
1272
1273    // see if we have data at all (owned or otherwise)
1274    bool have_data = blk && blk->isValid()
1275        && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1276                                blk->data);
1277
1278    // data we have is dirty if marked as such or if we have an
1279    // in-service MSHR that is pending a modified line
1280    bool have_dirty =
1281        have_data && (blk->isDirty() ||
1282                      (mshr && mshr->inService && mshr->isPendingModified()));
1283
1284    bool done = have_dirty
1285        || cpuSidePort->checkFunctional(pkt)
1286        || mshrQueue.checkFunctional(pkt, blk_addr)
1287        || writeBuffer.checkFunctional(pkt, blk_addr)
1288        || memSidePort->checkFunctional(pkt);
1289
1290    DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__,  pkt->print(),
1291            (blk && blk->isValid()) ? "valid " : "",
1292            have_data ? "data " : "", done ? "done " : "");
1293
1294    // We're leaving the cache, so pop cache->name() label
1295    pkt->popLabel();
1296
1297    if (done) {
1298        pkt->makeResponse();
1299    } else {
1300        // if it came as a request from the CPU side then make sure it
1301        // continues towards the memory side
1302        if (fromCpuSide) {
1303            memSidePort->sendFunctional(pkt);
1304        } else if (cpuSidePort->isSnooping()) {
1305            // if it came from the memory side, it must be a snoop request
1306            // and we should only forward it if we are forwarding snoops
1307            cpuSidePort->sendFunctionalSnoop(pkt);
1308        }
1309    }
1310}
1311
1312
1313/////////////////////////////////////////////////////
1314//
1315// Response handling: responses from the memory side
1316//
1317/////////////////////////////////////////////////////
1318
1319
1320void
1321Cache::handleUncacheableWriteResp(PacketPtr pkt)
1322{
1323    Tick completion_time = clockEdge(responseLatency) +
1324        pkt->headerDelay + pkt->payloadDelay;
1325
1326    // Reset the bus additional time as it is now accounted for
1327    pkt->headerDelay = pkt->payloadDelay = 0;
1328
1329    cpuSidePort->schedTimingResp(pkt, completion_time, true);
1330}
1331
1332void
1333Cache::recvTimingResp(PacketPtr pkt)
1334{
1335    assert(pkt->isResponse());
1336
1337    // all header delay should be paid for by the crossbar, unless
1338    // this is a prefetch response from above
1339    panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1340             "%s saw a non-zero packet delay\n", name());
1341
1342    bool is_error = pkt->isError();
1343
1344    if (is_error) {
1345        DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1346                pkt->print());
1347    }
1348
1349    DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1350            pkt->print());
1351
1352    // if this is a write, we should be looking at an uncacheable
1353    // write
1354    if (pkt->isWrite()) {
1355        assert(pkt->req->isUncacheable());
1356        handleUncacheableWriteResp(pkt);
1357        return;
1358    }
1359
1360    // we have dealt with any (uncacheable) writes above, from here on
1361    // we know we are dealing with an MSHR due to a miss or a prefetch
1362    MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1363    assert(mshr);
1364
1365    if (mshr == noTargetMSHR) {
1366        // we always clear at least one target
1367        clearBlocked(Blocked_NoTargets);
1368        noTargetMSHR = nullptr;
1369    }
1370
1371    // Initial target is used just for stats
1372    MSHR::Target *initial_tgt = mshr->getTarget();
1373    int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1374    Tick miss_latency = curTick() - initial_tgt->recvTime;
1375
1376    if (pkt->req->isUncacheable()) {
1377        assert(pkt->req->masterId() < system->maxMasters());
1378        mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1379            miss_latency;
1380    } else {
1381        assert(pkt->req->masterId() < system->maxMasters());
1382        mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1383            miss_latency;
1384    }
1385
1386    bool wasFull = mshrQueue.isFull();
1387
1388    PacketList writebacks;
1389
1390    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1391
1392    bool is_fill = !mshr->isForward &&
1393        (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1394
1395    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1396    const bool valid_blk = blk && blk->isValid();
1397    // If the response indicates that there are no sharers and we
1398    // either had the block already or the response is filling we can
1399    // promote our copy to writable
1400    if (!pkt->hasSharers() &&
1401        (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1402        mshr->promoteWritable();
1403    }
1404
1405    if (is_fill && !is_error) {
1406        DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1407                pkt->getAddr());
1408
1409        blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1410        assert(blk != nullptr);
1411    }
1412
1413    // allow invalidation responses originating from write-line
1414    // requests to be discarded
1415    bool is_invalidate = pkt->isInvalidate();
1416
1417    // The block was marked as not readable while there was a pending
1418    // cache maintenance operation, restore its flag.
1419    if (pkt->isClean() && !is_invalidate && valid_blk) {
1420        blk->status |= BlkReadable;
1421    }
1422
1423    // First offset for critical word first calculations
1424    int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1425
1426    bool from_cache = false;
1427    MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1428    for (auto &target: targets) {
1429        Packet *tgt_pkt = target.pkt;
1430        switch (target.source) {
1431          case MSHR::Target::FromCPU:
1432            Tick completion_time;
1433            // Here we charge on completion_time the delay of the xbar if the
1434            // packet comes from it, charged on headerDelay.
1435            completion_time = pkt->headerDelay;
1436
1437            // Software prefetch handling for cache closest to core
1438            if (tgt_pkt->cmd.isSWPrefetch()) {
1439                // a software prefetch would have already been ack'd
1440                // immediately with dummy data so the core would be able to
1441                // retire it. This request completes right here, so we
1442                // deallocate it.
1443                delete tgt_pkt->req;
1444                delete tgt_pkt;
1445                break; // skip response
1446            }
1447
1448            // keep track of whether we have responded to another
1449            // cache
1450            from_cache = from_cache || tgt_pkt->fromCache();
1451
1452            // unlike the other packet flows, where data is found in other
1453            // caches or memory and brought back, write-line requests always
1454            // have the data right away, so the above check for "is fill?"
1455            // cannot actually be determined until examining the stored MSHR
1456            // state. We "catch up" with that logic here, which is duplicated
1457            // from above.
1458            if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1459                assert(!is_error);
1460                // we got the block in a writable state, so promote
1461                // any deferred targets if possible
1462                mshr->promoteWritable();
1463                // NB: we use the original packet here and not the response!
1464                blk = handleFill(tgt_pkt, blk, writebacks,
1465                                 targets.allocOnFill);
1466                assert(blk != nullptr);
1467
1468                // treat as a fill, and discard the invalidation
1469                // response
1470                is_fill = true;
1471                is_invalidate = false;
1472            }
1473
1474            if (is_fill) {
1475                satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1476
1477                // How many bytes past the first request is this one
1478                int transfer_offset =
1479                    tgt_pkt->getOffset(blkSize) - initial_offset;
1480                if (transfer_offset < 0) {
1481                    transfer_offset += blkSize;
1482                }
1483
1484                // If not critical word (offset) return payloadDelay.
1485                // responseLatency is the latency of the return path
1486                // from lower level caches/memory to an upper level cache or
1487                // the core.
1488                completion_time += clockEdge(responseLatency) +
1489                    (transfer_offset ? pkt->payloadDelay : 0);
1490
1491                assert(!tgt_pkt->req->isUncacheable());
1492
1493                assert(tgt_pkt->req->masterId() < system->maxMasters());
1494                missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1495                    completion_time - target.recvTime;
1496            } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1497                // failed StoreCond upgrade
1498                assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1499                       tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1500                       tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1501                // responseLatency is the latency of the return path
1502                // from lower level caches/memory to an upper level cache or
1503                // the core.
1504                completion_time += clockEdge(responseLatency) +
1505                    pkt->payloadDelay;
1506                tgt_pkt->req->setExtraData(0);
1507            } else {
1508                // We are about to send a response to a cache above
1509                // that asked for an invalidation; we need to
1510                // invalidate our copy immediately as the most
1511                // up-to-date copy of the block will now be in the
1512                // cache above. It will also prevent this cache from
1513                // responding (if the block was previously dirty) to
1514                // snoops as they should snoop the caches above where
1515                // they will get the response from.
1516                if (is_invalidate && blk && blk->isValid()) {
1517                    invalidateBlock(blk);
1518                }
1519                // not a cache fill, just forwarding response
1520                // responseLatency is the latency of the return path
1521                // from lower level cahces/memory to the core.
1522                completion_time += clockEdge(responseLatency) +
1523                    pkt->payloadDelay;
1524                if (pkt->isRead() && !is_error) {
1525                    // sanity check
1526                    assert(pkt->getAddr() == tgt_pkt->getAddr());
1527                    assert(pkt->getSize() >= tgt_pkt->getSize());
1528
1529                    tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1530                }
1531            }
1532            tgt_pkt->makeTimingResponse();
1533            // if this packet is an error copy that to the new packet
1534            if (is_error)
1535                tgt_pkt->copyError(pkt);
1536            if (tgt_pkt->cmd == MemCmd::ReadResp &&
1537                (is_invalidate || mshr->hasPostInvalidate())) {
1538                // If intermediate cache got ReadRespWithInvalidate,
1539                // propagate that.  Response should not have
1540                // isInvalidate() set otherwise.
1541                tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1542                DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1543                        tgt_pkt->print());
1544            }
1545            // Reset the bus additional time as it is now accounted for
1546            tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1547            cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1548            break;
1549
1550          case MSHR::Target::FromPrefetcher:
1551            assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1552            if (blk)
1553                blk->status |= BlkHWPrefetched;
1554            delete tgt_pkt->req;
1555            delete tgt_pkt;
1556            break;
1557
1558          case MSHR::Target::FromSnoop:
1559            // I don't believe that a snoop can be in an error state
1560            assert(!is_error);
1561            // response to snoop request
1562            DPRINTF(Cache, "processing deferred snoop...\n");
1563            // If the response is invalidating, a snooping target can
1564            // be satisfied if it is also invalidating. If the reponse is, not
1565            // only invalidating, but more specifically an InvalidateResp and
1566            // the MSHR was created due to an InvalidateReq then a cache above
1567            // is waiting to satisfy a WriteLineReq. In this case even an
1568            // non-invalidating snoop is added as a target here since this is
1569            // the ordering point. When the InvalidateResp reaches this cache,
1570            // the snooping target will snoop further the cache above with the
1571            // WriteLineReq.
1572            assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
1573                   pkt->req->isCacheMaintenance() ||
1574                   mshr->hasPostInvalidate());
1575            handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1576            break;
1577
1578          default:
1579            panic("Illegal target->source enum %d\n", target.source);
1580        }
1581    }
1582
1583    maintainClusivity(from_cache, blk);
1584
1585    if (blk && blk->isValid()) {
1586        // an invalidate response stemming from a write line request
1587        // should not invalidate the block, so check if the
1588        // invalidation should be discarded
1589        if (is_invalidate || mshr->hasPostInvalidate()) {
1590            invalidateBlock(blk);
1591        } else if (mshr->hasPostDowngrade()) {
1592            blk->status &= ~BlkWritable;
1593        }
1594    }
1595
1596    if (mshr->promoteDeferredTargets()) {
1597        // avoid later read getting stale data while write miss is
1598        // outstanding.. see comment in timingAccess()
1599        if (blk) {
1600            blk->status &= ~BlkReadable;
1601        }
1602        mshrQueue.markPending(mshr);
1603        schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1604    } else {
1605        mshrQueue.deallocate(mshr);
1606        if (wasFull && !mshrQueue.isFull()) {
1607            clearBlocked(Blocked_NoMSHRs);
1608        }
1609
1610        // Request the bus for a prefetch if this deallocation freed enough
1611        // MSHRs for a prefetch to take place
1612        if (prefetcher && mshrQueue.canPrefetch()) {
1613            Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1614                                         clockEdge());
1615            if (next_pf_time != MaxTick)
1616                schedMemSideSendEvent(next_pf_time);
1617        }
1618    }
1619    // reset the xbar additional timinig  as it is now accounted for
1620    pkt->headerDelay = pkt->payloadDelay = 0;
1621
1622    // copy writebacks to write buffer
1623    doWritebacks(writebacks, forward_time);
1624
1625    // if we used temp block, check to see if its valid and then clear it out
1626    if (blk == tempBlock && tempBlock->isValid()) {
1627        // We use forwardLatency here because we are copying
1628        // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1629        // allocate an internal buffer and to schedule an event to the
1630        // queued port.
1631        if (blk->isDirty() || writebackClean) {
1632            PacketPtr wbPkt = writebackBlk(blk);
1633            allocateWriteBuffer(wbPkt, forward_time);
1634            // Set BLOCK_CACHED flag if cached above.
1635            if (isCachedAbove(wbPkt))
1636                wbPkt->setBlockCached();
1637        } else {
1638            PacketPtr wcPkt = cleanEvictBlk(blk);
1639            // Check to see if block is cached above. If not allocate
1640            // write buffer
1641            if (isCachedAbove(wcPkt))
1642                delete wcPkt;
1643            else
1644                allocateWriteBuffer(wcPkt, forward_time);
1645        }
1646        invalidateBlock(blk);
1647    }
1648
1649    DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1650    delete pkt;
1651}
1652
1653PacketPtr
1654Cache::writebackBlk(CacheBlk *blk)
1655{
1656    chatty_assert(!isReadOnly || writebackClean,
1657                  "Writeback from read-only cache");
1658    assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1659
1660    writebacks[Request::wbMasterId]++;
1661
1662    Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1663                               blkSize, 0, Request::wbMasterId);
1664    if (blk->isSecure())
1665        req->setFlags(Request::SECURE);
1666
1667    req->taskId(blk->task_id);
1668
1669    PacketPtr pkt =
1670        new Packet(req, blk->isDirty() ?
1671                   MemCmd::WritebackDirty : MemCmd::WritebackClean);
1672
1673    DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1674            pkt->print(), blk->isWritable(), blk->isDirty());
1675
1676    if (blk->isWritable()) {
1677        // not asserting shared means we pass the block in modified
1678        // state, mark our own block non-writeable
1679        blk->status &= ~BlkWritable;
1680    } else {
1681        // we are in the Owned state, tell the receiver
1682        pkt->setHasSharers();
1683    }
1684
1685    // make sure the block is not marked dirty
1686    blk->status &= ~BlkDirty;
1687
1688    pkt->allocate();
1689    std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1690
1691    return pkt;
1692}
1693
1694PacketPtr
1695Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1696{
1697    Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1698                               blkSize, 0, Request::wbMasterId);
1699    if (blk->isSecure()) {
1700        req->setFlags(Request::SECURE);
1701    }
1702    req->taskId(blk->task_id);
1703
1704    PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1705
1706    if (dest) {
1707        req->setFlags(dest);
1708        pkt->setWriteThrough();
1709    }
1710
1711    DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1712            blk->isWritable(), blk->isDirty());
1713
1714    if (blk->isWritable()) {
1715        // not asserting shared means we pass the block in modified
1716        // state, mark our own block non-writeable
1717        blk->status &= ~BlkWritable;
1718    } else {
1719        // we are in the Owned state, tell the receiver
1720        pkt->setHasSharers();
1721    }
1722
1723    // make sure the block is not marked dirty
1724    blk->status &= ~BlkDirty;
1725
1726    pkt->allocate();
1727    std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1728
1729    return pkt;
1730}
1731
1732
1733PacketPtr
1734Cache::cleanEvictBlk(CacheBlk *blk)
1735{
1736    assert(!writebackClean);
1737    assert(blk && blk->isValid() && !blk->isDirty());
1738    // Creating a zero sized write, a message to the snoop filter
1739    Request *req =
1740        new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1741                    Request::wbMasterId);
1742    if (blk->isSecure())
1743        req->setFlags(Request::SECURE);
1744
1745    req->taskId(blk->task_id);
1746
1747    PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1748    pkt->allocate();
1749    DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1750
1751    return pkt;
1752}
1753
1754void
1755Cache::memWriteback()
1756{
1757    CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1758    tags->forEachBlk(visitor);
1759}
1760
1761void
1762Cache::memInvalidate()
1763{
1764    CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1765    tags->forEachBlk(visitor);
1766}
1767
1768bool
1769Cache::isDirty() const
1770{
1771    CacheBlkIsDirtyVisitor visitor;
1772    tags->forEachBlk(visitor);
1773
1774    return visitor.isDirty();
1775}
1776
1777bool
1778Cache::writebackVisitor(CacheBlk &blk)
1779{
1780    if (blk.isDirty()) {
1781        assert(blk.isValid());
1782
1783        Request request(tags->regenerateBlkAddr(blk.tag, blk.set),
1784                        blkSize, 0, Request::funcMasterId);
1785        request.taskId(blk.task_id);
1786        if (blk.isSecure()) {
1787            request.setFlags(Request::SECURE);
1788        }
1789
1790        Packet packet(&request, MemCmd::WriteReq);
1791        packet.dataStatic(blk.data);
1792
1793        memSidePort->sendFunctional(&packet);
1794
1795        blk.status &= ~BlkDirty;
1796    }
1797
1798    return true;
1799}
1800
1801bool
1802Cache::invalidateVisitor(CacheBlk &blk)
1803{
1804
1805    if (blk.isDirty())
1806        warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1807
1808    if (blk.isValid()) {
1809        assert(!blk.isDirty());
1810        invalidateBlock(&blk);
1811    }
1812
1813    return true;
1814}
1815
1816CacheBlk*
1817Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1818{
1819    CacheBlk *blk = tags->findVictim(addr);
1820
1821    // It is valid to return nullptr if there is no victim
1822    if (!blk)
1823        return nullptr;
1824
1825    if (blk->isValid()) {
1826        Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set);
1827        MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1828        if (repl_mshr) {
1829            // must be an outstanding upgrade request
1830            // on a block we're about to replace...
1831            assert(!blk->isWritable() || blk->isDirty());
1832            assert(repl_mshr->needsWritable());
1833            // too hard to replace block with transient state
1834            // allocation failed, block not inserted
1835            return nullptr;
1836        } else {
1837            DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1838                    "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1839                    addr, is_secure ? "s" : "ns",
1840                    blk->isDirty() ? "writeback" : "clean");
1841
1842            if (blk->wasPrefetched()) {
1843                unusedPrefetches++;
1844            }
1845            // Will send up Writeback/CleanEvict snoops via isCachedAbove
1846            // when pushing this writeback list into the write buffer.
1847            if (blk->isDirty() || writebackClean) {
1848                // Save writeback packet for handling by caller
1849                writebacks.push_back(writebackBlk(blk));
1850            } else {
1851                writebacks.push_back(cleanEvictBlk(blk));
1852            }
1853        }
1854    }
1855
1856    return blk;
1857}
1858
1859void
1860Cache::invalidateBlock(CacheBlk *blk)
1861{
1862    if (blk != tempBlock)
1863        tags->invalidate(blk);
1864    blk->invalidate();
1865}
1866
1867// Note that the reason we return a list of writebacks rather than
1868// inserting them directly in the write buffer is that this function
1869// is called by both atomic and timing-mode accesses, and in atomic
1870// mode we don't mess with the write buffer (we just perform the
1871// writebacks atomically once the original request is complete).
1872CacheBlk*
1873Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1874                  bool allocate)
1875{
1876    assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1877    Addr addr = pkt->getAddr();
1878    bool is_secure = pkt->isSecure();
1879#if TRACING_ON
1880    CacheBlk::State old_state = blk ? blk->status : 0;
1881#endif
1882
1883    // When handling a fill, we should have no writes to this line.
1884    assert(addr == pkt->getBlockAddr(blkSize));
1885    assert(!writeBuffer.findMatch(addr, is_secure));
1886
1887    if (blk == nullptr) {
1888        // better have read new data...
1889        assert(pkt->hasData());
1890
1891        // only read responses and write-line requests have data;
1892        // note that we don't write the data here for write-line - that
1893        // happens in the subsequent call to satisfyRequest
1894        assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1895
1896        // need to do a replacement if allocating, otherwise we stick
1897        // with the temporary storage
1898        blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1899
1900        if (blk == nullptr) {
1901            // No replaceable block or a mostly exclusive
1902            // cache... just use temporary storage to complete the
1903            // current request and then get rid of it
1904            assert(!tempBlock->isValid());
1905            blk = tempBlock;
1906            tempBlock->set = tags->extractSet(addr);
1907            tempBlock->tag = tags->extractTag(addr);
1908            if (is_secure) {
1909                tempBlock->status |= BlkSecure;
1910            }
1911            DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1912                    is_secure ? "s" : "ns");
1913        } else {
1914            tags->insertBlock(pkt, blk);
1915        }
1916
1917        // we should never be overwriting a valid block
1918        assert(!blk->isValid());
1919    } else {
1920        // existing block... probably an upgrade
1921        assert(blk->tag == tags->extractTag(addr));
1922        // either we're getting new data or the block should already be valid
1923        assert(pkt->hasData() || blk->isValid());
1924        // don't clear block status... if block is already dirty we
1925        // don't want to lose that
1926    }
1927
1928    if (is_secure)
1929        blk->status |= BlkSecure;
1930    blk->status |= BlkValid | BlkReadable;
1931
1932    // sanity check for whole-line writes, which should always be
1933    // marked as writable as part of the fill, and then later marked
1934    // dirty as part of satisfyRequest
1935    if (pkt->cmd == MemCmd::WriteLineReq) {
1936        assert(!pkt->hasSharers());
1937    }
1938
1939    // here we deal with setting the appropriate state of the line,
1940    // and we start by looking at the hasSharers flag, and ignore the
1941    // cacheResponding flag (normally signalling dirty data) if the
1942    // packet has sharers, thus the line is never allocated as Owned
1943    // (dirty but not writable), and always ends up being either
1944    // Shared, Exclusive or Modified, see Packet::setCacheResponding
1945    // for more details
1946    if (!pkt->hasSharers()) {
1947        // we could get a writable line from memory (rather than a
1948        // cache) even in a read-only cache, note that we set this bit
1949        // even for a read-only cache, possibly revisit this decision
1950        blk->status |= BlkWritable;
1951
1952        // check if we got this via cache-to-cache transfer (i.e., from a
1953        // cache that had the block in Modified or Owned state)
1954        if (pkt->cacheResponding()) {
1955            // we got the block in Modified state, and invalidated the
1956            // owners copy
1957            blk->status |= BlkDirty;
1958
1959            chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1960                          "in read-only cache %s\n", name());
1961        }
1962    }
1963
1964    DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1965            addr, is_secure ? "s" : "ns", old_state, blk->print());
1966
1967    // if we got new data, copy it in (checking for a read response
1968    // and a response that has data is the same in the end)
1969    if (pkt->isRead()) {
1970        // sanity checks
1971        assert(pkt->hasData());
1972        assert(pkt->getSize() == blkSize);
1973
1974        std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
1975    }
1976    // We pay for fillLatency here.
1977    blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1978        pkt->payloadDelay;
1979
1980    return blk;
1981}
1982
1983
1984/////////////////////////////////////////////////////
1985//
1986// Snoop path: requests coming in from the memory side
1987//
1988/////////////////////////////////////////////////////
1989
1990void
1991Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1992                              bool already_copied, bool pending_inval)
1993{
1994    // sanity check
1995    assert(req_pkt->isRequest());
1996    assert(req_pkt->needsResponse());
1997
1998    DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1999    // timing-mode snoop responses require a new packet, unless we
2000    // already made a copy...
2001    PacketPtr pkt = req_pkt;
2002    if (!already_copied)
2003        // do not clear flags, and allocate space for data if the
2004        // packet needs it (the only packets that carry data are read
2005        // responses)
2006        pkt = new Packet(req_pkt, false, req_pkt->isRead());
2007
2008    assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
2009           pkt->hasSharers());
2010    pkt->makeTimingResponse();
2011    if (pkt->isRead()) {
2012        pkt->setDataFromBlock(blk_data, blkSize);
2013    }
2014    if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
2015        // Assume we defer a response to a read from a far-away cache
2016        // A, then later defer a ReadExcl from a cache B on the same
2017        // bus as us. We'll assert cacheResponding in both cases, but
2018        // in the latter case cacheResponding will keep the
2019        // invalidation from reaching cache A. This special response
2020        // tells cache A that it gets the block to satisfy its read,
2021        // but must immediately invalidate it.
2022        pkt->cmd = MemCmd::ReadRespWithInvalidate;
2023    }
2024    // Here we consider forward_time, paying for just forward latency and
2025    // also charging the delay provided by the xbar.
2026    // forward_time is used as send_time in next allocateWriteBuffer().
2027    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
2028    // Here we reset the timing of the packet.
2029    pkt->headerDelay = pkt->payloadDelay = 0;
2030    DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
2031            pkt->print(), forward_time);
2032    memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
2033}
2034
2035uint32_t
2036Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
2037                   bool is_deferred, bool pending_inval)
2038{
2039    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2040    // deferred snoops can only happen in timing mode
2041    assert(!(is_deferred && !is_timing));
2042    // pending_inval only makes sense on deferred snoops
2043    assert(!(pending_inval && !is_deferred));
2044    assert(pkt->isRequest());
2045
2046    // the packet may get modified if we or a forwarded snooper
2047    // responds in atomic mode, so remember a few things about the
2048    // original packet up front
2049    bool invalidate = pkt->isInvalidate();
2050    bool M5_VAR_USED needs_writable = pkt->needsWritable();
2051
2052    // at the moment we could get an uncacheable write which does not
2053    // have the invalidate flag, and we need a suitable way of dealing
2054    // with this case
2055    panic_if(invalidate && pkt->req->isUncacheable(),
2056             "%s got an invalidating uncacheable snoop request %s",
2057             name(), pkt->print());
2058
2059    uint32_t snoop_delay = 0;
2060
2061    if (forwardSnoops) {
2062        // first propagate snoop upward to see if anyone above us wants to
2063        // handle it.  save & restore packet src since it will get
2064        // rewritten to be relative to cpu-side bus (if any)
2065        bool alreadyResponded = pkt->cacheResponding();
2066        if (is_timing) {
2067            // copy the packet so that we can clear any flags before
2068            // forwarding it upwards, we also allocate data (passing
2069            // the pointer along in case of static data), in case
2070            // there is a snoop hit in upper levels
2071            Packet snoopPkt(pkt, true, true);
2072            snoopPkt.setExpressSnoop();
2073            // the snoop packet does not need to wait any additional
2074            // time
2075            snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2076            cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2077
2078            // add the header delay (including crossbar and snoop
2079            // delays) of the upward snoop to the snoop delay for this
2080            // cache
2081            snoop_delay += snoopPkt.headerDelay;
2082
2083            if (snoopPkt.cacheResponding()) {
2084                // cache-to-cache response from some upper cache
2085                assert(!alreadyResponded);
2086                pkt->setCacheResponding();
2087            }
2088            // upstream cache has the block, or has an outstanding
2089            // MSHR, pass the flag on
2090            if (snoopPkt.hasSharers()) {
2091                pkt->setHasSharers();
2092            }
2093            // If this request is a prefetch or clean evict and an upper level
2094            // signals block present, make sure to propagate the block
2095            // presence to the requester.
2096            if (snoopPkt.isBlockCached()) {
2097                pkt->setBlockCached();
2098            }
2099            // If the request was satisfied by snooping the cache
2100            // above, mark the original packet as satisfied too.
2101            if (snoopPkt.satisfied()) {
2102                pkt->setSatisfied();
2103            }
2104        } else {
2105            cpuSidePort->sendAtomicSnoop(pkt);
2106            if (!alreadyResponded && pkt->cacheResponding()) {
2107                // cache-to-cache response from some upper cache:
2108                // forward response to original requester
2109                assert(pkt->isResponse());
2110            }
2111        }
2112    }
2113
2114    bool respond = false;
2115    bool blk_valid = blk && blk->isValid();
2116    if (pkt->isClean()) {
2117        if (blk_valid && blk->isDirty()) {
2118            DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
2119                    __func__, pkt->print(), blk->print());
2120            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
2121            PacketList writebacks;
2122            writebacks.push_back(wb_pkt);
2123
2124            if (is_timing) {
2125                // anything that is merely forwarded pays for the forward
2126                // latency and the delay provided by the crossbar
2127                Tick forward_time = clockEdge(forwardLatency) +
2128                    pkt->headerDelay;
2129                doWritebacks(writebacks, forward_time);
2130            } else {
2131                doWritebacksAtomic(writebacks);
2132            }
2133            pkt->setSatisfied();
2134        }
2135    } else if (!blk_valid) {
2136        DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2137                pkt->print());
2138        if (is_deferred) {
2139            // we no longer have the block, and will not respond, but a
2140            // packet was allocated in MSHR::handleSnoop and we have
2141            // to delete it
2142            assert(pkt->needsResponse());
2143
2144            // we have passed the block to a cache upstream, that
2145            // cache should be responding
2146            assert(pkt->cacheResponding());
2147
2148            delete pkt;
2149        }
2150        return snoop_delay;
2151    } else {
2152        DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2153                pkt->print(), blk->print());
2154
2155        // We may end up modifying both the block state and the packet (if
2156        // we respond in atomic mode), so just figure out what to do now
2157        // and then do it later. We respond to all snoops that need
2158        // responses provided we have the block in dirty state. The
2159        // invalidation itself is taken care of below. We don't respond to
2160        // cache maintenance operations as this is done by the destination
2161        // xbar.
2162        respond = blk->isDirty() && pkt->needsResponse();
2163
2164        chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
2165                      "a dirty block in a read-only cache %s\n", name());
2166    }
2167
2168    // Invalidate any prefetch's from below that would strip write permissions
2169    // MemCmd::HardPFReq is only observed by upstream caches.  After missing
2170    // above and in it's own cache, a new MemCmd::ReadReq is created that
2171    // downstream caches observe.
2172    if (pkt->mustCheckAbove()) {
2173        DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2174                "from lower cache\n", pkt->getAddr(), pkt->print());
2175        pkt->setBlockCached();
2176        return snoop_delay;
2177    }
2178
2179    if (pkt->isRead() && !invalidate) {
2180        // reading without requiring the line in a writable state
2181        assert(!needs_writable);
2182        pkt->setHasSharers();
2183
2184        // if the requesting packet is uncacheable, retain the line in
2185        // the current state, otherwhise unset the writable flag,
2186        // which means we go from Modified to Owned (and will respond
2187        // below), remain in Owned (and will respond below), from
2188        // Exclusive to Shared, or remain in Shared
2189        if (!pkt->req->isUncacheable())
2190            blk->status &= ~BlkWritable;
2191        DPRINTF(Cache, "new state is %s\n", blk->print());
2192    }
2193
2194    if (respond) {
2195        // prevent anyone else from responding, cache as well as
2196        // memory, and also prevent any memory from even seeing the
2197        // request
2198        pkt->setCacheResponding();
2199        if (!pkt->isClean() && blk->isWritable()) {
2200            // inform the cache hierarchy that this cache had the line
2201            // in the Modified state so that we avoid unnecessary
2202            // invalidations (see Packet::setResponderHadWritable)
2203            pkt->setResponderHadWritable();
2204
2205            // in the case of an uncacheable request there is no point
2206            // in setting the responderHadWritable flag, but since the
2207            // recipient does not care there is no harm in doing so
2208        } else {
2209            // if the packet has needsWritable set we invalidate our
2210            // copy below and all other copies will be invalidates
2211            // through express snoops, and if needsWritable is not set
2212            // we already called setHasSharers above
2213        }
2214
2215        // if we are returning a writable and dirty (Modified) line,
2216        // we should be invalidating the line
2217        panic_if(!invalidate && !pkt->hasSharers(),
2218                 "%s is passing a Modified line through %s, "
2219                 "but keeping the block", name(), pkt->print());
2220
2221        if (is_timing) {
2222            doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2223        } else {
2224            pkt->makeAtomicResponse();
2225            // packets such as upgrades do not actually have any data
2226            // payload
2227            if (pkt->hasData())
2228                pkt->setDataFromBlock(blk->data, blkSize);
2229        }
2230    }
2231
2232    if (!respond && is_deferred) {
2233        assert(pkt->needsResponse());
2234
2235        // if we copied the deferred packet with the intention to
2236        // respond, but are not responding, then a cache above us must
2237        // be, and we can use this as the indication of whether this
2238        // is a packet where we created a copy of the request or not
2239        if (!pkt->cacheResponding()) {
2240            delete pkt->req;
2241        }
2242
2243        delete pkt;
2244    }
2245
2246    // Do this last in case it deallocates block data or something
2247    // like that
2248    if (blk_valid && invalidate) {
2249        invalidateBlock(blk);
2250        DPRINTF(Cache, "new state is %s\n", blk->print());
2251    }
2252
2253    return snoop_delay;
2254}
2255
2256
2257void
2258Cache::recvTimingSnoopReq(PacketPtr pkt)
2259{
2260    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2261
2262    // Snoops shouldn't happen when bypassing caches
2263    assert(!system->bypassCaches());
2264
2265    // no need to snoop requests that are not in range
2266    if (!inRange(pkt->getAddr())) {
2267        return;
2268    }
2269
2270    bool is_secure = pkt->isSecure();
2271    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2272
2273    Addr blk_addr = pkt->getBlockAddr(blkSize);
2274    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2275
2276    // Update the latency cost of the snoop so that the crossbar can
2277    // account for it. Do not overwrite what other neighbouring caches
2278    // have already done, rather take the maximum. The update is
2279    // tentative, for cases where we return before an upward snoop
2280    // happens below.
2281    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2282                                         lookupLatency * clockPeriod());
2283
2284    // Inform request(Prefetch, CleanEvict or Writeback) from below of
2285    // MSHR hit, set setBlockCached.
2286    if (mshr && pkt->mustCheckAbove()) {
2287        DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2288                "mshr hit\n", pkt->print());
2289        pkt->setBlockCached();
2290        return;
2291    }
2292
2293    // Bypass any existing cache maintenance requests if the request
2294    // has been satisfied already (i.e., the dirty block has been
2295    // found).
2296    if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
2297        return;
2298    }
2299
2300    // Let the MSHR itself track the snoop and decide whether we want
2301    // to go ahead and do the regular cache snoop
2302    if (mshr && mshr->handleSnoop(pkt, order++)) {
2303        DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2304                "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2305                mshr->print());
2306
2307        if (mshr->getNumTargets() > numTarget)
2308            warn("allocating bonus target for snoop"); //handle later
2309        return;
2310    }
2311
2312    //We also need to check the writeback buffers and handle those
2313    WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2314    if (wb_entry) {
2315        DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2316                pkt->getAddr(), is_secure ? "s" : "ns");
2317        // Expect to see only Writebacks and/or CleanEvicts here, both of
2318        // which should not be generated for uncacheable data.
2319        assert(!wb_entry->isUncacheable());
2320        // There should only be a single request responsible for generating
2321        // Writebacks/CleanEvicts.
2322        assert(wb_entry->getNumTargets() == 1);
2323        PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2324        assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2325
2326        if (pkt->isEviction()) {
2327            // if the block is found in the write queue, set the BLOCK_CACHED
2328            // flag for Writeback/CleanEvict snoop. On return the snoop will
2329            // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2330            // any CleanEvicts from travelling down the memory hierarchy.
2331            pkt->setBlockCached();
2332            DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2333                    "hit\n", __func__, pkt->print());
2334            return;
2335        }
2336
2337        // conceptually writebacks are no different to other blocks in
2338        // this cache, so the behaviour is modelled after handleSnoop,
2339        // the difference being that instead of querying the block
2340        // state to determine if it is dirty and writable, we use the
2341        // command and fields of the writeback packet
2342        bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2343            pkt->needsResponse();
2344        bool have_writable = !wb_pkt->hasSharers();
2345        bool invalidate = pkt->isInvalidate();
2346
2347        if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2348            assert(!pkt->needsWritable());
2349            pkt->setHasSharers();
2350            wb_pkt->setHasSharers();
2351        }
2352
2353        if (respond) {
2354            pkt->setCacheResponding();
2355
2356            if (have_writable) {
2357                pkt->setResponderHadWritable();
2358            }
2359
2360            doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2361                                   false, false);
2362        }
2363
2364        if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2365            // Invalidation trumps our writeback... discard here
2366            // Note: markInService will remove entry from writeback buffer.
2367            markInService(wb_entry);
2368            delete wb_pkt;
2369        }
2370    }
2371
2372    // If this was a shared writeback, there may still be
2373    // other shared copies above that require invalidation.
2374    // We could be more selective and return here if the
2375    // request is non-exclusive or if the writeback is
2376    // exclusive.
2377    uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2378
2379    // Override what we did when we first saw the snoop, as we now
2380    // also have the cost of the upwards snoops to account for
2381    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2382                                         lookupLatency * clockPeriod());
2383}
2384
2385bool
2386Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2387{
2388    // Express snoop responses from master to slave, e.g., from L1 to L2
2389    cache->recvTimingSnoopResp(pkt);
2390    return true;
2391}
2392
2393Tick
2394Cache::recvAtomicSnoop(PacketPtr pkt)
2395{
2396    // Snoops shouldn't happen when bypassing caches
2397    assert(!system->bypassCaches());
2398
2399    // no need to snoop requests that are not in range.
2400    if (!inRange(pkt->getAddr())) {
2401        return 0;
2402    }
2403
2404    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2405    uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2406    return snoop_delay + lookupLatency * clockPeriod();
2407}
2408
2409
2410QueueEntry*
2411Cache::getNextQueueEntry()
2412{
2413    // Check both MSHR queue and write buffer for potential requests,
2414    // note that null does not mean there is no request, it could
2415    // simply be that it is not ready
2416    MSHR *miss_mshr  = mshrQueue.getNext();
2417    WriteQueueEntry *wq_entry = writeBuffer.getNext();
2418
2419    // If we got a write buffer request ready, first priority is a
2420    // full write buffer, otherwise we favour the miss requests
2421    if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2422        // need to search MSHR queue for conflicting earlier miss.
2423        MSHR *conflict_mshr =
2424            mshrQueue.findPending(wq_entry->blkAddr,
2425                                  wq_entry->isSecure);
2426
2427        if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2428            // Service misses in order until conflict is cleared.
2429            return conflict_mshr;
2430
2431            // @todo Note that we ignore the ready time of the conflict here
2432        }
2433
2434        // No conflicts; issue write
2435        return wq_entry;
2436    } else if (miss_mshr) {
2437        // need to check for conflicting earlier writeback
2438        WriteQueueEntry *conflict_mshr =
2439            writeBuffer.findPending(miss_mshr->blkAddr,
2440                                    miss_mshr->isSecure);
2441        if (conflict_mshr) {
2442            // not sure why we don't check order here... it was in the
2443            // original code but commented out.
2444
2445            // The only way this happens is if we are
2446            // doing a write and we didn't have permissions
2447            // then subsequently saw a writeback (owned got evicted)
2448            // We need to make sure to perform the writeback first
2449            // To preserve the dirty data, then we can issue the write
2450
2451            // should we return wq_entry here instead?  I.e. do we
2452            // have to flush writes in order?  I don't think so... not
2453            // for Alpha anyway.  Maybe for x86?
2454            return conflict_mshr;
2455
2456            // @todo Note that we ignore the ready time of the conflict here
2457        }
2458
2459        // No conflicts; issue read
2460        return miss_mshr;
2461    }
2462
2463    // fall through... no pending requests.  Try a prefetch.
2464    assert(!miss_mshr && !wq_entry);
2465    if (prefetcher && mshrQueue.canPrefetch()) {
2466        // If we have a miss queue slot, we can try a prefetch
2467        PacketPtr pkt = prefetcher->getPacket();
2468        if (pkt) {
2469            Addr pf_addr = pkt->getBlockAddr(blkSize);
2470            if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2471                !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2472                !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2473                // Update statistic on number of prefetches issued
2474                // (hwpf_mshr_misses)
2475                assert(pkt->req->masterId() < system->maxMasters());
2476                mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2477
2478                // allocate an MSHR and return it, note
2479                // that we send the packet straight away, so do not
2480                // schedule the send
2481                return allocateMissBuffer(pkt, curTick(), false);
2482            } else {
2483                // free the request and packet
2484                delete pkt->req;
2485                delete pkt;
2486            }
2487        }
2488    }
2489
2490    return nullptr;
2491}
2492
2493bool
2494Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2495{
2496    if (!forwardSnoops)
2497        return false;
2498    // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2499    // Writeback snoops into upper level caches to check for copies of the
2500    // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2501    // packet, the cache can inform the crossbar below of presence or absence
2502    // of the block.
2503    if (is_timing) {
2504        Packet snoop_pkt(pkt, true, false);
2505        snoop_pkt.setExpressSnoop();
2506        // Assert that packet is either Writeback or CleanEvict and not a
2507        // prefetch request because prefetch requests need an MSHR and may
2508        // generate a snoop response.
2509        assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2510        snoop_pkt.senderState = nullptr;
2511        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2512        // Writeback/CleanEvict snoops do not generate a snoop response.
2513        assert(!(snoop_pkt.cacheResponding()));
2514        return snoop_pkt.isBlockCached();
2515    } else {
2516        cpuSidePort->sendAtomicSnoop(pkt);
2517        return pkt->isBlockCached();
2518    }
2519}
2520
2521Tick
2522Cache::nextQueueReadyTime() const
2523{
2524    Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2525                              writeBuffer.nextReadyTime());
2526
2527    // Don't signal prefetch ready time if no MSHRs available
2528    // Will signal once enoguh MSHRs are deallocated
2529    if (prefetcher && mshrQueue.canPrefetch()) {
2530        nextReady = std::min(nextReady,
2531                             prefetcher->nextPrefetchReadyTime());
2532    }
2533
2534    return nextReady;
2535}
2536
2537bool
2538Cache::sendMSHRQueuePacket(MSHR* mshr)
2539{
2540    assert(mshr);
2541
2542    // use request from 1st target
2543    PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2544
2545    DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2546
2547    CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2548
2549    if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2550        // we should never have hardware prefetches to allocated
2551        // blocks
2552        assert(blk == nullptr);
2553
2554        // We need to check the caches above us to verify that
2555        // they don't have a copy of this block in the dirty state
2556        // at the moment. Without this check we could get a stale
2557        // copy from memory that might get used in place of the
2558        // dirty one.
2559        Packet snoop_pkt(tgt_pkt, true, false);
2560        snoop_pkt.setExpressSnoop();
2561        // We are sending this packet upwards, but if it hits we will
2562        // get a snoop response that we end up treating just like a
2563        // normal response, hence it needs the MSHR as its sender
2564        // state
2565        snoop_pkt.senderState = mshr;
2566        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2567
2568        // Check to see if the prefetch was squashed by an upper cache (to
2569        // prevent us from grabbing the line) or if a Check to see if a
2570        // writeback arrived between the time the prefetch was placed in
2571        // the MSHRs and when it was selected to be sent or if the
2572        // prefetch was squashed by an upper cache.
2573
2574        // It is important to check cacheResponding before
2575        // prefetchSquashed. If another cache has committed to
2576        // responding, it will be sending a dirty response which will
2577        // arrive at the MSHR allocated for this request. Checking the
2578        // prefetchSquash first may result in the MSHR being
2579        // prematurely deallocated.
2580        if (snoop_pkt.cacheResponding()) {
2581            auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2582            assert(r.second);
2583
2584            // if we are getting a snoop response with no sharers it
2585            // will be allocated as Modified
2586            bool pending_modified_resp = !snoop_pkt.hasSharers();
2587            markInService(mshr, pending_modified_resp);
2588
2589            DPRINTF(Cache, "Upward snoop of prefetch for addr"
2590                    " %#x (%s) hit\n",
2591                    tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2592            return false;
2593        }
2594
2595        if (snoop_pkt.isBlockCached()) {
2596            DPRINTF(Cache, "Block present, prefetch squashed by cache.  "
2597                    "Deallocating mshr target %#x.\n",
2598                    mshr->blkAddr);
2599
2600            // Deallocate the mshr target
2601            if (mshrQueue.forceDeallocateTarget(mshr)) {
2602                // Clear block if this deallocation resulted freed an
2603                // mshr when all had previously been utilized
2604                clearBlocked(Blocked_NoMSHRs);
2605            }
2606
2607            // given that no response is expected, delete Request and Packet
2608            delete tgt_pkt->req;
2609            delete tgt_pkt;
2610
2611            return false;
2612        }
2613    }
2614
2615    // either a prefetch that is not present upstream, or a normal
2616    // MSHR request, proceed to get the packet to send downstream
2617    PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2618
2619    mshr->isForward = (pkt == nullptr);
2620
2621    if (mshr->isForward) {
2622        // not a cache block request, but a response is expected
2623        // make copy of current packet to forward, keep current
2624        // copy for response handling
2625        pkt = new Packet(tgt_pkt, false, true);
2626        assert(!pkt->isWrite());
2627    }
2628
2629    // play it safe and append (rather than set) the sender state,
2630    // as forwarded packets may already have existing state
2631    pkt->pushSenderState(mshr);
2632
2633    if (pkt->isClean() && blk && blk->isDirty()) {
2634        // A cache clean opearation is looking for a dirty block. Mark
2635        // the packet so that the destination xbar can determine that
2636        // there will be a follow-up write packet as well.
2637        pkt->setSatisfied();
2638    }
2639
2640    if (!memSidePort->sendTimingReq(pkt)) {
2641        // we are awaiting a retry, but we
2642        // delete the packet and will be creating a new packet
2643        // when we get the opportunity
2644        delete pkt;
2645
2646        // note that we have now masked any requestBus and
2647        // schedSendEvent (we will wait for a retry before
2648        // doing anything), and this is so even if we do not
2649        // care about this packet and might override it before
2650        // it gets retried
2651        return true;
2652    } else {
2653        // As part of the call to sendTimingReq the packet is
2654        // forwarded to all neighbouring caches (and any caches
2655        // above them) as a snoop. Thus at this point we know if
2656        // any of the neighbouring caches are responding, and if
2657        // so, we know it is dirty, and we can determine if it is
2658        // being passed as Modified, making our MSHR the ordering
2659        // point
2660        bool pending_modified_resp = !pkt->hasSharers() &&
2661            pkt->cacheResponding();
2662        markInService(mshr, pending_modified_resp);
2663        if (pkt->isClean() && blk && blk->isDirty()) {
2664            // A cache clean opearation is looking for a dirty
2665            // block. If a dirty block is encountered a WriteClean
2666            // will update any copies to the path to the memory
2667            // until the point of reference.
2668            DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
2669                    __func__, pkt->print(), blk->print());
2670            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
2671                                             pkt->id);
2672            PacketList writebacks;
2673            writebacks.push_back(wb_pkt);
2674            doWritebacks(writebacks, 0);
2675        }
2676
2677        return false;
2678    }
2679}
2680
2681bool
2682Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2683{
2684    assert(wq_entry);
2685
2686    // always a single target for write queue entries
2687    PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2688
2689    DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2690
2691    // forward as is, both for evictions and uncacheable writes
2692    if (!memSidePort->sendTimingReq(tgt_pkt)) {
2693        // note that we have now masked any requestBus and
2694        // schedSendEvent (we will wait for a retry before
2695        // doing anything), and this is so even if we do not
2696        // care about this packet and might override it before
2697        // it gets retried
2698        return true;
2699    } else {
2700        markInService(wq_entry);
2701        return false;
2702    }
2703}
2704
2705void
2706Cache::serialize(CheckpointOut &cp) const
2707{
2708    bool dirty(isDirty());
2709
2710    if (dirty) {
2711        warn("*** The cache still contains dirty data. ***\n");
2712        warn("    Make sure to drain the system using the correct flags.\n");
2713        warn("    This checkpoint will not restore correctly and dirty data "
2714             "    in the cache will be lost!\n");
2715    }
2716
2717    // Since we don't checkpoint the data in the cache, any dirty data
2718    // will be lost when restoring from a checkpoint of a system that
2719    // wasn't drained properly. Flag the checkpoint as invalid if the
2720    // cache contains dirty data.
2721    bool bad_checkpoint(dirty);
2722    SERIALIZE_SCALAR(bad_checkpoint);
2723}
2724
2725void
2726Cache::unserialize(CheckpointIn &cp)
2727{
2728    bool bad_checkpoint;
2729    UNSERIALIZE_SCALAR(bad_checkpoint);
2730    if (bad_checkpoint) {
2731        fatal("Restoring from checkpoints with dirty caches is not supported "
2732              "in the classic memory system. Please remove any caches or "
2733              " drain them properly before taking checkpoints.\n");
2734    }
2735}
2736
2737///////////////
2738//
2739// CpuSidePort
2740//
2741///////////////
2742
2743AddrRangeList
2744Cache::CpuSidePort::getAddrRanges() const
2745{
2746    return cache->getAddrRanges();
2747}
2748
2749bool
2750Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2751{
2752    assert(!cache->system->bypassCaches());
2753
2754    // always let express snoop packets through if even if blocked
2755    if (pkt->isExpressSnoop()) {
2756        return true;
2757    } else if (isBlocked() || mustSendRetry) {
2758        // either already committed to send a retry, or blocked
2759        mustSendRetry = true;
2760        return false;
2761    }
2762    mustSendRetry = false;
2763    return true;
2764}
2765
2766bool
2767Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2768{
2769    assert(!cache->system->bypassCaches());
2770
2771    // always let express snoop packets through if even if blocked
2772    if (pkt->isExpressSnoop()) {
2773        bool M5_VAR_USED bypass_success = cache->recvTimingReq(pkt);
2774        assert(bypass_success);
2775        return true;
2776    }
2777
2778    return tryTiming(pkt) && cache->recvTimingReq(pkt);
2779}
2780
2781Tick
2782Cache::CpuSidePort::recvAtomic(PacketPtr pkt)
2783{
2784    return cache->recvAtomic(pkt);
2785}
2786
2787void
2788Cache::CpuSidePort::recvFunctional(PacketPtr pkt)
2789{
2790    // functional request
2791    cache->functionalAccess(pkt, true);
2792}
2793
2794Cache::
2795CpuSidePort::CpuSidePort(const std::string &_name, Cache *_cache,
2796                         const std::string &_label)
2797    : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2798{
2799}
2800
2801Cache*
2802CacheParams::create()
2803{
2804    assert(tags);
2805
2806    return new Cache(this);
2807}
2808///////////////
2809//
2810// MemSidePort
2811//
2812///////////////
2813
2814bool
2815Cache::MemSidePort::recvTimingResp(PacketPtr pkt)
2816{
2817    cache->recvTimingResp(pkt);
2818    return true;
2819}
2820
2821// Express snooping requests to memside port
2822void
2823Cache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2824{
2825    // handle snooping requests
2826    cache->recvTimingSnoopReq(pkt);
2827}
2828
2829Tick
2830Cache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2831{
2832    return cache->recvAtomicSnoop(pkt);
2833}
2834
2835void
2836Cache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2837{
2838    // functional snoop (note that in contrast to atomic we don't have
2839    // a specific functionalSnoop method, as they have the same
2840    // behaviour regardless)
2841    cache->functionalAccess(pkt, false);
2842}
2843
2844void
2845Cache::CacheReqPacketQueue::sendDeferredPacket()
2846{
2847    // sanity check
2848    assert(!waitingOnRetry);
2849
2850    // there should never be any deferred request packets in the
2851    // queue, instead we resly on the cache to provide the packets
2852    // from the MSHR queue or write queue
2853    assert(deferredPacketReadyTime() == MaxTick);
2854
2855    // check for request packets (requests & writebacks)
2856    QueueEntry* entry = cache.getNextQueueEntry();
2857
2858    if (!entry) {
2859        // can happen if e.g. we attempt a writeback and fail, but
2860        // before the retry, the writeback is eliminated because
2861        // we snoop another cache's ReadEx.
2862    } else {
2863        // let our snoop responses go first if there are responses to
2864        // the same addresses
2865        if (checkConflictingSnoop(entry->blkAddr)) {
2866            return;
2867        }
2868        waitingOnRetry = entry->sendPacket(cache);
2869    }
2870
2871    // if we succeeded and are not waiting for a retry, schedule the
2872    // next send considering when the next queue is ready, note that
2873    // snoop responses have their own packet queue and thus schedule
2874    // their own events
2875    if (!waitingOnRetry) {
2876        schedSendEvent(cache.nextQueueReadyTime());
2877    }
2878}
2879
2880Cache::
2881MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2882                         const std::string &_label)
2883    : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2884      _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2885      _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2886{
2887}
2888