cache.cc revision 12702
1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 *          Dave Greene
43 *          Nathan Binkert
44 *          Steve Reinhardt
45 *          Ron Dreslinski
46 *          Andreas Sandberg
47 *          Nikos Nikoleris
48 */
49
50/**
51 * @file
52 * Cache definitions.
53 */
54
55#include "mem/cache/cache.hh"
56
57#include "base/logging.hh"
58#include "base/types.hh"
59#include "debug/Cache.hh"
60#include "debug/CachePort.hh"
61#include "debug/CacheTags.hh"
62#include "debug/CacheVerbose.hh"
63#include "mem/cache/blk.hh"
64#include "mem/cache/mshr.hh"
65#include "mem/cache/prefetch/base.hh"
66#include "sim/sim_exit.hh"
67
68Cache::Cache(const CacheParams *p)
69    : BaseCache(p, p->system->cacheLineSize()),
70      tags(p->tags),
71      prefetcher(p->prefetcher),
72      doFastWrites(true),
73      prefetchOnAccess(p->prefetch_on_access),
74      clusivity(p->clusivity),
75      writebackClean(p->writeback_clean),
76      tempBlockWriteback(nullptr),
77      writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
78                                    name(), false,
79                                    EventBase::Delayed_Writeback_Pri)
80{
81    tempBlock = new CacheBlk();
82    tempBlock->data = new uint8_t[blkSize];
83
84    cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
85                                  "CpuSidePort");
86    memSidePort = new MemSidePort(p->name + ".mem_side", this,
87                                  "MemSidePort");
88
89    tags->setCache(this);
90    if (prefetcher)
91        prefetcher->setCache(this);
92}
93
94Cache::~Cache()
95{
96    delete [] tempBlock->data;
97    delete tempBlock;
98
99    delete cpuSidePort;
100    delete memSidePort;
101}
102
103void
104Cache::regStats()
105{
106    BaseCache::regStats();
107}
108
109void
110Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
111{
112    assert(pkt->isRequest());
113
114    uint64_t overwrite_val;
115    bool overwrite_mem;
116    uint64_t condition_val64;
117    uint32_t condition_val32;
118
119    int offset = tags->extractBlkOffset(pkt->getAddr());
120    uint8_t *blk_data = blk->data + offset;
121
122    assert(sizeof(uint64_t) >= pkt->getSize());
123
124    overwrite_mem = true;
125    // keep a copy of our possible write value, and copy what is at the
126    // memory address into the packet
127    pkt->writeData((uint8_t *)&overwrite_val);
128    pkt->setData(blk_data);
129
130    if (pkt->req->isCondSwap()) {
131        if (pkt->getSize() == sizeof(uint64_t)) {
132            condition_val64 = pkt->req->getExtraData();
133            overwrite_mem = !std::memcmp(&condition_val64, blk_data,
134                                         sizeof(uint64_t));
135        } else if (pkt->getSize() == sizeof(uint32_t)) {
136            condition_val32 = (uint32_t)pkt->req->getExtraData();
137            overwrite_mem = !std::memcmp(&condition_val32, blk_data,
138                                         sizeof(uint32_t));
139        } else
140            panic("Invalid size for conditional read/write\n");
141    }
142
143    if (overwrite_mem) {
144        std::memcpy(blk_data, &overwrite_val, pkt->getSize());
145        blk->status |= BlkDirty;
146    }
147}
148
149
150void
151Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
152                      bool deferred_response, bool pending_downgrade)
153{
154    assert(pkt->isRequest());
155
156    assert(blk && blk->isValid());
157    // Occasionally this is not true... if we are a lower-level cache
158    // satisfying a string of Read and ReadEx requests from
159    // upper-level caches, a Read will mark the block as shared but we
160    // can satisfy a following ReadEx anyway since we can rely on the
161    // Read requester(s) to have buffered the ReadEx snoop and to
162    // invalidate their blocks after receiving them.
163    // assert(!pkt->needsWritable() || blk->isWritable());
164    assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
165
166    // Check RMW operations first since both isRead() and
167    // isWrite() will be true for them
168    if (pkt->cmd == MemCmd::SwapReq) {
169        cmpAndSwap(blk, pkt);
170    } else if (pkt->isWrite()) {
171        // we have the block in a writable state and can go ahead,
172        // note that the line may be also be considered writable in
173        // downstream caches along the path to memory, but always
174        // Exclusive, and never Modified
175        assert(blk->isWritable());
176        // Write or WriteLine at the first cache with block in writable state
177        if (blk->checkWrite(pkt)) {
178            pkt->writeDataToBlock(blk->data, blkSize);
179        }
180        // Always mark the line as dirty (and thus transition to the
181        // Modified state) even if we are a failed StoreCond so we
182        // supply data to any snoops that have appended themselves to
183        // this cache before knowing the store will fail.
184        blk->status |= BlkDirty;
185        DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
186    } else if (pkt->isRead()) {
187        if (pkt->isLLSC()) {
188            blk->trackLoadLocked(pkt);
189        }
190
191        // all read responses have a data payload
192        assert(pkt->hasRespData());
193        pkt->setDataFromBlock(blk->data, blkSize);
194
195        // determine if this read is from a (coherent) cache or not
196        if (pkt->fromCache()) {
197            assert(pkt->getSize() == blkSize);
198            // special handling for coherent block requests from
199            // upper-level caches
200            if (pkt->needsWritable()) {
201                // sanity check
202                assert(pkt->cmd == MemCmd::ReadExReq ||
203                       pkt->cmd == MemCmd::SCUpgradeFailReq);
204                assert(!pkt->hasSharers());
205
206                // if we have a dirty copy, make sure the recipient
207                // keeps it marked dirty (in the modified state)
208                if (blk->isDirty()) {
209                    pkt->setCacheResponding();
210                    blk->status &= ~BlkDirty;
211                }
212            } else if (blk->isWritable() && !pending_downgrade &&
213                       !pkt->hasSharers() &&
214                       pkt->cmd != MemCmd::ReadCleanReq) {
215                // we can give the requester a writable copy on a read
216                // request if:
217                // - we have a writable copy at this level (& below)
218                // - we don't have a pending snoop from below
219                //   signaling another read request
220                // - no other cache above has a copy (otherwise it
221                //   would have set hasSharers flag when
222                //   snooping the packet)
223                // - the read has explicitly asked for a clean
224                //   copy of the line
225                if (blk->isDirty()) {
226                    // special considerations if we're owner:
227                    if (!deferred_response) {
228                        // respond with the line in Modified state
229                        // (cacheResponding set, hasSharers not set)
230                        pkt->setCacheResponding();
231
232                        // if this cache is mostly inclusive, we
233                        // keep the block in the Exclusive state,
234                        // and pass it upwards as Modified
235                        // (writable and dirty), hence we have
236                        // multiple caches, all on the same path
237                        // towards memory, all considering the
238                        // same block writable, but only one
239                        // considering it Modified
240
241                        // we get away with multiple caches (on
242                        // the same path to memory) considering
243                        // the block writeable as we always enter
244                        // the cache hierarchy through a cache,
245                        // and first snoop upwards in all other
246                        // branches
247                        blk->status &= ~BlkDirty;
248                    } else {
249                        // if we're responding after our own miss,
250                        // there's a window where the recipient didn't
251                        // know it was getting ownership and may not
252                        // have responded to snoops correctly, so we
253                        // have to respond with a shared line
254                        pkt->setHasSharers();
255                    }
256                }
257            } else {
258                // otherwise only respond with a shared copy
259                pkt->setHasSharers();
260            }
261        }
262    } else if (pkt->isUpgrade()) {
263        // sanity check
264        assert(!pkt->hasSharers());
265
266        if (blk->isDirty()) {
267            // we were in the Owned state, and a cache above us that
268            // has the line in Shared state needs to be made aware
269            // that the data it already has is in fact dirty
270            pkt->setCacheResponding();
271            blk->status &= ~BlkDirty;
272        }
273    } else {
274        assert(pkt->isInvalidate());
275        invalidateBlock(blk);
276        DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
277                pkt->print());
278    }
279}
280
281/////////////////////////////////////////////////////
282//
283// Access path: requests coming in from the CPU side
284//
285/////////////////////////////////////////////////////
286
287bool
288Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
289              PacketList &writebacks)
290{
291    // sanity check
292    assert(pkt->isRequest());
293
294    chatty_assert(!(isReadOnly && pkt->isWrite()),
295                  "Should never see a write in a read-only cache %s\n",
296                  name());
297
298    DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
299
300    if (pkt->req->isUncacheable()) {
301        DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
302
303        // flush and invalidate any existing block
304        CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
305        if (old_blk && old_blk->isValid()) {
306            if (old_blk->isDirty() || writebackClean)
307                writebacks.push_back(writebackBlk(old_blk));
308            else
309                writebacks.push_back(cleanEvictBlk(old_blk));
310            invalidateBlock(old_blk);
311        }
312
313        blk = nullptr;
314        // lookupLatency is the latency in case the request is uncacheable.
315        lat = lookupLatency;
316        return false;
317    }
318
319    // Here lat is the value passed as parameter to accessBlock() function
320    // that can modify its value.
321    blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
322
323    DPRINTF(Cache, "%s %s\n", pkt->print(),
324            blk ? "hit " + blk->print() : "miss");
325
326    if (pkt->req->isCacheMaintenance()) {
327        // A cache maintenance operation is always forwarded to the
328        // memory below even if the block is found in dirty state.
329
330        // We defer any changes to the state of the block until we
331        // create and mark as in service the mshr for the downstream
332        // packet.
333        return false;
334    }
335
336    if (pkt->isEviction()) {
337        // We check for presence of block in above caches before issuing
338        // Writeback or CleanEvict to write buffer. Therefore the only
339        // possible cases can be of a CleanEvict packet coming from above
340        // encountering a Writeback generated in this cache peer cache and
341        // waiting in the write buffer. Cases of upper level peer caches
342        // generating CleanEvict and Writeback or simply CleanEvict and
343        // CleanEvict almost simultaneously will be caught by snoops sent out
344        // by crossbar.
345        WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
346                                                          pkt->isSecure());
347        if (wb_entry) {
348            assert(wb_entry->getNumTargets() == 1);
349            PacketPtr wbPkt = wb_entry->getTarget()->pkt;
350            assert(wbPkt->isWriteback());
351
352            if (pkt->isCleanEviction()) {
353                // The CleanEvict and WritebackClean snoops into other
354                // peer caches of the same level while traversing the
355                // crossbar. If a copy of the block is found, the
356                // packet is deleted in the crossbar. Hence, none of
357                // the other upper level caches connected to this
358                // cache have the block, so we can clear the
359                // BLOCK_CACHED flag in the Writeback if set and
360                // discard the CleanEvict by returning true.
361                wbPkt->clearBlockCached();
362                return true;
363            } else {
364                assert(pkt->cmd == MemCmd::WritebackDirty);
365                // Dirty writeback from above trumps our clean
366                // writeback... discard here
367                // Note: markInService will remove entry from writeback buffer.
368                markInService(wb_entry);
369                delete wbPkt;
370            }
371        }
372    }
373
374    // Writeback handling is special case.  We can write the block into
375    // the cache without having a writeable copy (or any copy at all).
376    if (pkt->isWriteback()) {
377        assert(blkSize == pkt->getSize());
378
379        // we could get a clean writeback while we are having
380        // outstanding accesses to a block, do the simple thing for
381        // now and drop the clean writeback so that we do not upset
382        // any ordering/decisions about ownership already taken
383        if (pkt->cmd == MemCmd::WritebackClean &&
384            mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
385            DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
386                    "dropping\n", pkt->getAddr());
387            return true;
388        }
389
390        if (blk == nullptr) {
391            // need to do a replacement
392            blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
393            if (blk == nullptr) {
394                // no replaceable block available: give up, fwd to next level.
395                incMissCount(pkt);
396                return false;
397            }
398            tags->insertBlock(pkt, blk);
399
400            blk->status |= (BlkValid | BlkReadable);
401        }
402        // only mark the block dirty if we got a writeback command,
403        // and leave it as is for a clean writeback
404        if (pkt->cmd == MemCmd::WritebackDirty) {
405            assert(!blk->isDirty());
406            blk->status |= BlkDirty;
407        }
408        // if the packet does not have sharers, it is passing
409        // writable, and we got the writeback in Modified or Exclusive
410        // state, if not we are in the Owned or Shared state
411        if (!pkt->hasSharers()) {
412            blk->status |= BlkWritable;
413        }
414        // nothing else to do; writeback doesn't expect response
415        assert(!pkt->needsResponse());
416        pkt->writeDataToBlock(blk->data, blkSize);
417        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
418        incHitCount(pkt);
419        // populate the time when the block will be ready to access.
420        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
421            pkt->payloadDelay;
422        return true;
423    } else if (pkt->cmd == MemCmd::CleanEvict) {
424        if (blk != nullptr) {
425            // Found the block in the tags, need to stop CleanEvict from
426            // propagating further down the hierarchy. Returning true will
427            // treat the CleanEvict like a satisfied write request and delete
428            // it.
429            return true;
430        }
431        // We didn't find the block here, propagate the CleanEvict further
432        // down the memory hierarchy. Returning false will treat the CleanEvict
433        // like a Writeback which could not find a replaceable block so has to
434        // go to next level.
435        return false;
436    } else if (pkt->cmd == MemCmd::WriteClean) {
437        // WriteClean handling is a special case. We can allocate a
438        // block directly if it doesn't exist and we can update the
439        // block immediately. The WriteClean transfers the ownership
440        // of the block as well.
441        assert(blkSize == pkt->getSize());
442
443        if (!blk) {
444            if (pkt->writeThrough()) {
445                // if this is a write through packet, we don't try to
446                // allocate if the block is not present
447                return false;
448            } else {
449                // a writeback that misses needs to allocate a new block
450                blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
451                                    writebacks);
452                if (!blk) {
453                    // no replaceable block available: give up, fwd to
454                    // next level.
455                    incMissCount(pkt);
456                    return false;
457                }
458                tags->insertBlock(pkt, blk);
459
460                blk->status |= (BlkValid | BlkReadable);
461            }
462        }
463
464        // at this point either this is a writeback or a write-through
465        // write clean operation and the block is already in this
466        // cache, we need to update the data and the block flags
467        assert(blk);
468        assert(!blk->isDirty());
469        if (!pkt->writeThrough()) {
470            blk->status |= BlkDirty;
471        }
472        // nothing else to do; writeback doesn't expect response
473        assert(!pkt->needsResponse());
474        pkt->writeDataToBlock(blk->data, blkSize);
475        DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
476
477        incHitCount(pkt);
478        // populate the time when the block will be ready to access.
479        blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
480            pkt->payloadDelay;
481        // if this a write-through packet it will be sent to cache
482        // below
483        return !pkt->writeThrough();
484    } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
485                       blk->isReadable())) {
486        // OK to satisfy access
487        incHitCount(pkt);
488        satisfyRequest(pkt, blk);
489        maintainClusivity(pkt->fromCache(), blk);
490
491        return true;
492    }
493
494    // Can't satisfy access normally... either no block (blk == nullptr)
495    // or have block but need writable
496
497    incMissCount(pkt);
498
499    if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
500        // complete miss on store conditional... just give up now
501        pkt->req->setExtraData(0);
502        return true;
503    }
504
505    return false;
506}
507
508void
509Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
510{
511    if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
512        clusivity == Enums::mostly_excl) {
513        // if we have responded to a cache, and our block is still
514        // valid, but not dirty, and this cache is mostly exclusive
515        // with respect to the cache above, drop the block
516        invalidateBlock(blk);
517    }
518}
519
520void
521Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
522{
523    while (!writebacks.empty()) {
524        PacketPtr wbPkt = writebacks.front();
525        // We use forwardLatency here because we are copying writebacks to
526        // write buffer.
527
528        // Call isCachedAbove for Writebacks, CleanEvicts and
529        // WriteCleans to discover if the block is cached above.
530        if (isCachedAbove(wbPkt)) {
531            if (wbPkt->cmd == MemCmd::CleanEvict) {
532                // Delete CleanEvict because cached copies exist above. The
533                // packet destructor will delete the request object because
534                // this is a non-snoop request packet which does not require a
535                // response.
536                delete wbPkt;
537            } else if (wbPkt->cmd == MemCmd::WritebackClean) {
538                // clean writeback, do not send since the block is
539                // still cached above
540                assert(writebackClean);
541                delete wbPkt;
542            } else {
543                assert(wbPkt->cmd == MemCmd::WritebackDirty ||
544                       wbPkt->cmd == MemCmd::WriteClean);
545                // Set BLOCK_CACHED flag in Writeback and send below, so that
546                // the Writeback does not reset the bit corresponding to this
547                // address in the snoop filter below.
548                wbPkt->setBlockCached();
549                allocateWriteBuffer(wbPkt, forward_time);
550            }
551        } else {
552            // If the block is not cached above, send packet below. Both
553            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
554            // reset the bit corresponding to this address in the snoop filter
555            // below.
556            allocateWriteBuffer(wbPkt, forward_time);
557        }
558        writebacks.pop_front();
559    }
560}
561
562void
563Cache::doWritebacksAtomic(PacketList& writebacks)
564{
565    while (!writebacks.empty()) {
566        PacketPtr wbPkt = writebacks.front();
567        // Call isCachedAbove for both Writebacks and CleanEvicts. If
568        // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
569        // and discard CleanEvicts.
570        if (isCachedAbove(wbPkt, false)) {
571            if (wbPkt->cmd == MemCmd::WritebackDirty ||
572                wbPkt->cmd == MemCmd::WriteClean) {
573                // Set BLOCK_CACHED flag in Writeback and send below,
574                // so that the Writeback does not reset the bit
575                // corresponding to this address in the snoop filter
576                // below. We can discard CleanEvicts because cached
577                // copies exist above. Atomic mode isCachedAbove
578                // modifies packet to set BLOCK_CACHED flag
579                memSidePort->sendAtomic(wbPkt);
580            }
581        } else {
582            // If the block is not cached above, send packet below. Both
583            // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
584            // reset the bit corresponding to this address in the snoop filter
585            // below.
586            memSidePort->sendAtomic(wbPkt);
587        }
588        writebacks.pop_front();
589        // In case of CleanEvicts, the packet destructor will delete the
590        // request object because this is a non-snoop request packet which
591        // does not require a response.
592        delete wbPkt;
593    }
594}
595
596
597void
598Cache::recvTimingSnoopResp(PacketPtr pkt)
599{
600    DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
601
602    assert(pkt->isResponse());
603    assert(!system->bypassCaches());
604
605    // determine if the response is from a snoop request we created
606    // (in which case it should be in the outstandingSnoop), or if we
607    // merely forwarded someone else's snoop request
608    const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
609        outstandingSnoop.end();
610
611    if (!forwardAsSnoop) {
612        // the packet came from this cache, so sink it here and do not
613        // forward it
614        assert(pkt->cmd == MemCmd::HardPFResp);
615
616        outstandingSnoop.erase(pkt->req);
617
618        DPRINTF(Cache, "Got prefetch response from above for addr "
619                "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
620        recvTimingResp(pkt);
621        return;
622    }
623
624    // forwardLatency is set here because there is a response from an
625    // upper level cache.
626    // To pay the delay that occurs if the packet comes from the bus,
627    // we charge also headerDelay.
628    Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
629    // Reset the timing of the packet.
630    pkt->headerDelay = pkt->payloadDelay = 0;
631    memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
632}
633
634void
635Cache::promoteWholeLineWrites(PacketPtr pkt)
636{
637    // Cache line clearing instructions
638    if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
639        (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
640        pkt->cmd = MemCmd::WriteLineReq;
641        DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
642    }
643}
644
645void
646Cache::recvTimingReq(PacketPtr pkt)
647{
648    DPRINTF(CacheTags, "%s tags:\n%s\n", __func__, tags->print());
649
650    assert(pkt->isRequest());
651
652    // Just forward the packet if caches are disabled.
653    if (system->bypassCaches()) {
654        // @todo This should really enqueue the packet rather
655        bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
656        assert(success);
657        return;
658    }
659
660    promoteWholeLineWrites(pkt);
661
662    // Cache maintenance operations have to visit all the caches down
663    // to the specified xbar (PoC, PoU, etc.). Even if a cache above
664    // is responding we forward the packet to the memory below rather
665    // than creating an express snoop.
666    if (pkt->cacheResponding()) {
667        // a cache above us (but not where the packet came from) is
668        // responding to the request, in other words it has the line
669        // in Modified or Owned state
670        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
671                pkt->print());
672
673        // if the packet needs the block to be writable, and the cache
674        // that has promised to respond (setting the cache responding
675        // flag) is not providing writable (it is in Owned rather than
676        // the Modified state), we know that there may be other Shared
677        // copies in the system; go out and invalidate them all
678        assert(pkt->needsWritable() && !pkt->responderHadWritable());
679
680        // an upstream cache that had the line in Owned state
681        // (dirty, but not writable), is responding and thus
682        // transferring the dirty line from one branch of the
683        // cache hierarchy to another
684
685        // send out an express snoop and invalidate all other
686        // copies (snooping a packet that needs writable is the
687        // same as an invalidation), thus turning the Owned line
688        // into a Modified line, note that we don't invalidate the
689        // block in the current cache or any other cache on the
690        // path to memory
691
692        // create a downstream express snoop with cleared packet
693        // flags, there is no need to allocate any data as the
694        // packet is merely used to co-ordinate state transitions
695        Packet *snoop_pkt = new Packet(pkt, true, false);
696
697        // also reset the bus time that the original packet has
698        // not yet paid for
699        snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
700
701        // make this an instantaneous express snoop, and let the
702        // other caches in the system know that the another cache
703        // is responding, because we have found the authorative
704        // copy (Modified or Owned) that will supply the right
705        // data
706        snoop_pkt->setExpressSnoop();
707        snoop_pkt->setCacheResponding();
708
709        // this express snoop travels towards the memory, and at
710        // every crossbar it is snooped upwards thus reaching
711        // every cache in the system
712        bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt);
713        // express snoops always succeed
714        assert(success);
715
716        // main memory will delete the snoop packet
717
718        // queue for deletion, as opposed to immediate deletion, as
719        // the sending cache is still relying on the packet
720        pendingDelete.reset(pkt);
721
722        // no need to take any further action in this particular cache
723        // as an upstram cache has already committed to responding,
724        // and we have already sent out any express snoops in the
725        // section above to ensure all other copies in the system are
726        // invalidated
727        return;
728    }
729
730    // anything that is merely forwarded pays for the forward latency and
731    // the delay provided by the crossbar
732    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
733
734    // We use lookupLatency here because it is used to specify the latency
735    // to access.
736    Cycles lat = lookupLatency;
737    CacheBlk *blk = nullptr;
738    bool satisfied = false;
739    {
740        PacketList writebacks;
741        // Note that lat is passed by reference here. The function
742        // access() calls accessBlock() which can modify lat value.
743        satisfied = access(pkt, blk, lat, writebacks);
744
745        // copy writebacks to write buffer here to ensure they logically
746        // proceed anything happening below
747        doWritebacks(writebacks, forward_time);
748    }
749
750    // Here we charge the headerDelay that takes into account the latencies
751    // of the bus, if the packet comes from it.
752    // The latency charged it is just lat that is the value of lookupLatency
753    // modified by access() function, or if not just lookupLatency.
754    // In case of a hit we are neglecting response latency.
755    // In case of a miss we are neglecting forward latency.
756    Tick request_time = clockEdge(lat) + pkt->headerDelay;
757    // Here we reset the timing of the packet.
758    pkt->headerDelay = pkt->payloadDelay = 0;
759
760    // track time of availability of next prefetch, if any
761    Tick next_pf_time = MaxTick;
762
763    bool needsResponse = pkt->needsResponse();
764
765    if (satisfied) {
766        // should never be satisfying an uncacheable access as we
767        // flush and invalidate any existing block as part of the
768        // lookup
769        assert(!pkt->req->isUncacheable());
770
771        // hit (for all other request types)
772
773        if (prefetcher && (prefetchOnAccess ||
774                           (blk && blk->wasPrefetched()))) {
775            if (blk)
776                blk->status &= ~BlkHWPrefetched;
777
778            // Don't notify on SWPrefetch
779            if (!pkt->cmd.isSWPrefetch()) {
780                assert(!pkt->req->isCacheMaintenance());
781                next_pf_time = prefetcher->notify(pkt);
782            }
783        }
784
785        if (needsResponse) {
786            pkt->makeTimingResponse();
787            // @todo: Make someone pay for this
788            pkt->headerDelay = pkt->payloadDelay = 0;
789
790            // In this case we are considering request_time that takes
791            // into account the delay of the xbar, if any, and just
792            // lat, neglecting responseLatency, modelling hit latency
793            // just as lookupLatency or or the value of lat overriden
794            // by access(), that calls accessBlock() function.
795            cpuSidePort->schedTimingResp(pkt, request_time, true);
796        } else {
797            DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
798                    pkt->print());
799
800            // queue the packet for deletion, as the sending cache is
801            // still relying on it; if the block is found in access(),
802            // CleanEvict and Writeback messages will be deleted
803            // here as well
804            pendingDelete.reset(pkt);
805        }
806    } else {
807        // miss
808
809        Addr blk_addr = pkt->getBlockAddr(blkSize);
810
811        // ignore any existing MSHR if we are dealing with an
812        // uncacheable request
813        MSHR *mshr = pkt->req->isUncacheable() ? nullptr :
814            mshrQueue.findMatch(blk_addr, pkt->isSecure());
815
816        // Software prefetch handling:
817        // To keep the core from waiting on data it won't look at
818        // anyway, send back a response with dummy data. Miss handling
819        // will continue asynchronously. Unfortunately, the core will
820        // insist upon freeing original Packet/Request, so we have to
821        // create a new pair with a different lifecycle. Note that this
822        // processing happens before any MSHR munging on the behalf of
823        // this request because this new Request will be the one stored
824        // into the MSHRs, not the original.
825        if (pkt->cmd.isSWPrefetch()) {
826            assert(needsResponse);
827            assert(pkt->req->hasPaddr());
828            assert(!pkt->req->isUncacheable());
829
830            // There's no reason to add a prefetch as an additional target
831            // to an existing MSHR. If an outstanding request is already
832            // in progress, there is nothing for the prefetch to do.
833            // If this is the case, we don't even create a request at all.
834            PacketPtr pf = nullptr;
835
836            if (!mshr) {
837                // copy the request and create a new SoftPFReq packet
838                RequestPtr req = new Request(pkt->req->getPaddr(),
839                                             pkt->req->getSize(),
840                                             pkt->req->getFlags(),
841                                             pkt->req->masterId());
842                pf = new Packet(req, pkt->cmd);
843                pf->allocate();
844                assert(pf->getAddr() == pkt->getAddr());
845                assert(pf->getSize() == pkt->getSize());
846            }
847
848            pkt->makeTimingResponse();
849
850            // request_time is used here, taking into account lat and the delay
851            // charged if the packet comes from the xbar.
852            cpuSidePort->schedTimingResp(pkt, request_time, true);
853
854            // If an outstanding request is in progress (we found an
855            // MSHR) this is set to null
856            pkt = pf;
857        }
858
859        if (mshr) {
860            /// MSHR hit
861            /// @note writebacks will be checked in getNextMSHR()
862            /// for any conflicting requests to the same block
863
864            //@todo remove hw_pf here
865
866            // Coalesce unless it was a software prefetch (see above).
867            if (pkt) {
868                assert(!pkt->isWriteback());
869                // CleanEvicts corresponding to blocks which have
870                // outstanding requests in MSHRs are simply sunk here
871                if (pkt->cmd == MemCmd::CleanEvict) {
872                    pendingDelete.reset(pkt);
873                } else if (pkt->cmd == MemCmd::WriteClean) {
874                    // A WriteClean should never coalesce with any
875                    // outstanding cache maintenance requests.
876
877                    // We use forward_time here because there is an
878                    // uncached memory write, forwarded to WriteBuffer.
879                    allocateWriteBuffer(pkt, forward_time);
880                } else {
881                    DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
882                            pkt->print());
883
884                    assert(pkt->req->masterId() < system->maxMasters());
885                    mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
886                    // We use forward_time here because it is the same
887                    // considering new targets. We have multiple
888                    // requests for the same address here. It
889                    // specifies the latency to allocate an internal
890                    // buffer and to schedule an event to the queued
891                    // port and also takes into account the additional
892                    // delay of the xbar.
893                    mshr->allocateTarget(pkt, forward_time, order++,
894                                         allocOnFill(pkt->cmd));
895                    if (mshr->getNumTargets() == numTarget) {
896                        noTargetMSHR = mshr;
897                        setBlocked(Blocked_NoTargets);
898                        // need to be careful with this... if this mshr isn't
899                        // ready yet (i.e. time > curTick()), we don't want to
900                        // move it ahead of mshrs that are ready
901                        // mshrQueue.moveToFront(mshr);
902                    }
903                }
904                // We should call the prefetcher reguardless if the request is
905                // satisfied or not, reguardless if the request is in the MSHR
906                // or not.  The request could be a ReadReq hit, but still not
907                // satisfied (potentially because of a prior write to the same
908                // cache line.  So, even when not satisfied, tehre is an MSHR
909                // already allocated for this, we need to let the prefetcher
910                // know about the request
911                if (prefetcher) {
912                    // Don't notify on SWPrefetch
913                    if (!pkt->cmd.isSWPrefetch() &&
914                        !pkt->req->isCacheMaintenance())
915                        next_pf_time = prefetcher->notify(pkt);
916                }
917            }
918        } else {
919            // no MSHR
920            assert(pkt->req->masterId() < system->maxMasters());
921            if (pkt->req->isUncacheable()) {
922                mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
923            } else {
924                mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
925            }
926
927            if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
928                (pkt->req->isUncacheable() && pkt->isWrite())) {
929                // We use forward_time here because there is an
930                // uncached memory write, forwarded to WriteBuffer.
931                allocateWriteBuffer(pkt, forward_time);
932            } else {
933                if (blk && blk->isValid()) {
934                    // should have flushed and have no valid block
935                    assert(!pkt->req->isUncacheable());
936
937                    // If we have a write miss to a valid block, we
938                    // need to mark the block non-readable.  Otherwise
939                    // if we allow reads while there's an outstanding
940                    // write miss, the read could return stale data
941                    // out of the cache block... a more aggressive
942                    // system could detect the overlap (if any) and
943                    // forward data out of the MSHRs, but we don't do
944                    // that yet.  Note that we do need to leave the
945                    // block valid so that it stays in the cache, in
946                    // case we get an upgrade response (and hence no
947                    // new data) when the write miss completes.
948                    // As long as CPUs do proper store/load forwarding
949                    // internally, and have a sufficiently weak memory
950                    // model, this is probably unnecessary, but at some
951                    // point it must have seemed like we needed it...
952                    assert((pkt->needsWritable() && !blk->isWritable()) ||
953                           pkt->req->isCacheMaintenance());
954                    blk->status &= ~BlkReadable;
955                }
956                // Here we are using forward_time, modelling the latency of
957                // a miss (outbound) just as forwardLatency, neglecting the
958                // lookupLatency component.
959                allocateMissBuffer(pkt, forward_time);
960            }
961
962            if (prefetcher) {
963                // Don't notify on SWPrefetch
964                if (!pkt->cmd.isSWPrefetch() &&
965                    !pkt->req->isCacheMaintenance())
966                    next_pf_time = prefetcher->notify(pkt);
967            }
968        }
969    }
970
971    if (next_pf_time != MaxTick)
972        schedMemSideSendEvent(next_pf_time);
973}
974
975PacketPtr
976Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
977                        bool needsWritable) const
978{
979    // should never see evictions here
980    assert(!cpu_pkt->isEviction());
981
982    bool blkValid = blk && blk->isValid();
983
984    if (cpu_pkt->req->isUncacheable() ||
985        (!blkValid && cpu_pkt->isUpgrade()) ||
986        cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
987        // uncacheable requests and upgrades from upper-level caches
988        // that missed completely just go through as is
989        return nullptr;
990    }
991
992    assert(cpu_pkt->needsResponse());
993
994    MemCmd cmd;
995    // @TODO make useUpgrades a parameter.
996    // Note that ownership protocols require upgrade, otherwise a
997    // write miss on a shared owned block will generate a ReadExcl,
998    // which will clobber the owned copy.
999    const bool useUpgrades = true;
1000    if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
1001        assert(!blkValid || !blk->isWritable());
1002        // forward as invalidate to all other caches, this gives us
1003        // the line in Exclusive state, and invalidates all other
1004        // copies
1005        cmd = MemCmd::InvalidateReq;
1006    } else if (blkValid && useUpgrades) {
1007        // only reason to be here is that blk is read only and we need
1008        // it to be writable
1009        assert(needsWritable);
1010        assert(!blk->isWritable());
1011        cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
1012    } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
1013               cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
1014        // Even though this SC will fail, we still need to send out the
1015        // request and get the data to supply it to other snoopers in the case
1016        // where the determination the StoreCond fails is delayed due to
1017        // all caches not being on the same local bus.
1018        cmd = MemCmd::SCUpgradeFailReq;
1019    } else {
1020        // block is invalid
1021
1022        // If the request does not need a writable there are two cases
1023        // where we need to ensure the response will not fetch the
1024        // block in dirty state:
1025        // * this cache is read only and it does not perform
1026        //   writebacks,
1027        // * this cache is mostly exclusive and will not fill (since
1028        //   it does not fill it will have to writeback the dirty data
1029        //   immediately which generates uneccesary writebacks).
1030        bool force_clean_rsp = isReadOnly || clusivity == Enums::mostly_excl;
1031        cmd = needsWritable ? MemCmd::ReadExReq :
1032            (force_clean_rsp ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1033    }
1034    PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1035
1036    // if there are upstream caches that have already marked the
1037    // packet as having sharers (not passing writable), pass that info
1038    // downstream
1039    if (cpu_pkt->hasSharers() && !needsWritable) {
1040        // note that cpu_pkt may have spent a considerable time in the
1041        // MSHR queue and that the information could possibly be out
1042        // of date, however, there is no harm in conservatively
1043        // assuming the block has sharers
1044        pkt->setHasSharers();
1045        DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1046                __func__, cpu_pkt->print(), pkt->print());
1047    }
1048
1049    // the packet should be block aligned
1050    assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1051
1052    pkt->allocate();
1053    DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1054            cpu_pkt->print());
1055    return pkt;
1056}
1057
1058
1059Tick
1060Cache::recvAtomic(PacketPtr pkt)
1061{
1062    // We are in atomic mode so we pay just for lookupLatency here.
1063    Cycles lat = lookupLatency;
1064
1065    // Forward the request if the system is in cache bypass mode.
1066    if (system->bypassCaches())
1067        return ticksToCycles(memSidePort->sendAtomic(pkt));
1068
1069    promoteWholeLineWrites(pkt);
1070
1071    // follow the same flow as in recvTimingReq, and check if a cache
1072    // above us is responding
1073    if (pkt->cacheResponding() && !pkt->isClean()) {
1074        assert(!pkt->req->isCacheInvalidate());
1075        DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1076                pkt->print());
1077
1078        // if a cache is responding, and it had the line in Owned
1079        // rather than Modified state, we need to invalidate any
1080        // copies that are not on the same path to memory
1081        assert(pkt->needsWritable() && !pkt->responderHadWritable());
1082        lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1083
1084        return lat * clockPeriod();
1085    }
1086
1087    // should assert here that there are no outstanding MSHRs or
1088    // writebacks... that would mean that someone used an atomic
1089    // access in timing mode
1090
1091    CacheBlk *blk = nullptr;
1092    PacketList writebacks;
1093    bool satisfied = access(pkt, blk, lat, writebacks);
1094
1095    if (pkt->isClean() && blk && blk->isDirty()) {
1096        // A cache clean opearation is looking for a dirty
1097        // block. If a dirty block is encountered a WriteClean
1098        // will update any copies to the path to the memory
1099        // until the point of reference.
1100        DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1101                __func__, pkt->print(), blk->print());
1102        PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
1103        writebacks.push_back(wb_pkt);
1104        pkt->setSatisfied();
1105    }
1106
1107    // handle writebacks resulting from the access here to ensure they
1108    // logically proceed anything happening below
1109    doWritebacksAtomic(writebacks);
1110
1111    if (!satisfied) {
1112        // MISS
1113
1114        // deal with the packets that go through the write path of
1115        // the cache, i.e. any evictions and writes
1116        if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1117            (pkt->req->isUncacheable() && pkt->isWrite())) {
1118            lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1119            return lat * clockPeriod();
1120        }
1121        // only misses left
1122
1123        PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1124
1125        bool is_forward = (bus_pkt == nullptr);
1126
1127        if (is_forward) {
1128            // just forwarding the same request to the next level
1129            // no local cache operation involved
1130            bus_pkt = pkt;
1131        }
1132
1133        DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1134                bus_pkt->print());
1135
1136#if TRACING_ON
1137        CacheBlk::State old_state = blk ? blk->status : 0;
1138#endif
1139
1140        lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1141
1142        bool is_invalidate = bus_pkt->isInvalidate();
1143
1144        // We are now dealing with the response handling
1145        DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1146                bus_pkt->print(), old_state);
1147
1148        // If packet was a forward, the response (if any) is already
1149        // in place in the bus_pkt == pkt structure, so we don't need
1150        // to do anything.  Otherwise, use the separate bus_pkt to
1151        // generate response to pkt and then delete it.
1152        if (!is_forward) {
1153            if (pkt->needsResponse()) {
1154                assert(bus_pkt->isResponse());
1155                if (bus_pkt->isError()) {
1156                    pkt->makeAtomicResponse();
1157                    pkt->copyError(bus_pkt);
1158                } else if (pkt->cmd == MemCmd::WriteLineReq) {
1159                    // note the use of pkt, not bus_pkt here.
1160
1161                    // write-line request to the cache that promoted
1162                    // the write to a whole line
1163                    blk = handleFill(pkt, blk, writebacks,
1164                                     allocOnFill(pkt->cmd));
1165                    assert(blk != NULL);
1166                    is_invalidate = false;
1167                    satisfyRequest(pkt, blk);
1168                } else if (bus_pkt->isRead() ||
1169                           bus_pkt->cmd == MemCmd::UpgradeResp) {
1170                    // we're updating cache state to allow us to
1171                    // satisfy the upstream request from the cache
1172                    blk = handleFill(bus_pkt, blk, writebacks,
1173                                     allocOnFill(pkt->cmd));
1174                    satisfyRequest(pkt, blk);
1175                    maintainClusivity(pkt->fromCache(), blk);
1176                } else {
1177                    // we're satisfying the upstream request without
1178                    // modifying cache state, e.g., a write-through
1179                    pkt->makeAtomicResponse();
1180                }
1181            }
1182            delete bus_pkt;
1183        }
1184
1185        if (is_invalidate && blk && blk->isValid()) {
1186            invalidateBlock(blk);
1187        }
1188    }
1189
1190    // Note that we don't invoke the prefetcher at all in atomic mode.
1191    // It's not clear how to do it properly, particularly for
1192    // prefetchers that aggressively generate prefetch candidates and
1193    // rely on bandwidth contention to throttle them; these will tend
1194    // to pollute the cache in atomic mode since there is no bandwidth
1195    // contention.  If we ever do want to enable prefetching in atomic
1196    // mode, though, this is the place to do it... see timingAccess()
1197    // for an example (though we'd want to issue the prefetch(es)
1198    // immediately rather than calling requestMemSideBus() as we do
1199    // there).
1200
1201    // do any writebacks resulting from the response handling
1202    doWritebacksAtomic(writebacks);
1203
1204    // if we used temp block, check to see if its valid and if so
1205    // clear it out, but only do so after the call to recvAtomic is
1206    // finished so that any downstream observers (such as a snoop
1207    // filter), first see the fill, and only then see the eviction
1208    if (blk == tempBlock && tempBlock->isValid()) {
1209        // the atomic CPU calls recvAtomic for fetch and load/store
1210        // sequentuially, and we may already have a tempBlock
1211        // writeback from the fetch that we have not yet sent
1212        if (tempBlockWriteback) {
1213            // if that is the case, write the prevoius one back, and
1214            // do not schedule any new event
1215            writebackTempBlockAtomic();
1216        } else {
1217            // the writeback/clean eviction happens after the call to
1218            // recvAtomic has finished (but before any successive
1219            // calls), so that the response handling from the fill is
1220            // allowed to happen first
1221            schedule(writebackTempBlockAtomicEvent, curTick());
1222        }
1223
1224        tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1225            writebackBlk(blk) : cleanEvictBlk(blk);
1226        invalidateBlock(blk);
1227    }
1228
1229    if (pkt->needsResponse()) {
1230        pkt->makeAtomicResponse();
1231    }
1232
1233    return lat * clockPeriod();
1234}
1235
1236
1237void
1238Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1239{
1240    if (system->bypassCaches()) {
1241        // Packets from the memory side are snoop request and
1242        // shouldn't happen in bypass mode.
1243        assert(fromCpuSide);
1244
1245        // The cache should be flushed if we are in cache bypass mode,
1246        // so we don't need to check if we need to update anything.
1247        memSidePort->sendFunctional(pkt);
1248        return;
1249    }
1250
1251    Addr blk_addr = pkt->getBlockAddr(blkSize);
1252    bool is_secure = pkt->isSecure();
1253    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1254    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1255
1256    pkt->pushLabel(name());
1257
1258    CacheBlkPrintWrapper cbpw(blk);
1259
1260    // Note that just because an L2/L3 has valid data doesn't mean an
1261    // L1 doesn't have a more up-to-date modified copy that still
1262    // needs to be found.  As a result we always update the request if
1263    // we have it, but only declare it satisfied if we are the owner.
1264
1265    // see if we have data at all (owned or otherwise)
1266    bool have_data = blk && blk->isValid()
1267        && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1268                                blk->data);
1269
1270    // data we have is dirty if marked as such or if we have an
1271    // in-service MSHR that is pending a modified line
1272    bool have_dirty =
1273        have_data && (blk->isDirty() ||
1274                      (mshr && mshr->inService && mshr->isPendingModified()));
1275
1276    bool done = have_dirty
1277        || cpuSidePort->checkFunctional(pkt)
1278        || mshrQueue.checkFunctional(pkt, blk_addr)
1279        || writeBuffer.checkFunctional(pkt, blk_addr)
1280        || memSidePort->checkFunctional(pkt);
1281
1282    DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__,  pkt->print(),
1283            (blk && blk->isValid()) ? "valid " : "",
1284            have_data ? "data " : "", done ? "done " : "");
1285
1286    // We're leaving the cache, so pop cache->name() label
1287    pkt->popLabel();
1288
1289    if (done) {
1290        pkt->makeResponse();
1291    } else {
1292        // if it came as a request from the CPU side then make sure it
1293        // continues towards the memory side
1294        if (fromCpuSide) {
1295            memSidePort->sendFunctional(pkt);
1296        } else if (cpuSidePort->isSnooping()) {
1297            // if it came from the memory side, it must be a snoop request
1298            // and we should only forward it if we are forwarding snoops
1299            cpuSidePort->sendFunctionalSnoop(pkt);
1300        }
1301    }
1302}
1303
1304
1305/////////////////////////////////////////////////////
1306//
1307// Response handling: responses from the memory side
1308//
1309/////////////////////////////////////////////////////
1310
1311
1312void
1313Cache::handleUncacheableWriteResp(PacketPtr pkt)
1314{
1315    Tick completion_time = clockEdge(responseLatency) +
1316        pkt->headerDelay + pkt->payloadDelay;
1317
1318    // Reset the bus additional time as it is now accounted for
1319    pkt->headerDelay = pkt->payloadDelay = 0;
1320
1321    cpuSidePort->schedTimingResp(pkt, completion_time, true);
1322}
1323
1324void
1325Cache::recvTimingResp(PacketPtr pkt)
1326{
1327    assert(pkt->isResponse());
1328
1329    // all header delay should be paid for by the crossbar, unless
1330    // this is a prefetch response from above
1331    panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1332             "%s saw a non-zero packet delay\n", name());
1333
1334    bool is_error = pkt->isError();
1335
1336    if (is_error) {
1337        DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1338                pkt->print());
1339    }
1340
1341    DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1342            pkt->print());
1343
1344    // if this is a write, we should be looking at an uncacheable
1345    // write
1346    if (pkt->isWrite()) {
1347        assert(pkt->req->isUncacheable());
1348        handleUncacheableWriteResp(pkt);
1349        return;
1350    }
1351
1352    // we have dealt with any (uncacheable) writes above, from here on
1353    // we know we are dealing with an MSHR due to a miss or a prefetch
1354    MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1355    assert(mshr);
1356
1357    if (mshr == noTargetMSHR) {
1358        // we always clear at least one target
1359        clearBlocked(Blocked_NoTargets);
1360        noTargetMSHR = nullptr;
1361    }
1362
1363    // Initial target is used just for stats
1364    MSHR::Target *initial_tgt = mshr->getTarget();
1365    int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1366    Tick miss_latency = curTick() - initial_tgt->recvTime;
1367
1368    if (pkt->req->isUncacheable()) {
1369        assert(pkt->req->masterId() < system->maxMasters());
1370        mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1371            miss_latency;
1372    } else {
1373        assert(pkt->req->masterId() < system->maxMasters());
1374        mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1375            miss_latency;
1376    }
1377
1378    bool wasFull = mshrQueue.isFull();
1379
1380    PacketList writebacks;
1381
1382    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1383
1384    bool is_fill = !mshr->isForward &&
1385        (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1386
1387    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1388    const bool valid_blk = blk && blk->isValid();
1389    // If the response indicates that there are no sharers and we
1390    // either had the block already or the response is filling we can
1391    // promote our copy to writable
1392    if (!pkt->hasSharers() &&
1393        (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1394        mshr->promoteWritable();
1395    }
1396
1397    if (is_fill && !is_error) {
1398        DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1399                pkt->getAddr());
1400
1401        blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1402        assert(blk != nullptr);
1403    }
1404
1405    // allow invalidation responses originating from write-line
1406    // requests to be discarded
1407    bool is_invalidate = pkt->isInvalidate();
1408
1409    // The block was marked as not readable while there was a pending
1410    // cache maintenance operation, restore its flag.
1411    if (pkt->isClean() && !is_invalidate && valid_blk) {
1412        blk->status |= BlkReadable;
1413    }
1414
1415    // First offset for critical word first calculations
1416    int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1417
1418    bool from_cache = false;
1419    MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1420    for (auto &target: targets) {
1421        Packet *tgt_pkt = target.pkt;
1422        switch (target.source) {
1423          case MSHR::Target::FromCPU:
1424            Tick completion_time;
1425            // Here we charge on completion_time the delay of the xbar if the
1426            // packet comes from it, charged on headerDelay.
1427            completion_time = pkt->headerDelay;
1428
1429            // Software prefetch handling for cache closest to core
1430            if (tgt_pkt->cmd.isSWPrefetch()) {
1431                // a software prefetch would have already been ack'd
1432                // immediately with dummy data so the core would be able to
1433                // retire it. This request completes right here, so we
1434                // deallocate it.
1435                delete tgt_pkt->req;
1436                delete tgt_pkt;
1437                break; // skip response
1438            }
1439
1440            // keep track of whether we have responded to another
1441            // cache
1442            from_cache = from_cache || tgt_pkt->fromCache();
1443
1444            // unlike the other packet flows, where data is found in other
1445            // caches or memory and brought back, write-line requests always
1446            // have the data right away, so the above check for "is fill?"
1447            // cannot actually be determined until examining the stored MSHR
1448            // state. We "catch up" with that logic here, which is duplicated
1449            // from above.
1450            if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1451                assert(!is_error);
1452                // we got the block in a writable state, so promote
1453                // any deferred targets if possible
1454                mshr->promoteWritable();
1455                // NB: we use the original packet here and not the response!
1456                blk = handleFill(tgt_pkt, blk, writebacks,
1457                                 targets.allocOnFill);
1458                assert(blk != nullptr);
1459
1460                // treat as a fill, and discard the invalidation
1461                // response
1462                is_fill = true;
1463                is_invalidate = false;
1464            }
1465
1466            if (is_fill) {
1467                satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1468
1469                // How many bytes past the first request is this one
1470                int transfer_offset =
1471                    tgt_pkt->getOffset(blkSize) - initial_offset;
1472                if (transfer_offset < 0) {
1473                    transfer_offset += blkSize;
1474                }
1475
1476                // If not critical word (offset) return payloadDelay.
1477                // responseLatency is the latency of the return path
1478                // from lower level caches/memory to an upper level cache or
1479                // the core.
1480                completion_time += clockEdge(responseLatency) +
1481                    (transfer_offset ? pkt->payloadDelay : 0);
1482
1483                assert(!tgt_pkt->req->isUncacheable());
1484
1485                assert(tgt_pkt->req->masterId() < system->maxMasters());
1486                missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1487                    completion_time - target.recvTime;
1488            } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1489                // failed StoreCond upgrade
1490                assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1491                       tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1492                       tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1493                // responseLatency is the latency of the return path
1494                // from lower level caches/memory to an upper level cache or
1495                // the core.
1496                completion_time += clockEdge(responseLatency) +
1497                    pkt->payloadDelay;
1498                tgt_pkt->req->setExtraData(0);
1499            } else {
1500                // We are about to send a response to a cache above
1501                // that asked for an invalidation; we need to
1502                // invalidate our copy immediately as the most
1503                // up-to-date copy of the block will now be in the
1504                // cache above. It will also prevent this cache from
1505                // responding (if the block was previously dirty) to
1506                // snoops as they should snoop the caches above where
1507                // they will get the response from.
1508                if (is_invalidate && blk && blk->isValid()) {
1509                    invalidateBlock(blk);
1510                }
1511                // not a cache fill, just forwarding response
1512                // responseLatency is the latency of the return path
1513                // from lower level cahces/memory to the core.
1514                completion_time += clockEdge(responseLatency) +
1515                    pkt->payloadDelay;
1516                if (pkt->isRead() && !is_error) {
1517                    // sanity check
1518                    assert(pkt->getAddr() == tgt_pkt->getAddr());
1519                    assert(pkt->getSize() >= tgt_pkt->getSize());
1520
1521                    tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1522                }
1523            }
1524            tgt_pkt->makeTimingResponse();
1525            // if this packet is an error copy that to the new packet
1526            if (is_error)
1527                tgt_pkt->copyError(pkt);
1528            if (tgt_pkt->cmd == MemCmd::ReadResp &&
1529                (is_invalidate || mshr->hasPostInvalidate())) {
1530                // If intermediate cache got ReadRespWithInvalidate,
1531                // propagate that.  Response should not have
1532                // isInvalidate() set otherwise.
1533                tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1534                DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1535                        tgt_pkt->print());
1536            }
1537            // Reset the bus additional time as it is now accounted for
1538            tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1539            cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1540            break;
1541
1542          case MSHR::Target::FromPrefetcher:
1543            assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1544            if (blk)
1545                blk->status |= BlkHWPrefetched;
1546            delete tgt_pkt->req;
1547            delete tgt_pkt;
1548            break;
1549
1550          case MSHR::Target::FromSnoop:
1551            // I don't believe that a snoop can be in an error state
1552            assert(!is_error);
1553            // response to snoop request
1554            DPRINTF(Cache, "processing deferred snoop...\n");
1555            // If the response is invalidating, a snooping target can
1556            // be satisfied if it is also invalidating. If the reponse is, not
1557            // only invalidating, but more specifically an InvalidateResp and
1558            // the MSHR was created due to an InvalidateReq then a cache above
1559            // is waiting to satisfy a WriteLineReq. In this case even an
1560            // non-invalidating snoop is added as a target here since this is
1561            // the ordering point. When the InvalidateResp reaches this cache,
1562            // the snooping target will snoop further the cache above with the
1563            // WriteLineReq.
1564            assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
1565                   pkt->req->isCacheMaintenance() ||
1566                   mshr->hasPostInvalidate());
1567            handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1568            break;
1569
1570          default:
1571            panic("Illegal target->source enum %d\n", target.source);
1572        }
1573    }
1574
1575    maintainClusivity(from_cache, blk);
1576
1577    if (blk && blk->isValid()) {
1578        // an invalidate response stemming from a write line request
1579        // should not invalidate the block, so check if the
1580        // invalidation should be discarded
1581        if (is_invalidate || mshr->hasPostInvalidate()) {
1582            invalidateBlock(blk);
1583        } else if (mshr->hasPostDowngrade()) {
1584            blk->status &= ~BlkWritable;
1585        }
1586    }
1587
1588    if (mshr->promoteDeferredTargets()) {
1589        // avoid later read getting stale data while write miss is
1590        // outstanding.. see comment in timingAccess()
1591        if (blk) {
1592            blk->status &= ~BlkReadable;
1593        }
1594        mshrQueue.markPending(mshr);
1595        schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1596    } else {
1597        mshrQueue.deallocate(mshr);
1598        if (wasFull && !mshrQueue.isFull()) {
1599            clearBlocked(Blocked_NoMSHRs);
1600        }
1601
1602        // Request the bus for a prefetch if this deallocation freed enough
1603        // MSHRs for a prefetch to take place
1604        if (prefetcher && mshrQueue.canPrefetch()) {
1605            Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1606                                         clockEdge());
1607            if (next_pf_time != MaxTick)
1608                schedMemSideSendEvent(next_pf_time);
1609        }
1610    }
1611    // reset the xbar additional timinig  as it is now accounted for
1612    pkt->headerDelay = pkt->payloadDelay = 0;
1613
1614    // if we used temp block, check to see if its valid and then clear it out
1615    if (blk == tempBlock && tempBlock->isValid()) {
1616        PacketPtr wb_pkt = tempBlock->isDirty() || writebackClean ?
1617            writebackBlk(blk) : cleanEvictBlk(blk);
1618        writebacks.push_back(wb_pkt);
1619        invalidateBlock(tempBlock);
1620    }
1621
1622    // copy writebacks to write buffer
1623    doWritebacks(writebacks, forward_time);
1624
1625    DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1626    delete pkt;
1627}
1628
1629PacketPtr
1630Cache::writebackBlk(CacheBlk *blk)
1631{
1632    chatty_assert(!isReadOnly || writebackClean,
1633                  "Writeback from read-only cache");
1634    assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1635
1636    writebacks[Request::wbMasterId]++;
1637
1638    Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1639                               Request::wbMasterId);
1640    if (blk->isSecure())
1641        req->setFlags(Request::SECURE);
1642
1643    req->taskId(blk->task_id);
1644
1645    PacketPtr pkt =
1646        new Packet(req, blk->isDirty() ?
1647                   MemCmd::WritebackDirty : MemCmd::WritebackClean);
1648
1649    DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1650            pkt->print(), blk->isWritable(), blk->isDirty());
1651
1652    if (blk->isWritable()) {
1653        // not asserting shared means we pass the block in modified
1654        // state, mark our own block non-writeable
1655        blk->status &= ~BlkWritable;
1656    } else {
1657        // we are in the Owned state, tell the receiver
1658        pkt->setHasSharers();
1659    }
1660
1661    // make sure the block is not marked dirty
1662    blk->status &= ~BlkDirty;
1663
1664    pkt->allocate();
1665    pkt->setDataFromBlock(blk->data, blkSize);
1666
1667    return pkt;
1668}
1669
1670PacketPtr
1671Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1672{
1673    Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1674                               Request::wbMasterId);
1675    if (blk->isSecure()) {
1676        req->setFlags(Request::SECURE);
1677    }
1678    req->taskId(blk->task_id);
1679
1680    PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1681
1682    if (dest) {
1683        req->setFlags(dest);
1684        pkt->setWriteThrough();
1685    }
1686
1687    DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1688            blk->isWritable(), blk->isDirty());
1689
1690    if (blk->isWritable()) {
1691        // not asserting shared means we pass the block in modified
1692        // state, mark our own block non-writeable
1693        blk->status &= ~BlkWritable;
1694    } else {
1695        // we are in the Owned state, tell the receiver
1696        pkt->setHasSharers();
1697    }
1698
1699    // make sure the block is not marked dirty
1700    blk->status &= ~BlkDirty;
1701
1702    pkt->allocate();
1703    pkt->setDataFromBlock(blk->data, blkSize);
1704
1705    return pkt;
1706}
1707
1708
1709PacketPtr
1710Cache::cleanEvictBlk(CacheBlk *blk)
1711{
1712    assert(!writebackClean);
1713    assert(blk && blk->isValid() && !blk->isDirty());
1714    // Creating a zero sized write, a message to the snoop filter
1715    Request *req =
1716        new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1717                    Request::wbMasterId);
1718    if (blk->isSecure())
1719        req->setFlags(Request::SECURE);
1720
1721    req->taskId(blk->task_id);
1722
1723    PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1724    pkt->allocate();
1725    DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1726
1727    return pkt;
1728}
1729
1730void
1731Cache::memWriteback()
1732{
1733    CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1734    tags->forEachBlk(visitor);
1735}
1736
1737void
1738Cache::memInvalidate()
1739{
1740    CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1741    tags->forEachBlk(visitor);
1742}
1743
1744bool
1745Cache::isDirty() const
1746{
1747    CacheBlkIsDirtyVisitor visitor;
1748    tags->forEachBlk(visitor);
1749
1750    return visitor.isDirty();
1751}
1752
1753bool
1754Cache::writebackVisitor(CacheBlk &blk)
1755{
1756    if (blk.isDirty()) {
1757        assert(blk.isValid());
1758
1759        Request request(tags->regenerateBlkAddr(&blk), blkSize, 0,
1760                        Request::funcMasterId);
1761        request.taskId(blk.task_id);
1762        if (blk.isSecure()) {
1763            request.setFlags(Request::SECURE);
1764        }
1765
1766        Packet packet(&request, MemCmd::WriteReq);
1767        packet.dataStatic(blk.data);
1768
1769        memSidePort->sendFunctional(&packet);
1770
1771        blk.status &= ~BlkDirty;
1772    }
1773
1774    return true;
1775}
1776
1777bool
1778Cache::invalidateVisitor(CacheBlk &blk)
1779{
1780
1781    if (blk.isDirty())
1782        warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1783
1784    if (blk.isValid()) {
1785        assert(!blk.isDirty());
1786        invalidateBlock(&blk);
1787    }
1788
1789    return true;
1790}
1791
1792CacheBlk*
1793Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1794{
1795    // Find replacement victim
1796    CacheBlk *blk = tags->findVictim(addr);
1797
1798    // It is valid to return nullptr if there is no victim
1799    if (!blk)
1800        return nullptr;
1801
1802    if (blk->isValid()) {
1803        Addr repl_addr = tags->regenerateBlkAddr(blk);
1804        MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1805        if (repl_mshr) {
1806            // must be an outstanding upgrade or clean request
1807            // on a block we're about to replace...
1808            assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
1809                   repl_mshr->isCleaning());
1810            // too hard to replace block with transient state
1811            // allocation failed, block not inserted
1812            return nullptr;
1813        } else {
1814            DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1815                    "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1816                    addr, is_secure ? "s" : "ns",
1817                    blk->isDirty() ? "writeback" : "clean");
1818
1819            if (blk->wasPrefetched()) {
1820                unusedPrefetches++;
1821            }
1822            // Will send up Writeback/CleanEvict snoops via isCachedAbove
1823            // when pushing this writeback list into the write buffer.
1824            if (blk->isDirty() || writebackClean) {
1825                // Save writeback packet for handling by caller
1826                writebacks.push_back(writebackBlk(blk));
1827            } else {
1828                writebacks.push_back(cleanEvictBlk(blk));
1829            }
1830            replacements++;
1831        }
1832    }
1833
1834    return blk;
1835}
1836
1837void
1838Cache::invalidateBlock(CacheBlk *blk)
1839{
1840    if (blk != tempBlock)
1841        tags->invalidate(blk);
1842    blk->invalidate();
1843}
1844
1845// Note that the reason we return a list of writebacks rather than
1846// inserting them directly in the write buffer is that this function
1847// is called by both atomic and timing-mode accesses, and in atomic
1848// mode we don't mess with the write buffer (we just perform the
1849// writebacks atomically once the original request is complete).
1850CacheBlk*
1851Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1852                  bool allocate)
1853{
1854    assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1855    Addr addr = pkt->getAddr();
1856    bool is_secure = pkt->isSecure();
1857#if TRACING_ON
1858    CacheBlk::State old_state = blk ? blk->status : 0;
1859#endif
1860
1861    // When handling a fill, we should have no writes to this line.
1862    assert(addr == pkt->getBlockAddr(blkSize));
1863    assert(!writeBuffer.findMatch(addr, is_secure));
1864
1865    if (blk == nullptr) {
1866        // better have read new data...
1867        assert(pkt->hasData());
1868
1869        // only read responses and write-line requests have data;
1870        // note that we don't write the data here for write-line - that
1871        // happens in the subsequent call to satisfyRequest
1872        assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1873
1874        // need to do a replacement if allocating, otherwise we stick
1875        // with the temporary storage
1876        blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1877
1878        if (blk == nullptr) {
1879            // No replaceable block or a mostly exclusive
1880            // cache... just use temporary storage to complete the
1881            // current request and then get rid of it
1882            assert(!tempBlock->isValid());
1883            blk = tempBlock;
1884            tempBlock->set = tags->extractSet(addr);
1885            tempBlock->tag = tags->extractTag(addr);
1886            if (is_secure) {
1887                tempBlock->status |= BlkSecure;
1888            }
1889            DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1890                    is_secure ? "s" : "ns");
1891        } else {
1892            tags->insertBlock(pkt, blk);
1893        }
1894
1895        // we should never be overwriting a valid block
1896        assert(!blk->isValid());
1897    } else {
1898        // existing block... probably an upgrade
1899        assert(blk->tag == tags->extractTag(addr));
1900        // either we're getting new data or the block should already be valid
1901        assert(pkt->hasData() || blk->isValid());
1902        // don't clear block status... if block is already dirty we
1903        // don't want to lose that
1904    }
1905
1906    if (is_secure)
1907        blk->status |= BlkSecure;
1908    blk->status |= BlkValid | BlkReadable;
1909
1910    // sanity check for whole-line writes, which should always be
1911    // marked as writable as part of the fill, and then later marked
1912    // dirty as part of satisfyRequest
1913    if (pkt->cmd == MemCmd::WriteLineReq) {
1914        assert(!pkt->hasSharers());
1915    }
1916
1917    // here we deal with setting the appropriate state of the line,
1918    // and we start by looking at the hasSharers flag, and ignore the
1919    // cacheResponding flag (normally signalling dirty data) if the
1920    // packet has sharers, thus the line is never allocated as Owned
1921    // (dirty but not writable), and always ends up being either
1922    // Shared, Exclusive or Modified, see Packet::setCacheResponding
1923    // for more details
1924    if (!pkt->hasSharers()) {
1925        // we could get a writable line from memory (rather than a
1926        // cache) even in a read-only cache, note that we set this bit
1927        // even for a read-only cache, possibly revisit this decision
1928        blk->status |= BlkWritable;
1929
1930        // check if we got this via cache-to-cache transfer (i.e., from a
1931        // cache that had the block in Modified or Owned state)
1932        if (pkt->cacheResponding()) {
1933            // we got the block in Modified state, and invalidated the
1934            // owners copy
1935            blk->status |= BlkDirty;
1936
1937            chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1938                          "in read-only cache %s\n", name());
1939        }
1940    }
1941
1942    DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1943            addr, is_secure ? "s" : "ns", old_state, blk->print());
1944
1945    // if we got new data, copy it in (checking for a read response
1946    // and a response that has data is the same in the end)
1947    if (pkt->isRead()) {
1948        // sanity checks
1949        assert(pkt->hasData());
1950        assert(pkt->getSize() == blkSize);
1951
1952        pkt->writeDataToBlock(blk->data, blkSize);
1953    }
1954    // We pay for fillLatency here.
1955    blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1956        pkt->payloadDelay;
1957
1958    return blk;
1959}
1960
1961
1962/////////////////////////////////////////////////////
1963//
1964// Snoop path: requests coming in from the memory side
1965//
1966/////////////////////////////////////////////////////
1967
1968void
1969Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1970                              bool already_copied, bool pending_inval)
1971{
1972    // sanity check
1973    assert(req_pkt->isRequest());
1974    assert(req_pkt->needsResponse());
1975
1976    DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1977    // timing-mode snoop responses require a new packet, unless we
1978    // already made a copy...
1979    PacketPtr pkt = req_pkt;
1980    if (!already_copied)
1981        // do not clear flags, and allocate space for data if the
1982        // packet needs it (the only packets that carry data are read
1983        // responses)
1984        pkt = new Packet(req_pkt, false, req_pkt->isRead());
1985
1986    assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
1987           pkt->hasSharers());
1988    pkt->makeTimingResponse();
1989    if (pkt->isRead()) {
1990        pkt->setDataFromBlock(blk_data, blkSize);
1991    }
1992    if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
1993        // Assume we defer a response to a read from a far-away cache
1994        // A, then later defer a ReadExcl from a cache B on the same
1995        // bus as us. We'll assert cacheResponding in both cases, but
1996        // in the latter case cacheResponding will keep the
1997        // invalidation from reaching cache A. This special response
1998        // tells cache A that it gets the block to satisfy its read,
1999        // but must immediately invalidate it.
2000        pkt->cmd = MemCmd::ReadRespWithInvalidate;
2001    }
2002    // Here we consider forward_time, paying for just forward latency and
2003    // also charging the delay provided by the xbar.
2004    // forward_time is used as send_time in next allocateWriteBuffer().
2005    Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
2006    // Here we reset the timing of the packet.
2007    pkt->headerDelay = pkt->payloadDelay = 0;
2008    DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
2009            pkt->print(), forward_time);
2010    memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
2011}
2012
2013uint32_t
2014Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
2015                   bool is_deferred, bool pending_inval)
2016{
2017    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2018    // deferred snoops can only happen in timing mode
2019    assert(!(is_deferred && !is_timing));
2020    // pending_inval only makes sense on deferred snoops
2021    assert(!(pending_inval && !is_deferred));
2022    assert(pkt->isRequest());
2023
2024    // the packet may get modified if we or a forwarded snooper
2025    // responds in atomic mode, so remember a few things about the
2026    // original packet up front
2027    bool invalidate = pkt->isInvalidate();
2028    bool M5_VAR_USED needs_writable = pkt->needsWritable();
2029
2030    // at the moment we could get an uncacheable write which does not
2031    // have the invalidate flag, and we need a suitable way of dealing
2032    // with this case
2033    panic_if(invalidate && pkt->req->isUncacheable(),
2034             "%s got an invalidating uncacheable snoop request %s",
2035             name(), pkt->print());
2036
2037    uint32_t snoop_delay = 0;
2038
2039    if (forwardSnoops) {
2040        // first propagate snoop upward to see if anyone above us wants to
2041        // handle it.  save & restore packet src since it will get
2042        // rewritten to be relative to cpu-side bus (if any)
2043        bool alreadyResponded = pkt->cacheResponding();
2044        if (is_timing) {
2045            // copy the packet so that we can clear any flags before
2046            // forwarding it upwards, we also allocate data (passing
2047            // the pointer along in case of static data), in case
2048            // there is a snoop hit in upper levels
2049            Packet snoopPkt(pkt, true, true);
2050            snoopPkt.setExpressSnoop();
2051            // the snoop packet does not need to wait any additional
2052            // time
2053            snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2054            cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2055
2056            // add the header delay (including crossbar and snoop
2057            // delays) of the upward snoop to the snoop delay for this
2058            // cache
2059            snoop_delay += snoopPkt.headerDelay;
2060
2061            if (snoopPkt.cacheResponding()) {
2062                // cache-to-cache response from some upper cache
2063                assert(!alreadyResponded);
2064                pkt->setCacheResponding();
2065            }
2066            // upstream cache has the block, or has an outstanding
2067            // MSHR, pass the flag on
2068            if (snoopPkt.hasSharers()) {
2069                pkt->setHasSharers();
2070            }
2071            // If this request is a prefetch or clean evict and an upper level
2072            // signals block present, make sure to propagate the block
2073            // presence to the requester.
2074            if (snoopPkt.isBlockCached()) {
2075                pkt->setBlockCached();
2076            }
2077            // If the request was satisfied by snooping the cache
2078            // above, mark the original packet as satisfied too.
2079            if (snoopPkt.satisfied()) {
2080                pkt->setSatisfied();
2081            }
2082        } else {
2083            cpuSidePort->sendAtomicSnoop(pkt);
2084            if (!alreadyResponded && pkt->cacheResponding()) {
2085                // cache-to-cache response from some upper cache:
2086                // forward response to original requester
2087                assert(pkt->isResponse());
2088            }
2089        }
2090    }
2091
2092    bool respond = false;
2093    bool blk_valid = blk && blk->isValid();
2094    if (pkt->isClean()) {
2095        if (blk_valid && blk->isDirty()) {
2096            DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
2097                    __func__, pkt->print(), blk->print());
2098            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
2099            PacketList writebacks;
2100            writebacks.push_back(wb_pkt);
2101
2102            if (is_timing) {
2103                // anything that is merely forwarded pays for the forward
2104                // latency and the delay provided by the crossbar
2105                Tick forward_time = clockEdge(forwardLatency) +
2106                    pkt->headerDelay;
2107                doWritebacks(writebacks, forward_time);
2108            } else {
2109                doWritebacksAtomic(writebacks);
2110            }
2111            pkt->setSatisfied();
2112        }
2113    } else if (!blk_valid) {
2114        DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2115                pkt->print());
2116        if (is_deferred) {
2117            // we no longer have the block, and will not respond, but a
2118            // packet was allocated in MSHR::handleSnoop and we have
2119            // to delete it
2120            assert(pkt->needsResponse());
2121
2122            // we have passed the block to a cache upstream, that
2123            // cache should be responding
2124            assert(pkt->cacheResponding());
2125
2126            delete pkt;
2127        }
2128        return snoop_delay;
2129    } else {
2130        DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2131                pkt->print(), blk->print());
2132
2133        // We may end up modifying both the block state and the packet (if
2134        // we respond in atomic mode), so just figure out what to do now
2135        // and then do it later. We respond to all snoops that need
2136        // responses provided we have the block in dirty state. The
2137        // invalidation itself is taken care of below. We don't respond to
2138        // cache maintenance operations as this is done by the destination
2139        // xbar.
2140        respond = blk->isDirty() && pkt->needsResponse();
2141
2142        chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
2143                      "a dirty block in a read-only cache %s\n", name());
2144    }
2145
2146    // Invalidate any prefetch's from below that would strip write permissions
2147    // MemCmd::HardPFReq is only observed by upstream caches.  After missing
2148    // above and in it's own cache, a new MemCmd::ReadReq is created that
2149    // downstream caches observe.
2150    if (pkt->mustCheckAbove()) {
2151        DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2152                "from lower cache\n", pkt->getAddr(), pkt->print());
2153        pkt->setBlockCached();
2154        return snoop_delay;
2155    }
2156
2157    if (pkt->isRead() && !invalidate) {
2158        // reading without requiring the line in a writable state
2159        assert(!needs_writable);
2160        pkt->setHasSharers();
2161
2162        // if the requesting packet is uncacheable, retain the line in
2163        // the current state, otherwhise unset the writable flag,
2164        // which means we go from Modified to Owned (and will respond
2165        // below), remain in Owned (and will respond below), from
2166        // Exclusive to Shared, or remain in Shared
2167        if (!pkt->req->isUncacheable())
2168            blk->status &= ~BlkWritable;
2169        DPRINTF(Cache, "new state is %s\n", blk->print());
2170    }
2171
2172    if (respond) {
2173        // prevent anyone else from responding, cache as well as
2174        // memory, and also prevent any memory from even seeing the
2175        // request
2176        pkt->setCacheResponding();
2177        if (!pkt->isClean() && blk->isWritable()) {
2178            // inform the cache hierarchy that this cache had the line
2179            // in the Modified state so that we avoid unnecessary
2180            // invalidations (see Packet::setResponderHadWritable)
2181            pkt->setResponderHadWritable();
2182
2183            // in the case of an uncacheable request there is no point
2184            // in setting the responderHadWritable flag, but since the
2185            // recipient does not care there is no harm in doing so
2186        } else {
2187            // if the packet has needsWritable set we invalidate our
2188            // copy below and all other copies will be invalidates
2189            // through express snoops, and if needsWritable is not set
2190            // we already called setHasSharers above
2191        }
2192
2193        // if we are returning a writable and dirty (Modified) line,
2194        // we should be invalidating the line
2195        panic_if(!invalidate && !pkt->hasSharers(),
2196                 "%s is passing a Modified line through %s, "
2197                 "but keeping the block", name(), pkt->print());
2198
2199        if (is_timing) {
2200            doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2201        } else {
2202            pkt->makeAtomicResponse();
2203            // packets such as upgrades do not actually have any data
2204            // payload
2205            if (pkt->hasData())
2206                pkt->setDataFromBlock(blk->data, blkSize);
2207        }
2208    }
2209
2210    if (!respond && is_deferred) {
2211        assert(pkt->needsResponse());
2212
2213        // if we copied the deferred packet with the intention to
2214        // respond, but are not responding, then a cache above us must
2215        // be, and we can use this as the indication of whether this
2216        // is a packet where we created a copy of the request or not
2217        if (!pkt->cacheResponding()) {
2218            delete pkt->req;
2219        }
2220
2221        delete pkt;
2222    }
2223
2224    // Do this last in case it deallocates block data or something
2225    // like that
2226    if (blk_valid && invalidate) {
2227        invalidateBlock(blk);
2228        DPRINTF(Cache, "new state is %s\n", blk->print());
2229    }
2230
2231    return snoop_delay;
2232}
2233
2234
2235void
2236Cache::recvTimingSnoopReq(PacketPtr pkt)
2237{
2238    DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2239
2240    // Snoops shouldn't happen when bypassing caches
2241    assert(!system->bypassCaches());
2242
2243    // no need to snoop requests that are not in range
2244    if (!inRange(pkt->getAddr())) {
2245        return;
2246    }
2247
2248    bool is_secure = pkt->isSecure();
2249    CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2250
2251    Addr blk_addr = pkt->getBlockAddr(blkSize);
2252    MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2253
2254    // Update the latency cost of the snoop so that the crossbar can
2255    // account for it. Do not overwrite what other neighbouring caches
2256    // have already done, rather take the maximum. The update is
2257    // tentative, for cases where we return before an upward snoop
2258    // happens below.
2259    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2260                                         lookupLatency * clockPeriod());
2261
2262    // Inform request(Prefetch, CleanEvict or Writeback) from below of
2263    // MSHR hit, set setBlockCached.
2264    if (mshr && pkt->mustCheckAbove()) {
2265        DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2266                "mshr hit\n", pkt->print());
2267        pkt->setBlockCached();
2268        return;
2269    }
2270
2271    // Bypass any existing cache maintenance requests if the request
2272    // has been satisfied already (i.e., the dirty block has been
2273    // found).
2274    if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
2275        return;
2276    }
2277
2278    // Let the MSHR itself track the snoop and decide whether we want
2279    // to go ahead and do the regular cache snoop
2280    if (mshr && mshr->handleSnoop(pkt, order++)) {
2281        DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2282                "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2283                mshr->print());
2284
2285        if (mshr->getNumTargets() > numTarget)
2286            warn("allocating bonus target for snoop"); //handle later
2287        return;
2288    }
2289
2290    //We also need to check the writeback buffers and handle those
2291    WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2292    if (wb_entry) {
2293        DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2294                pkt->getAddr(), is_secure ? "s" : "ns");
2295        // Expect to see only Writebacks and/or CleanEvicts here, both of
2296        // which should not be generated for uncacheable data.
2297        assert(!wb_entry->isUncacheable());
2298        // There should only be a single request responsible for generating
2299        // Writebacks/CleanEvicts.
2300        assert(wb_entry->getNumTargets() == 1);
2301        PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2302        assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2303
2304        if (pkt->isEviction()) {
2305            // if the block is found in the write queue, set the BLOCK_CACHED
2306            // flag for Writeback/CleanEvict snoop. On return the snoop will
2307            // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2308            // any CleanEvicts from travelling down the memory hierarchy.
2309            pkt->setBlockCached();
2310            DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2311                    "hit\n", __func__, pkt->print());
2312            return;
2313        }
2314
2315        // conceptually writebacks are no different to other blocks in
2316        // this cache, so the behaviour is modelled after handleSnoop,
2317        // the difference being that instead of querying the block
2318        // state to determine if it is dirty and writable, we use the
2319        // command and fields of the writeback packet
2320        bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2321            pkt->needsResponse();
2322        bool have_writable = !wb_pkt->hasSharers();
2323        bool invalidate = pkt->isInvalidate();
2324
2325        if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2326            assert(!pkt->needsWritable());
2327            pkt->setHasSharers();
2328            wb_pkt->setHasSharers();
2329        }
2330
2331        if (respond) {
2332            pkt->setCacheResponding();
2333
2334            if (have_writable) {
2335                pkt->setResponderHadWritable();
2336            }
2337
2338            doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2339                                   false, false);
2340        }
2341
2342        if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2343            // Invalidation trumps our writeback... discard here
2344            // Note: markInService will remove entry from writeback buffer.
2345            markInService(wb_entry);
2346            delete wb_pkt;
2347        }
2348    }
2349
2350    // If this was a shared writeback, there may still be
2351    // other shared copies above that require invalidation.
2352    // We could be more selective and return here if the
2353    // request is non-exclusive or if the writeback is
2354    // exclusive.
2355    uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2356
2357    // Override what we did when we first saw the snoop, as we now
2358    // also have the cost of the upwards snoops to account for
2359    pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2360                                         lookupLatency * clockPeriod());
2361}
2362
2363bool
2364Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2365{
2366    // Express snoop responses from master to slave, e.g., from L1 to L2
2367    cache->recvTimingSnoopResp(pkt);
2368    return true;
2369}
2370
2371Tick
2372Cache::recvAtomicSnoop(PacketPtr pkt)
2373{
2374    // Snoops shouldn't happen when bypassing caches
2375    assert(!system->bypassCaches());
2376
2377    // no need to snoop requests that are not in range.
2378    if (!inRange(pkt->getAddr())) {
2379        return 0;
2380    }
2381
2382    CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2383    uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2384    return snoop_delay + lookupLatency * clockPeriod();
2385}
2386
2387
2388QueueEntry*
2389Cache::getNextQueueEntry()
2390{
2391    // Check both MSHR queue and write buffer for potential requests,
2392    // note that null does not mean there is no request, it could
2393    // simply be that it is not ready
2394    MSHR *miss_mshr  = mshrQueue.getNext();
2395    WriteQueueEntry *wq_entry = writeBuffer.getNext();
2396
2397    // If we got a write buffer request ready, first priority is a
2398    // full write buffer, otherwise we favour the miss requests
2399    if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2400        // need to search MSHR queue for conflicting earlier miss.
2401        MSHR *conflict_mshr =
2402            mshrQueue.findPending(wq_entry->blkAddr,
2403                                  wq_entry->isSecure);
2404
2405        if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2406            // Service misses in order until conflict is cleared.
2407            return conflict_mshr;
2408
2409            // @todo Note that we ignore the ready time of the conflict here
2410        }
2411
2412        // No conflicts; issue write
2413        return wq_entry;
2414    } else if (miss_mshr) {
2415        // need to check for conflicting earlier writeback
2416        WriteQueueEntry *conflict_mshr =
2417            writeBuffer.findPending(miss_mshr->blkAddr,
2418                                    miss_mshr->isSecure);
2419        if (conflict_mshr) {
2420            // not sure why we don't check order here... it was in the
2421            // original code but commented out.
2422
2423            // The only way this happens is if we are
2424            // doing a write and we didn't have permissions
2425            // then subsequently saw a writeback (owned got evicted)
2426            // We need to make sure to perform the writeback first
2427            // To preserve the dirty data, then we can issue the write
2428
2429            // should we return wq_entry here instead?  I.e. do we
2430            // have to flush writes in order?  I don't think so... not
2431            // for Alpha anyway.  Maybe for x86?
2432            return conflict_mshr;
2433
2434            // @todo Note that we ignore the ready time of the conflict here
2435        }
2436
2437        // No conflicts; issue read
2438        return miss_mshr;
2439    }
2440
2441    // fall through... no pending requests.  Try a prefetch.
2442    assert(!miss_mshr && !wq_entry);
2443    if (prefetcher && mshrQueue.canPrefetch()) {
2444        // If we have a miss queue slot, we can try a prefetch
2445        PacketPtr pkt = prefetcher->getPacket();
2446        if (pkt) {
2447            Addr pf_addr = pkt->getBlockAddr(blkSize);
2448            if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2449                !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2450                !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2451                // Update statistic on number of prefetches issued
2452                // (hwpf_mshr_misses)
2453                assert(pkt->req->masterId() < system->maxMasters());
2454                mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2455
2456                // allocate an MSHR and return it, note
2457                // that we send the packet straight away, so do not
2458                // schedule the send
2459                return allocateMissBuffer(pkt, curTick(), false);
2460            } else {
2461                // free the request and packet
2462                delete pkt->req;
2463                delete pkt;
2464            }
2465        }
2466    }
2467
2468    return nullptr;
2469}
2470
2471bool
2472Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2473{
2474    if (!forwardSnoops)
2475        return false;
2476    // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2477    // Writeback snoops into upper level caches to check for copies of the
2478    // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2479    // packet, the cache can inform the crossbar below of presence or absence
2480    // of the block.
2481    if (is_timing) {
2482        Packet snoop_pkt(pkt, true, false);
2483        snoop_pkt.setExpressSnoop();
2484        // Assert that packet is either Writeback or CleanEvict and not a
2485        // prefetch request because prefetch requests need an MSHR and may
2486        // generate a snoop response.
2487        assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2488        snoop_pkt.senderState = nullptr;
2489        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2490        // Writeback/CleanEvict snoops do not generate a snoop response.
2491        assert(!(snoop_pkt.cacheResponding()));
2492        return snoop_pkt.isBlockCached();
2493    } else {
2494        cpuSidePort->sendAtomicSnoop(pkt);
2495        return pkt->isBlockCached();
2496    }
2497}
2498
2499Tick
2500Cache::nextQueueReadyTime() const
2501{
2502    Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2503                              writeBuffer.nextReadyTime());
2504
2505    // Don't signal prefetch ready time if no MSHRs available
2506    // Will signal once enoguh MSHRs are deallocated
2507    if (prefetcher && mshrQueue.canPrefetch()) {
2508        nextReady = std::min(nextReady,
2509                             prefetcher->nextPrefetchReadyTime());
2510    }
2511
2512    return nextReady;
2513}
2514
2515bool
2516Cache::sendMSHRQueuePacket(MSHR* mshr)
2517{
2518    assert(mshr);
2519
2520    // use request from 1st target
2521    PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2522
2523    DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2524
2525    CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2526
2527    if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2528        // we should never have hardware prefetches to allocated
2529        // blocks
2530        assert(blk == nullptr);
2531
2532        // We need to check the caches above us to verify that
2533        // they don't have a copy of this block in the dirty state
2534        // at the moment. Without this check we could get a stale
2535        // copy from memory that might get used in place of the
2536        // dirty one.
2537        Packet snoop_pkt(tgt_pkt, true, false);
2538        snoop_pkt.setExpressSnoop();
2539        // We are sending this packet upwards, but if it hits we will
2540        // get a snoop response that we end up treating just like a
2541        // normal response, hence it needs the MSHR as its sender
2542        // state
2543        snoop_pkt.senderState = mshr;
2544        cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2545
2546        // Check to see if the prefetch was squashed by an upper cache (to
2547        // prevent us from grabbing the line) or if a Check to see if a
2548        // writeback arrived between the time the prefetch was placed in
2549        // the MSHRs and when it was selected to be sent or if the
2550        // prefetch was squashed by an upper cache.
2551
2552        // It is important to check cacheResponding before
2553        // prefetchSquashed. If another cache has committed to
2554        // responding, it will be sending a dirty response which will
2555        // arrive at the MSHR allocated for this request. Checking the
2556        // prefetchSquash first may result in the MSHR being
2557        // prematurely deallocated.
2558        if (snoop_pkt.cacheResponding()) {
2559            auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2560            assert(r.second);
2561
2562            // if we are getting a snoop response with no sharers it
2563            // will be allocated as Modified
2564            bool pending_modified_resp = !snoop_pkt.hasSharers();
2565            markInService(mshr, pending_modified_resp);
2566
2567            DPRINTF(Cache, "Upward snoop of prefetch for addr"
2568                    " %#x (%s) hit\n",
2569                    tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2570            return false;
2571        }
2572
2573        if (snoop_pkt.isBlockCached()) {
2574            DPRINTF(Cache, "Block present, prefetch squashed by cache.  "
2575                    "Deallocating mshr target %#x.\n",
2576                    mshr->blkAddr);
2577
2578            // Deallocate the mshr target
2579            if (mshrQueue.forceDeallocateTarget(mshr)) {
2580                // Clear block if this deallocation resulted freed an
2581                // mshr when all had previously been utilized
2582                clearBlocked(Blocked_NoMSHRs);
2583            }
2584
2585            // given that no response is expected, delete Request and Packet
2586            delete tgt_pkt->req;
2587            delete tgt_pkt;
2588
2589            return false;
2590        }
2591    }
2592
2593    // either a prefetch that is not present upstream, or a normal
2594    // MSHR request, proceed to get the packet to send downstream
2595    PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2596
2597    mshr->isForward = (pkt == nullptr);
2598
2599    if (mshr->isForward) {
2600        // not a cache block request, but a response is expected
2601        // make copy of current packet to forward, keep current
2602        // copy for response handling
2603        pkt = new Packet(tgt_pkt, false, true);
2604        assert(!pkt->isWrite());
2605    }
2606
2607    // play it safe and append (rather than set) the sender state,
2608    // as forwarded packets may already have existing state
2609    pkt->pushSenderState(mshr);
2610
2611    if (pkt->isClean() && blk && blk->isDirty()) {
2612        // A cache clean opearation is looking for a dirty block. Mark
2613        // the packet so that the destination xbar can determine that
2614        // there will be a follow-up write packet as well.
2615        pkt->setSatisfied();
2616    }
2617
2618    if (!memSidePort->sendTimingReq(pkt)) {
2619        // we are awaiting a retry, but we
2620        // delete the packet and will be creating a new packet
2621        // when we get the opportunity
2622        delete pkt;
2623
2624        // note that we have now masked any requestBus and
2625        // schedSendEvent (we will wait for a retry before
2626        // doing anything), and this is so even if we do not
2627        // care about this packet and might override it before
2628        // it gets retried
2629        return true;
2630    } else {
2631        // As part of the call to sendTimingReq the packet is
2632        // forwarded to all neighbouring caches (and any caches
2633        // above them) as a snoop. Thus at this point we know if
2634        // any of the neighbouring caches are responding, and if
2635        // so, we know it is dirty, and we can determine if it is
2636        // being passed as Modified, making our MSHR the ordering
2637        // point
2638        bool pending_modified_resp = !pkt->hasSharers() &&
2639            pkt->cacheResponding();
2640        markInService(mshr, pending_modified_resp);
2641        if (pkt->isClean() && blk && blk->isDirty()) {
2642            // A cache clean opearation is looking for a dirty
2643            // block. If a dirty block is encountered a WriteClean
2644            // will update any copies to the path to the memory
2645            // until the point of reference.
2646            DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
2647                    __func__, pkt->print(), blk->print());
2648            PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
2649                                             pkt->id);
2650            PacketList writebacks;
2651            writebacks.push_back(wb_pkt);
2652            doWritebacks(writebacks, 0);
2653        }
2654
2655        return false;
2656    }
2657}
2658
2659bool
2660Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2661{
2662    assert(wq_entry);
2663
2664    // always a single target for write queue entries
2665    PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2666
2667    DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2668
2669    // forward as is, both for evictions and uncacheable writes
2670    if (!memSidePort->sendTimingReq(tgt_pkt)) {
2671        // note that we have now masked any requestBus and
2672        // schedSendEvent (we will wait for a retry before
2673        // doing anything), and this is so even if we do not
2674        // care about this packet and might override it before
2675        // it gets retried
2676        return true;
2677    } else {
2678        markInService(wq_entry);
2679        return false;
2680    }
2681}
2682
2683void
2684Cache::serialize(CheckpointOut &cp) const
2685{
2686    bool dirty(isDirty());
2687
2688    if (dirty) {
2689        warn("*** The cache still contains dirty data. ***\n");
2690        warn("    Make sure to drain the system using the correct flags.\n");
2691        warn("    This checkpoint will not restore correctly and dirty data "
2692             "    in the cache will be lost!\n");
2693    }
2694
2695    // Since we don't checkpoint the data in the cache, any dirty data
2696    // will be lost when restoring from a checkpoint of a system that
2697    // wasn't drained properly. Flag the checkpoint as invalid if the
2698    // cache contains dirty data.
2699    bool bad_checkpoint(dirty);
2700    SERIALIZE_SCALAR(bad_checkpoint);
2701}
2702
2703void
2704Cache::unserialize(CheckpointIn &cp)
2705{
2706    bool bad_checkpoint;
2707    UNSERIALIZE_SCALAR(bad_checkpoint);
2708    if (bad_checkpoint) {
2709        fatal("Restoring from checkpoints with dirty caches is not supported "
2710              "in the classic memory system. Please remove any caches or "
2711              " drain them properly before taking checkpoints.\n");
2712    }
2713}
2714
2715///////////////
2716//
2717// CpuSidePort
2718//
2719///////////////
2720
2721AddrRangeList
2722Cache::CpuSidePort::getAddrRanges() const
2723{
2724    return cache->getAddrRanges();
2725}
2726
2727bool
2728Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2729{
2730    assert(!cache->system->bypassCaches());
2731
2732    // always let express snoop packets through if even if blocked
2733    if (pkt->isExpressSnoop()) {
2734        return true;
2735    } else if (isBlocked() || mustSendRetry) {
2736        // either already committed to send a retry, or blocked
2737        mustSendRetry = true;
2738        return false;
2739    }
2740    mustSendRetry = false;
2741    return true;
2742}
2743
2744bool
2745Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2746{
2747    assert(!cache->system->bypassCaches());
2748
2749    // always let express snoop packets through if even if blocked
2750    if (pkt->isExpressSnoop() || tryTiming(pkt)) {
2751        cache->recvTimingReq(pkt);
2752        return true;
2753    }
2754    return false;
2755}
2756
2757Tick
2758Cache::CpuSidePort::recvAtomic(PacketPtr pkt)
2759{
2760    return cache->recvAtomic(pkt);
2761}
2762
2763void
2764Cache::CpuSidePort::recvFunctional(PacketPtr pkt)
2765{
2766    // functional request
2767    cache->functionalAccess(pkt, true);
2768}
2769
2770Cache::
2771CpuSidePort::CpuSidePort(const std::string &_name, Cache *_cache,
2772                         const std::string &_label)
2773    : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2774{
2775}
2776
2777Cache*
2778CacheParams::create()
2779{
2780    assert(tags);
2781    assert(replacement_policy);
2782
2783    return new Cache(this);
2784}
2785///////////////
2786//
2787// MemSidePort
2788//
2789///////////////
2790
2791bool
2792Cache::MemSidePort::recvTimingResp(PacketPtr pkt)
2793{
2794    cache->recvTimingResp(pkt);
2795    return true;
2796}
2797
2798// Express snooping requests to memside port
2799void
2800Cache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2801{
2802    // handle snooping requests
2803    cache->recvTimingSnoopReq(pkt);
2804}
2805
2806Tick
2807Cache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2808{
2809    return cache->recvAtomicSnoop(pkt);
2810}
2811
2812void
2813Cache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2814{
2815    // functional snoop (note that in contrast to atomic we don't have
2816    // a specific functionalSnoop method, as they have the same
2817    // behaviour regardless)
2818    cache->functionalAccess(pkt, false);
2819}
2820
2821void
2822Cache::CacheReqPacketQueue::sendDeferredPacket()
2823{
2824    // sanity check
2825    assert(!waitingOnRetry);
2826
2827    // there should never be any deferred request packets in the
2828    // queue, instead we resly on the cache to provide the packets
2829    // from the MSHR queue or write queue
2830    assert(deferredPacketReadyTime() == MaxTick);
2831
2832    // check for request packets (requests & writebacks)
2833    QueueEntry* entry = cache.getNextQueueEntry();
2834
2835    if (!entry) {
2836        // can happen if e.g. we attempt a writeback and fail, but
2837        // before the retry, the writeback is eliminated because
2838        // we snoop another cache's ReadEx.
2839    } else {
2840        // let our snoop responses go first if there are responses to
2841        // the same addresses
2842        if (checkConflictingSnoop(entry->blkAddr)) {
2843            return;
2844        }
2845        waitingOnRetry = entry->sendPacket(cache);
2846    }
2847
2848    // if we succeeded and are not waiting for a retry, schedule the
2849    // next send considering when the next queue is ready, note that
2850    // snoop responses have their own packet queue and thus schedule
2851    // their own events
2852    if (!waitingOnRetry) {
2853        schedSendEvent(cache.nextQueueReadyTime());
2854    }
2855}
2856
2857Cache::
2858MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2859                         const std::string &_label)
2860    : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2861      _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2862      _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2863{
2864}
2865