1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 * Dave Greene
43 * Nathan Binkert
44 * Steve Reinhardt
45 * Ron Dreslinski
46 * Andreas Sandberg
47 * Nikos Nikoleris
48 */
49
50/**
51 * @file
52 * Cache definitions.
53 */
54
55#include "mem/cache/cache.hh"
56
57#include "base/logging.hh"
58#include "base/types.hh"
59#include "debug/Cache.hh"
60#include "debug/CachePort.hh"
61#include "debug/CacheTags.hh"
62#include "debug/CacheVerbose.hh"
63#include "mem/cache/blk.hh"
64#include "mem/cache/mshr.hh"
65#include "mem/cache/prefetch/base.hh"
66#include "sim/sim_exit.hh"
67
68Cache::Cache(const CacheParams *p)
69 : BaseCache(p, p->system->cacheLineSize()),
70 tags(p->tags),
71 prefetcher(p->prefetcher),
72 doFastWrites(true),
73 prefetchOnAccess(p->prefetch_on_access),
74 clusivity(p->clusivity),
75 writebackClean(p->writeback_clean),
76 tempBlockWriteback(nullptr),
77 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
78 name(), false,
79 EventBase::Delayed_Writeback_Pri)
80{
81 tempBlock = new CacheBlk();
82 tempBlock->data = new uint8_t[blkSize];
83
84 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
85 "CpuSidePort");
86 memSidePort = new MemSidePort(p->name + ".mem_side", this,
87 "MemSidePort");
88
89 tags->setCache(this);
90 if (prefetcher)
91 prefetcher->setCache(this);
92}
93
94Cache::~Cache()
95{
96 delete [] tempBlock->data;
97 delete tempBlock;
98
99 delete cpuSidePort;
100 delete memSidePort;
101}
102
103void
104Cache::regStats()
105{
106 BaseCache::regStats();
107}
108
109void
110Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
111{
112 assert(pkt->isRequest());
113
114 uint64_t overwrite_val;
115 bool overwrite_mem;
116 uint64_t condition_val64;
117 uint32_t condition_val32;
118
119 int offset = tags->extractBlkOffset(pkt->getAddr());
120 uint8_t *blk_data = blk->data + offset;
121
122 assert(sizeof(uint64_t) >= pkt->getSize());
123
124 overwrite_mem = true;
125 // keep a copy of our possible write value, and copy what is at the
126 // memory address into the packet
127 pkt->writeData((uint8_t *)&overwrite_val);
128 pkt->setData(blk_data);
129
130 if (pkt->req->isCondSwap()) {
131 if (pkt->getSize() == sizeof(uint64_t)) {
132 condition_val64 = pkt->req->getExtraData();
133 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
134 sizeof(uint64_t));
135 } else if (pkt->getSize() == sizeof(uint32_t)) {
136 condition_val32 = (uint32_t)pkt->req->getExtraData();
137 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
138 sizeof(uint32_t));
139 } else
140 panic("Invalid size for conditional read/write\n");
141 }
142
143 if (overwrite_mem) {
144 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
145 blk->status |= BlkDirty;
146 }
147}
148
149
150void
151Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
152 bool deferred_response, bool pending_downgrade)
153{
154 assert(pkt->isRequest());
155
156 assert(blk && blk->isValid());
157 // Occasionally this is not true... if we are a lower-level cache
158 // satisfying a string of Read and ReadEx requests from
159 // upper-level caches, a Read will mark the block as shared but we
160 // can satisfy a following ReadEx anyway since we can rely on the
161 // Read requester(s) to have buffered the ReadEx snoop and to
162 // invalidate their blocks after receiving them.
163 // assert(!pkt->needsWritable() || blk->isWritable());
164 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
165
166 // Check RMW operations first since both isRead() and
167 // isWrite() will be true for them
168 if (pkt->cmd == MemCmd::SwapReq) {
169 cmpAndSwap(blk, pkt);
170 } else if (pkt->isWrite()) {
171 // we have the block in a writable state and can go ahead,
172 // note that the line may be also be considered writable in
173 // downstream caches along the path to memory, but always
174 // Exclusive, and never Modified
175 assert(blk->isWritable());
176 // Write or WriteLine at the first cache with block in writable state
177 if (blk->checkWrite(pkt)) {
178 pkt->writeDataToBlock(blk->data, blkSize);
179 }
180 // Always mark the line as dirty (and thus transition to the
181 // Modified state) even if we are a failed StoreCond so we
182 // supply data to any snoops that have appended themselves to
183 // this cache before knowing the store will fail.
184 blk->status |= BlkDirty;
185 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
186 } else if (pkt->isRead()) {
187 if (pkt->isLLSC()) {
188 blk->trackLoadLocked(pkt);
189 }
190
191 // all read responses have a data payload
192 assert(pkt->hasRespData());
193 pkt->setDataFromBlock(blk->data, blkSize);
194
195 // determine if this read is from a (coherent) cache or not
196 if (pkt->fromCache()) {
197 assert(pkt->getSize() == blkSize);
198 // special handling for coherent block requests from
199 // upper-level caches
200 if (pkt->needsWritable()) {
201 // sanity check
202 assert(pkt->cmd == MemCmd::ReadExReq ||
203 pkt->cmd == MemCmd::SCUpgradeFailReq);
204 assert(!pkt->hasSharers());
205
206 // if we have a dirty copy, make sure the recipient
207 // keeps it marked dirty (in the modified state)
208 if (blk->isDirty()) {
209 pkt->setCacheResponding();
210 blk->status &= ~BlkDirty;
211 }
212 } else if (blk->isWritable() && !pending_downgrade &&
213 !pkt->hasSharers() &&
214 pkt->cmd != MemCmd::ReadCleanReq) {
215 // we can give the requester a writable copy on a read
216 // request if:
217 // - we have a writable copy at this level (& below)
218 // - we don't have a pending snoop from below
219 // signaling another read request
220 // - no other cache above has a copy (otherwise it
221 // would have set hasSharers flag when
222 // snooping the packet)
223 // - the read has explicitly asked for a clean
224 // copy of the line
225 if (blk->isDirty()) {
226 // special considerations if we're owner:
227 if (!deferred_response) {
228 // respond with the line in Modified state
229 // (cacheResponding set, hasSharers not set)
230 pkt->setCacheResponding();
231
232 // if this cache is mostly inclusive, we
233 // keep the block in the Exclusive state,
234 // and pass it upwards as Modified
235 // (writable and dirty), hence we have
236 // multiple caches, all on the same path
237 // towards memory, all considering the
238 // same block writable, but only one
239 // considering it Modified
240
241 // we get away with multiple caches (on
242 // the same path to memory) considering
243 // the block writeable as we always enter
244 // the cache hierarchy through a cache,
245 // and first snoop upwards in all other
246 // branches
247 blk->status &= ~BlkDirty;
248 } else {
249 // if we're responding after our own miss,
250 // there's a window where the recipient didn't
251 // know it was getting ownership and may not
252 // have responded to snoops correctly, so we
253 // have to respond with a shared line
254 pkt->setHasSharers();
255 }
256 }
257 } else {
258 // otherwise only respond with a shared copy
259 pkt->setHasSharers();
260 }
261 }
262 } else if (pkt->isUpgrade()) {
263 // sanity check
264 assert(!pkt->hasSharers());
265
266 if (blk->isDirty()) {
267 // we were in the Owned state, and a cache above us that
268 // has the line in Shared state needs to be made aware
269 // that the data it already has is in fact dirty
270 pkt->setCacheResponding();
271 blk->status &= ~BlkDirty;
272 }
273 } else {
274 assert(pkt->isInvalidate());
275 invalidateBlock(blk);
276 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
277 pkt->print());
278 }
279}
280
281/////////////////////////////////////////////////////
282//
283// Access path: requests coming in from the CPU side
284//
285/////////////////////////////////////////////////////
286
287bool
288Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
289 PacketList &writebacks)
290{
291 // sanity check
292 assert(pkt->isRequest());
293
294 chatty_assert(!(isReadOnly && pkt->isWrite()),
295 "Should never see a write in a read-only cache %s\n",
296 name());
297
298 DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
299
300 if (pkt->req->isUncacheable()) {
301 DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
302
303 // flush and invalidate any existing block
304 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
305 if (old_blk && old_blk->isValid()) {
306 if (old_blk->isDirty() || writebackClean)
307 writebacks.push_back(writebackBlk(old_blk));
308 else
309 writebacks.push_back(cleanEvictBlk(old_blk));
310 invalidateBlock(old_blk);
311 }
312
313 blk = nullptr;
314 // lookupLatency is the latency in case the request is uncacheable.
315 lat = lookupLatency;
316 return false;
317 }
318
319 // Here lat is the value passed as parameter to accessBlock() function
320 // that can modify its value.
321 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
322
323 DPRINTF(Cache, "%s %s\n", pkt->print(),
324 blk ? "hit " + blk->print() : "miss");
325
326 if (pkt->req->isCacheMaintenance()) {
327 // A cache maintenance operation is always forwarded to the
328 // memory below even if the block is found in dirty state.
329
330 // We defer any changes to the state of the block until we
331 // create and mark as in service the mshr for the downstream
332 // packet.
333 return false;
334 }
335
336 if (pkt->isEviction()) {
337 // We check for presence of block in above caches before issuing
338 // Writeback or CleanEvict to write buffer. Therefore the only
339 // possible cases can be of a CleanEvict packet coming from above
340 // encountering a Writeback generated in this cache peer cache and
341 // waiting in the write buffer. Cases of upper level peer caches
342 // generating CleanEvict and Writeback or simply CleanEvict and
343 // CleanEvict almost simultaneously will be caught by snoops sent out
344 // by crossbar.
345 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
346 pkt->isSecure());
347 if (wb_entry) {
348 assert(wb_entry->getNumTargets() == 1);
349 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
350 assert(wbPkt->isWriteback());
351
352 if (pkt->isCleanEviction()) {
353 // The CleanEvict and WritebackClean snoops into other
354 // peer caches of the same level while traversing the
355 // crossbar. If a copy of the block is found, the
356 // packet is deleted in the crossbar. Hence, none of
357 // the other upper level caches connected to this
358 // cache have the block, so we can clear the
359 // BLOCK_CACHED flag in the Writeback if set and
360 // discard the CleanEvict by returning true.
361 wbPkt->clearBlockCached();
362 return true;
363 } else {
364 assert(pkt->cmd == MemCmd::WritebackDirty);
365 // Dirty writeback from above trumps our clean
366 // writeback... discard here
367 // Note: markInService will remove entry from writeback buffer.
368 markInService(wb_entry);
369 delete wbPkt;
370 }
371 }
372 }
373
374 // Writeback handling is special case. We can write the block into
375 // the cache without having a writeable copy (or any copy at all).
376 if (pkt->isWriteback()) {
377 assert(blkSize == pkt->getSize());
378
379 // we could get a clean writeback while we are having
380 // outstanding accesses to a block, do the simple thing for
381 // now and drop the clean writeback so that we do not upset
382 // any ordering/decisions about ownership already taken
383 if (pkt->cmd == MemCmd::WritebackClean &&
384 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
385 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
386 "dropping\n", pkt->getAddr());
387 return true;
388 }
389
390 if (blk == nullptr) {
391 // need to do a replacement
392 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
393 if (blk == nullptr) {
394 // no replaceable block available: give up, fwd to next level.
395 incMissCount(pkt);
396 return false;
397 }
398 tags->insertBlock(pkt, blk);
399
400 blk->status = (BlkValid | BlkReadable);
401 if (pkt->isSecure()) {
402 blk->status |= BlkSecure;
403 }
404 }
405 // only mark the block dirty if we got a writeback command,
406 // and leave it as is for a clean writeback
407 if (pkt->cmd == MemCmd::WritebackDirty) {
408 assert(!blk->isDirty());
409 blk->status |= BlkDirty;
410 }
411 // if the packet does not have sharers, it is passing
412 // writable, and we got the writeback in Modified or Exclusive
413 // state, if not we are in the Owned or Shared state
414 if (!pkt->hasSharers()) {
415 blk->status |= BlkWritable;
416 }
417 // nothing else to do; writeback doesn't expect response
418 assert(!pkt->needsResponse());
419 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
420 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
421 incHitCount(pkt);
422 // populate the time when the block will be ready to access.
423 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
424 pkt->payloadDelay;
425 return true;
426 } else if (pkt->cmd == MemCmd::CleanEvict) {
427 if (blk != nullptr) {
428 // Found the block in the tags, need to stop CleanEvict from
429 // propagating further down the hierarchy. Returning true will
430 // treat the CleanEvict like a satisfied write request and delete
431 // it.
432 return true;
433 }
434 // We didn't find the block here, propagate the CleanEvict further
435 // down the memory hierarchy. Returning false will treat the CleanEvict
436 // like a Writeback which could not find a replaceable block so has to
437 // go to next level.
438 return false;
439 } else if (pkt->cmd == MemCmd::WriteClean) {
440 // WriteClean handling is a special case. We can allocate a
441 // block directly if it doesn't exist and we can update the
442 // block immediately. The WriteClean transfers the ownership
443 // of the block as well.
444 assert(blkSize == pkt->getSize());
445
446 if (!blk) {
447 if (pkt->writeThrough()) {
448 // if this is a write through packet, we don't try to
449 // allocate if the block is not present
450 return false;
451 } else {
452 // a writeback that misses needs to allocate a new block
453 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
454 writebacks);
455 if (!blk) {
456 // no replaceable block available: give up, fwd to
457 // next level.
458 incMissCount(pkt);
459 return false;
460 }
461 tags->insertBlock(pkt, blk);
462
463 blk->status = (BlkValid | BlkReadable);
464 if (pkt->isSecure()) {
465 blk->status |= BlkSecure;
466 }
467 }
468 }
469
470 // at this point either this is a writeback or a write-through
471 // write clean operation and the block is already in this
472 // cache, we need to update the data and the block flags
473 assert(blk);
474 assert(!blk->isDirty());
475 if (!pkt->writeThrough()) {
476 blk->status |= BlkDirty;
477 }
478 // nothing else to do; writeback doesn't expect response
479 assert(!pkt->needsResponse());
480 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
481 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
482
483 incHitCount(pkt);
484 // populate the time when the block will be ready to access.
485 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
486 pkt->payloadDelay;
487 // if this a write-through packet it will be sent to cache
488 // below
489 return !pkt->writeThrough();
490 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
491 blk->isReadable())) {
492 // OK to satisfy access
493 incHitCount(pkt);
494 satisfyRequest(pkt, blk);
495 maintainClusivity(pkt->fromCache(), blk);
496
497 return true;
498 }
499
500 // Can't satisfy access normally... either no block (blk == nullptr)
501 // or have block but need writable
502
503 incMissCount(pkt);
504
505 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
506 // complete miss on store conditional... just give up now
507 pkt->req->setExtraData(0);
508 return true;
509 }
510
511 return false;
512}
513
514void
515Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
516{
517 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
518 clusivity == Enums::mostly_excl) {
519 // if we have responded to a cache, and our block is still
520 // valid, but not dirty, and this cache is mostly exclusive
521 // with respect to the cache above, drop the block
522 invalidateBlock(blk);
523 }
524}
525
526void
527Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
528{
529 while (!writebacks.empty()) {
530 PacketPtr wbPkt = writebacks.front();
531 // We use forwardLatency here because we are copying writebacks to
532 // write buffer.
533
534 // Call isCachedAbove for Writebacks, CleanEvicts and
535 // WriteCleans to discover if the block is cached above.
536 if (isCachedAbove(wbPkt)) {
537 if (wbPkt->cmd == MemCmd::CleanEvict) {
538 // Delete CleanEvict because cached copies exist above. The
539 // packet destructor will delete the request object because
540 // this is a non-snoop request packet which does not require a
541 // response.
542 delete wbPkt;
543 } else if (wbPkt->cmd == MemCmd::WritebackClean) {
544 // clean writeback, do not send since the block is
545 // still cached above
546 assert(writebackClean);
547 delete wbPkt;
548 } else {
549 assert(wbPkt->cmd == MemCmd::WritebackDirty ||
550 wbPkt->cmd == MemCmd::WriteClean);
551 // Set BLOCK_CACHED flag in Writeback and send below, so that
552 // the Writeback does not reset the bit corresponding to this
553 // address in the snoop filter below.
554 wbPkt->setBlockCached();
555 allocateWriteBuffer(wbPkt, forward_time);
556 }
557 } else {
558 // If the block is not cached above, send packet below. Both
559 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
560 // reset the bit corresponding to this address in the snoop filter
561 // below.
562 allocateWriteBuffer(wbPkt, forward_time);
563 }
564 writebacks.pop_front();
565 }
566}
567
568void
569Cache::doWritebacksAtomic(PacketList& writebacks)
570{
571 while (!writebacks.empty()) {
572 PacketPtr wbPkt = writebacks.front();
573 // Call isCachedAbove for both Writebacks and CleanEvicts. If
574 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
575 // and discard CleanEvicts.
576 if (isCachedAbove(wbPkt, false)) {
577 if (wbPkt->cmd == MemCmd::WritebackDirty ||
578 wbPkt->cmd == MemCmd::WriteClean) {
579 // Set BLOCK_CACHED flag in Writeback and send below,
580 // so that the Writeback does not reset the bit
581 // corresponding to this address in the snoop filter
582 // below. We can discard CleanEvicts because cached
583 // copies exist above. Atomic mode isCachedAbove
584 // modifies packet to set BLOCK_CACHED flag
585 memSidePort->sendAtomic(wbPkt);
586 }
587 } else {
588 // If the block is not cached above, send packet below. Both
589 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
590 // reset the bit corresponding to this address in the snoop filter
591 // below.
592 memSidePort->sendAtomic(wbPkt);
593 }
594 writebacks.pop_front();
595 // In case of CleanEvicts, the packet destructor will delete the
596 // request object because this is a non-snoop request packet which
597 // does not require a response.
598 delete wbPkt;
599 }
600}
601
602
603void
604Cache::recvTimingSnoopResp(PacketPtr pkt)
605{
606 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
607
608 assert(pkt->isResponse());
609 assert(!system->bypassCaches());
610
611 // determine if the response is from a snoop request we created
612 // (in which case it should be in the outstandingSnoop), or if we
613 // merely forwarded someone else's snoop request
614 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
615 outstandingSnoop.end();
616
617 if (!forwardAsSnoop) {
618 // the packet came from this cache, so sink it here and do not
619 // forward it
620 assert(pkt->cmd == MemCmd::HardPFResp);
621
622 outstandingSnoop.erase(pkt->req);
623
624 DPRINTF(Cache, "Got prefetch response from above for addr "
625 "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
626 recvTimingResp(pkt);
627 return;
628 }
629
630 // forwardLatency is set here because there is a response from an
631 // upper level cache.
632 // To pay the delay that occurs if the packet comes from the bus,
633 // we charge also headerDelay.
634 Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
635 // Reset the timing of the packet.
636 pkt->headerDelay = pkt->payloadDelay = 0;
637 memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
638}
639
640void
641Cache::promoteWholeLineWrites(PacketPtr pkt)
642{
643 // Cache line clearing instructions
644 if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
645 (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
646 pkt->cmd = MemCmd::WriteLineReq;
647 DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
648 }
649}
650
| 1/*
2 * Copyright (c) 2010-2018 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 * Dave Greene
43 * Nathan Binkert
44 * Steve Reinhardt
45 * Ron Dreslinski
46 * Andreas Sandberg
47 * Nikos Nikoleris
48 */
49
50/**
51 * @file
52 * Cache definitions.
53 */
54
55#include "mem/cache/cache.hh"
56
57#include "base/logging.hh"
58#include "base/types.hh"
59#include "debug/Cache.hh"
60#include "debug/CachePort.hh"
61#include "debug/CacheTags.hh"
62#include "debug/CacheVerbose.hh"
63#include "mem/cache/blk.hh"
64#include "mem/cache/mshr.hh"
65#include "mem/cache/prefetch/base.hh"
66#include "sim/sim_exit.hh"
67
68Cache::Cache(const CacheParams *p)
69 : BaseCache(p, p->system->cacheLineSize()),
70 tags(p->tags),
71 prefetcher(p->prefetcher),
72 doFastWrites(true),
73 prefetchOnAccess(p->prefetch_on_access),
74 clusivity(p->clusivity),
75 writebackClean(p->writeback_clean),
76 tempBlockWriteback(nullptr),
77 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
78 name(), false,
79 EventBase::Delayed_Writeback_Pri)
80{
81 tempBlock = new CacheBlk();
82 tempBlock->data = new uint8_t[blkSize];
83
84 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
85 "CpuSidePort");
86 memSidePort = new MemSidePort(p->name + ".mem_side", this,
87 "MemSidePort");
88
89 tags->setCache(this);
90 if (prefetcher)
91 prefetcher->setCache(this);
92}
93
94Cache::~Cache()
95{
96 delete [] tempBlock->data;
97 delete tempBlock;
98
99 delete cpuSidePort;
100 delete memSidePort;
101}
102
103void
104Cache::regStats()
105{
106 BaseCache::regStats();
107}
108
109void
110Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
111{
112 assert(pkt->isRequest());
113
114 uint64_t overwrite_val;
115 bool overwrite_mem;
116 uint64_t condition_val64;
117 uint32_t condition_val32;
118
119 int offset = tags->extractBlkOffset(pkt->getAddr());
120 uint8_t *blk_data = blk->data + offset;
121
122 assert(sizeof(uint64_t) >= pkt->getSize());
123
124 overwrite_mem = true;
125 // keep a copy of our possible write value, and copy what is at the
126 // memory address into the packet
127 pkt->writeData((uint8_t *)&overwrite_val);
128 pkt->setData(blk_data);
129
130 if (pkt->req->isCondSwap()) {
131 if (pkt->getSize() == sizeof(uint64_t)) {
132 condition_val64 = pkt->req->getExtraData();
133 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
134 sizeof(uint64_t));
135 } else if (pkt->getSize() == sizeof(uint32_t)) {
136 condition_val32 = (uint32_t)pkt->req->getExtraData();
137 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
138 sizeof(uint32_t));
139 } else
140 panic("Invalid size for conditional read/write\n");
141 }
142
143 if (overwrite_mem) {
144 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
145 blk->status |= BlkDirty;
146 }
147}
148
149
150void
151Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
152 bool deferred_response, bool pending_downgrade)
153{
154 assert(pkt->isRequest());
155
156 assert(blk && blk->isValid());
157 // Occasionally this is not true... if we are a lower-level cache
158 // satisfying a string of Read and ReadEx requests from
159 // upper-level caches, a Read will mark the block as shared but we
160 // can satisfy a following ReadEx anyway since we can rely on the
161 // Read requester(s) to have buffered the ReadEx snoop and to
162 // invalidate their blocks after receiving them.
163 // assert(!pkt->needsWritable() || blk->isWritable());
164 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
165
166 // Check RMW operations first since both isRead() and
167 // isWrite() will be true for them
168 if (pkt->cmd == MemCmd::SwapReq) {
169 cmpAndSwap(blk, pkt);
170 } else if (pkt->isWrite()) {
171 // we have the block in a writable state and can go ahead,
172 // note that the line may be also be considered writable in
173 // downstream caches along the path to memory, but always
174 // Exclusive, and never Modified
175 assert(blk->isWritable());
176 // Write or WriteLine at the first cache with block in writable state
177 if (blk->checkWrite(pkt)) {
178 pkt->writeDataToBlock(blk->data, blkSize);
179 }
180 // Always mark the line as dirty (and thus transition to the
181 // Modified state) even if we are a failed StoreCond so we
182 // supply data to any snoops that have appended themselves to
183 // this cache before knowing the store will fail.
184 blk->status |= BlkDirty;
185 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
186 } else if (pkt->isRead()) {
187 if (pkt->isLLSC()) {
188 blk->trackLoadLocked(pkt);
189 }
190
191 // all read responses have a data payload
192 assert(pkt->hasRespData());
193 pkt->setDataFromBlock(blk->data, blkSize);
194
195 // determine if this read is from a (coherent) cache or not
196 if (pkt->fromCache()) {
197 assert(pkt->getSize() == blkSize);
198 // special handling for coherent block requests from
199 // upper-level caches
200 if (pkt->needsWritable()) {
201 // sanity check
202 assert(pkt->cmd == MemCmd::ReadExReq ||
203 pkt->cmd == MemCmd::SCUpgradeFailReq);
204 assert(!pkt->hasSharers());
205
206 // if we have a dirty copy, make sure the recipient
207 // keeps it marked dirty (in the modified state)
208 if (blk->isDirty()) {
209 pkt->setCacheResponding();
210 blk->status &= ~BlkDirty;
211 }
212 } else if (blk->isWritable() && !pending_downgrade &&
213 !pkt->hasSharers() &&
214 pkt->cmd != MemCmd::ReadCleanReq) {
215 // we can give the requester a writable copy on a read
216 // request if:
217 // - we have a writable copy at this level (& below)
218 // - we don't have a pending snoop from below
219 // signaling another read request
220 // - no other cache above has a copy (otherwise it
221 // would have set hasSharers flag when
222 // snooping the packet)
223 // - the read has explicitly asked for a clean
224 // copy of the line
225 if (blk->isDirty()) {
226 // special considerations if we're owner:
227 if (!deferred_response) {
228 // respond with the line in Modified state
229 // (cacheResponding set, hasSharers not set)
230 pkt->setCacheResponding();
231
232 // if this cache is mostly inclusive, we
233 // keep the block in the Exclusive state,
234 // and pass it upwards as Modified
235 // (writable and dirty), hence we have
236 // multiple caches, all on the same path
237 // towards memory, all considering the
238 // same block writable, but only one
239 // considering it Modified
240
241 // we get away with multiple caches (on
242 // the same path to memory) considering
243 // the block writeable as we always enter
244 // the cache hierarchy through a cache,
245 // and first snoop upwards in all other
246 // branches
247 blk->status &= ~BlkDirty;
248 } else {
249 // if we're responding after our own miss,
250 // there's a window where the recipient didn't
251 // know it was getting ownership and may not
252 // have responded to snoops correctly, so we
253 // have to respond with a shared line
254 pkt->setHasSharers();
255 }
256 }
257 } else {
258 // otherwise only respond with a shared copy
259 pkt->setHasSharers();
260 }
261 }
262 } else if (pkt->isUpgrade()) {
263 // sanity check
264 assert(!pkt->hasSharers());
265
266 if (blk->isDirty()) {
267 // we were in the Owned state, and a cache above us that
268 // has the line in Shared state needs to be made aware
269 // that the data it already has is in fact dirty
270 pkt->setCacheResponding();
271 blk->status &= ~BlkDirty;
272 }
273 } else {
274 assert(pkt->isInvalidate());
275 invalidateBlock(blk);
276 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
277 pkt->print());
278 }
279}
280
281/////////////////////////////////////////////////////
282//
283// Access path: requests coming in from the CPU side
284//
285/////////////////////////////////////////////////////
286
287bool
288Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
289 PacketList &writebacks)
290{
291 // sanity check
292 assert(pkt->isRequest());
293
294 chatty_assert(!(isReadOnly && pkt->isWrite()),
295 "Should never see a write in a read-only cache %s\n",
296 name());
297
298 DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
299
300 if (pkt->req->isUncacheable()) {
301 DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
302
303 // flush and invalidate any existing block
304 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
305 if (old_blk && old_blk->isValid()) {
306 if (old_blk->isDirty() || writebackClean)
307 writebacks.push_back(writebackBlk(old_blk));
308 else
309 writebacks.push_back(cleanEvictBlk(old_blk));
310 invalidateBlock(old_blk);
311 }
312
313 blk = nullptr;
314 // lookupLatency is the latency in case the request is uncacheable.
315 lat = lookupLatency;
316 return false;
317 }
318
319 // Here lat is the value passed as parameter to accessBlock() function
320 // that can modify its value.
321 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
322
323 DPRINTF(Cache, "%s %s\n", pkt->print(),
324 blk ? "hit " + blk->print() : "miss");
325
326 if (pkt->req->isCacheMaintenance()) {
327 // A cache maintenance operation is always forwarded to the
328 // memory below even if the block is found in dirty state.
329
330 // We defer any changes to the state of the block until we
331 // create and mark as in service the mshr for the downstream
332 // packet.
333 return false;
334 }
335
336 if (pkt->isEviction()) {
337 // We check for presence of block in above caches before issuing
338 // Writeback or CleanEvict to write buffer. Therefore the only
339 // possible cases can be of a CleanEvict packet coming from above
340 // encountering a Writeback generated in this cache peer cache and
341 // waiting in the write buffer. Cases of upper level peer caches
342 // generating CleanEvict and Writeback or simply CleanEvict and
343 // CleanEvict almost simultaneously will be caught by snoops sent out
344 // by crossbar.
345 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
346 pkt->isSecure());
347 if (wb_entry) {
348 assert(wb_entry->getNumTargets() == 1);
349 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
350 assert(wbPkt->isWriteback());
351
352 if (pkt->isCleanEviction()) {
353 // The CleanEvict and WritebackClean snoops into other
354 // peer caches of the same level while traversing the
355 // crossbar. If a copy of the block is found, the
356 // packet is deleted in the crossbar. Hence, none of
357 // the other upper level caches connected to this
358 // cache have the block, so we can clear the
359 // BLOCK_CACHED flag in the Writeback if set and
360 // discard the CleanEvict by returning true.
361 wbPkt->clearBlockCached();
362 return true;
363 } else {
364 assert(pkt->cmd == MemCmd::WritebackDirty);
365 // Dirty writeback from above trumps our clean
366 // writeback... discard here
367 // Note: markInService will remove entry from writeback buffer.
368 markInService(wb_entry);
369 delete wbPkt;
370 }
371 }
372 }
373
374 // Writeback handling is special case. We can write the block into
375 // the cache without having a writeable copy (or any copy at all).
376 if (pkt->isWriteback()) {
377 assert(blkSize == pkt->getSize());
378
379 // we could get a clean writeback while we are having
380 // outstanding accesses to a block, do the simple thing for
381 // now and drop the clean writeback so that we do not upset
382 // any ordering/decisions about ownership already taken
383 if (pkt->cmd == MemCmd::WritebackClean &&
384 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
385 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
386 "dropping\n", pkt->getAddr());
387 return true;
388 }
389
390 if (blk == nullptr) {
391 // need to do a replacement
392 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
393 if (blk == nullptr) {
394 // no replaceable block available: give up, fwd to next level.
395 incMissCount(pkt);
396 return false;
397 }
398 tags->insertBlock(pkt, blk);
399
400 blk->status = (BlkValid | BlkReadable);
401 if (pkt->isSecure()) {
402 blk->status |= BlkSecure;
403 }
404 }
405 // only mark the block dirty if we got a writeback command,
406 // and leave it as is for a clean writeback
407 if (pkt->cmd == MemCmd::WritebackDirty) {
408 assert(!blk->isDirty());
409 blk->status |= BlkDirty;
410 }
411 // if the packet does not have sharers, it is passing
412 // writable, and we got the writeback in Modified or Exclusive
413 // state, if not we are in the Owned or Shared state
414 if (!pkt->hasSharers()) {
415 blk->status |= BlkWritable;
416 }
417 // nothing else to do; writeback doesn't expect response
418 assert(!pkt->needsResponse());
419 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
420 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
421 incHitCount(pkt);
422 // populate the time when the block will be ready to access.
423 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
424 pkt->payloadDelay;
425 return true;
426 } else if (pkt->cmd == MemCmd::CleanEvict) {
427 if (blk != nullptr) {
428 // Found the block in the tags, need to stop CleanEvict from
429 // propagating further down the hierarchy. Returning true will
430 // treat the CleanEvict like a satisfied write request and delete
431 // it.
432 return true;
433 }
434 // We didn't find the block here, propagate the CleanEvict further
435 // down the memory hierarchy. Returning false will treat the CleanEvict
436 // like a Writeback which could not find a replaceable block so has to
437 // go to next level.
438 return false;
439 } else if (pkt->cmd == MemCmd::WriteClean) {
440 // WriteClean handling is a special case. We can allocate a
441 // block directly if it doesn't exist and we can update the
442 // block immediately. The WriteClean transfers the ownership
443 // of the block as well.
444 assert(blkSize == pkt->getSize());
445
446 if (!blk) {
447 if (pkt->writeThrough()) {
448 // if this is a write through packet, we don't try to
449 // allocate if the block is not present
450 return false;
451 } else {
452 // a writeback that misses needs to allocate a new block
453 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
454 writebacks);
455 if (!blk) {
456 // no replaceable block available: give up, fwd to
457 // next level.
458 incMissCount(pkt);
459 return false;
460 }
461 tags->insertBlock(pkt, blk);
462
463 blk->status = (BlkValid | BlkReadable);
464 if (pkt->isSecure()) {
465 blk->status |= BlkSecure;
466 }
467 }
468 }
469
470 // at this point either this is a writeback or a write-through
471 // write clean operation and the block is already in this
472 // cache, we need to update the data and the block flags
473 assert(blk);
474 assert(!blk->isDirty());
475 if (!pkt->writeThrough()) {
476 blk->status |= BlkDirty;
477 }
478 // nothing else to do; writeback doesn't expect response
479 assert(!pkt->needsResponse());
480 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
481 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
482
483 incHitCount(pkt);
484 // populate the time when the block will be ready to access.
485 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
486 pkt->payloadDelay;
487 // if this a write-through packet it will be sent to cache
488 // below
489 return !pkt->writeThrough();
490 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
491 blk->isReadable())) {
492 // OK to satisfy access
493 incHitCount(pkt);
494 satisfyRequest(pkt, blk);
495 maintainClusivity(pkt->fromCache(), blk);
496
497 return true;
498 }
499
500 // Can't satisfy access normally... either no block (blk == nullptr)
501 // or have block but need writable
502
503 incMissCount(pkt);
504
505 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
506 // complete miss on store conditional... just give up now
507 pkt->req->setExtraData(0);
508 return true;
509 }
510
511 return false;
512}
513
514void
515Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
516{
517 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
518 clusivity == Enums::mostly_excl) {
519 // if we have responded to a cache, and our block is still
520 // valid, but not dirty, and this cache is mostly exclusive
521 // with respect to the cache above, drop the block
522 invalidateBlock(blk);
523 }
524}
525
526void
527Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
528{
529 while (!writebacks.empty()) {
530 PacketPtr wbPkt = writebacks.front();
531 // We use forwardLatency here because we are copying writebacks to
532 // write buffer.
533
534 // Call isCachedAbove for Writebacks, CleanEvicts and
535 // WriteCleans to discover if the block is cached above.
536 if (isCachedAbove(wbPkt)) {
537 if (wbPkt->cmd == MemCmd::CleanEvict) {
538 // Delete CleanEvict because cached copies exist above. The
539 // packet destructor will delete the request object because
540 // this is a non-snoop request packet which does not require a
541 // response.
542 delete wbPkt;
543 } else if (wbPkt->cmd == MemCmd::WritebackClean) {
544 // clean writeback, do not send since the block is
545 // still cached above
546 assert(writebackClean);
547 delete wbPkt;
548 } else {
549 assert(wbPkt->cmd == MemCmd::WritebackDirty ||
550 wbPkt->cmd == MemCmd::WriteClean);
551 // Set BLOCK_CACHED flag in Writeback and send below, so that
552 // the Writeback does not reset the bit corresponding to this
553 // address in the snoop filter below.
554 wbPkt->setBlockCached();
555 allocateWriteBuffer(wbPkt, forward_time);
556 }
557 } else {
558 // If the block is not cached above, send packet below. Both
559 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
560 // reset the bit corresponding to this address in the snoop filter
561 // below.
562 allocateWriteBuffer(wbPkt, forward_time);
563 }
564 writebacks.pop_front();
565 }
566}
567
568void
569Cache::doWritebacksAtomic(PacketList& writebacks)
570{
571 while (!writebacks.empty()) {
572 PacketPtr wbPkt = writebacks.front();
573 // Call isCachedAbove for both Writebacks and CleanEvicts. If
574 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
575 // and discard CleanEvicts.
576 if (isCachedAbove(wbPkt, false)) {
577 if (wbPkt->cmd == MemCmd::WritebackDirty ||
578 wbPkt->cmd == MemCmd::WriteClean) {
579 // Set BLOCK_CACHED flag in Writeback and send below,
580 // so that the Writeback does not reset the bit
581 // corresponding to this address in the snoop filter
582 // below. We can discard CleanEvicts because cached
583 // copies exist above. Atomic mode isCachedAbove
584 // modifies packet to set BLOCK_CACHED flag
585 memSidePort->sendAtomic(wbPkt);
586 }
587 } else {
588 // If the block is not cached above, send packet below. Both
589 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
590 // reset the bit corresponding to this address in the snoop filter
591 // below.
592 memSidePort->sendAtomic(wbPkt);
593 }
594 writebacks.pop_front();
595 // In case of CleanEvicts, the packet destructor will delete the
596 // request object because this is a non-snoop request packet which
597 // does not require a response.
598 delete wbPkt;
599 }
600}
601
602
603void
604Cache::recvTimingSnoopResp(PacketPtr pkt)
605{
606 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
607
608 assert(pkt->isResponse());
609 assert(!system->bypassCaches());
610
611 // determine if the response is from a snoop request we created
612 // (in which case it should be in the outstandingSnoop), or if we
613 // merely forwarded someone else's snoop request
614 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
615 outstandingSnoop.end();
616
617 if (!forwardAsSnoop) {
618 // the packet came from this cache, so sink it here and do not
619 // forward it
620 assert(pkt->cmd == MemCmd::HardPFResp);
621
622 outstandingSnoop.erase(pkt->req);
623
624 DPRINTF(Cache, "Got prefetch response from above for addr "
625 "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
626 recvTimingResp(pkt);
627 return;
628 }
629
630 // forwardLatency is set here because there is a response from an
631 // upper level cache.
632 // To pay the delay that occurs if the packet comes from the bus,
633 // we charge also headerDelay.
634 Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
635 // Reset the timing of the packet.
636 pkt->headerDelay = pkt->payloadDelay = 0;
637 memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
638}
639
640void
641Cache::promoteWholeLineWrites(PacketPtr pkt)
642{
643 // Cache line clearing instructions
644 if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
645 (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
646 pkt->cmd = MemCmd::WriteLineReq;
647 DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
648 }
649}
650
|
981}
982
983PacketPtr
984Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
985 bool needsWritable) const
986{
987 // should never see evictions here
988 assert(!cpu_pkt->isEviction());
989
990 bool blkValid = blk && blk->isValid();
991
992 if (cpu_pkt->req->isUncacheable() ||
993 (!blkValid && cpu_pkt->isUpgrade()) ||
994 cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
995 // uncacheable requests and upgrades from upper-level caches
996 // that missed completely just go through as is
997 return nullptr;
998 }
999
1000 assert(cpu_pkt->needsResponse());
1001
1002 MemCmd cmd;
1003 // @TODO make useUpgrades a parameter.
1004 // Note that ownership protocols require upgrade, otherwise a
1005 // write miss on a shared owned block will generate a ReadExcl,
1006 // which will clobber the owned copy.
1007 const bool useUpgrades = true;
1008 if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
1009 assert(!blkValid || !blk->isWritable());
1010 // forward as invalidate to all other caches, this gives us
1011 // the line in Exclusive state, and invalidates all other
1012 // copies
1013 cmd = MemCmd::InvalidateReq;
1014 } else if (blkValid && useUpgrades) {
1015 // only reason to be here is that blk is read only and we need
1016 // it to be writable
1017 assert(needsWritable);
1018 assert(!blk->isWritable());
1019 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
1020 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
1021 cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
1022 // Even though this SC will fail, we still need to send out the
1023 // request and get the data to supply it to other snoopers in the case
1024 // where the determination the StoreCond fails is delayed due to
1025 // all caches not being on the same local bus.
1026 cmd = MemCmd::SCUpgradeFailReq;
1027 } else {
1028 // block is invalid
1029
1030 // If the request does not need a writable there are two cases
1031 // where we need to ensure the response will not fetch the
1032 // block in dirty state:
1033 // * this cache is read only and it does not perform
1034 // writebacks,
1035 // * this cache is mostly exclusive and will not fill (since
1036 // it does not fill it will have to writeback the dirty data
1037 // immediately which generates uneccesary writebacks).
1038 bool force_clean_rsp = isReadOnly || clusivity == Enums::mostly_excl;
1039 cmd = needsWritable ? MemCmd::ReadExReq :
1040 (force_clean_rsp ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1041 }
1042 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1043
1044 // if there are upstream caches that have already marked the
1045 // packet as having sharers (not passing writable), pass that info
1046 // downstream
1047 if (cpu_pkt->hasSharers() && !needsWritable) {
1048 // note that cpu_pkt may have spent a considerable time in the
1049 // MSHR queue and that the information could possibly be out
1050 // of date, however, there is no harm in conservatively
1051 // assuming the block has sharers
1052 pkt->setHasSharers();
1053 DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1054 __func__, cpu_pkt->print(), pkt->print());
1055 }
1056
1057 // the packet should be block aligned
1058 assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1059
1060 pkt->allocate();
1061 DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1062 cpu_pkt->print());
1063 return pkt;
1064}
1065
1066
1067Tick
1068Cache::recvAtomic(PacketPtr pkt)
1069{
1070 // We are in atomic mode so we pay just for lookupLatency here.
1071 Cycles lat = lookupLatency;
1072
1073 // Forward the request if the system is in cache bypass mode.
1074 if (system->bypassCaches())
1075 return ticksToCycles(memSidePort->sendAtomic(pkt));
1076
1077 promoteWholeLineWrites(pkt);
1078
1079 // follow the same flow as in recvTimingReq, and check if a cache
1080 // above us is responding
1081 if (pkt->cacheResponding() && !pkt->isClean()) {
1082 assert(!pkt->req->isCacheInvalidate());
1083 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1084 pkt->print());
1085
1086 // if a cache is responding, and it had the line in Owned
1087 // rather than Modified state, we need to invalidate any
1088 // copies that are not on the same path to memory
1089 assert(pkt->needsWritable() && !pkt->responderHadWritable());
1090 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1091
1092 return lat * clockPeriod();
1093 }
1094
1095 // should assert here that there are no outstanding MSHRs or
1096 // writebacks... that would mean that someone used an atomic
1097 // access in timing mode
1098
1099 CacheBlk *blk = nullptr;
1100 PacketList writebacks;
1101 bool satisfied = access(pkt, blk, lat, writebacks);
1102
1103 if (pkt->isClean() && blk && blk->isDirty()) {
1104 // A cache clean opearation is looking for a dirty
1105 // block. If a dirty block is encountered a WriteClean
1106 // will update any copies to the path to the memory
1107 // until the point of reference.
1108 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1109 __func__, pkt->print(), blk->print());
1110 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
1111 writebacks.push_back(wb_pkt);
1112 pkt->setSatisfied();
1113 }
1114
1115 // handle writebacks resulting from the access here to ensure they
1116 // logically proceed anything happening below
1117 doWritebacksAtomic(writebacks);
1118
1119 if (!satisfied) {
1120 // MISS
1121
1122 // deal with the packets that go through the write path of
1123 // the cache, i.e. any evictions and writes
1124 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1125 (pkt->req->isUncacheable() && pkt->isWrite())) {
1126 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1127 return lat * clockPeriod();
1128 }
1129 // only misses left
1130
1131 PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1132
1133 bool is_forward = (bus_pkt == nullptr);
1134
1135 if (is_forward) {
1136 // just forwarding the same request to the next level
1137 // no local cache operation involved
1138 bus_pkt = pkt;
1139 }
1140
1141 DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1142 bus_pkt->print());
1143
1144#if TRACING_ON
1145 CacheBlk::State old_state = blk ? blk->status : 0;
1146#endif
1147
1148 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1149
1150 bool is_invalidate = bus_pkt->isInvalidate();
1151
1152 // We are now dealing with the response handling
1153 DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1154 bus_pkt->print(), old_state);
1155
1156 // If packet was a forward, the response (if any) is already
1157 // in place in the bus_pkt == pkt structure, so we don't need
1158 // to do anything. Otherwise, use the separate bus_pkt to
1159 // generate response to pkt and then delete it.
1160 if (!is_forward) {
1161 if (pkt->needsResponse()) {
1162 assert(bus_pkt->isResponse());
1163 if (bus_pkt->isError()) {
1164 pkt->makeAtomicResponse();
1165 pkt->copyError(bus_pkt);
1166 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1167 // note the use of pkt, not bus_pkt here.
1168
1169 // write-line request to the cache that promoted
1170 // the write to a whole line
1171 blk = handleFill(pkt, blk, writebacks,
1172 allocOnFill(pkt->cmd));
1173 assert(blk != NULL);
1174 is_invalidate = false;
1175 satisfyRequest(pkt, blk);
1176 } else if (bus_pkt->isRead() ||
1177 bus_pkt->cmd == MemCmd::UpgradeResp) {
1178 // we're updating cache state to allow us to
1179 // satisfy the upstream request from the cache
1180 blk = handleFill(bus_pkt, blk, writebacks,
1181 allocOnFill(pkt->cmd));
1182 satisfyRequest(pkt, blk);
1183 maintainClusivity(pkt->fromCache(), blk);
1184 } else {
1185 // we're satisfying the upstream request without
1186 // modifying cache state, e.g., a write-through
1187 pkt->makeAtomicResponse();
1188 }
1189 }
1190 delete bus_pkt;
1191 }
1192
1193 if (is_invalidate && blk && blk->isValid()) {
1194 invalidateBlock(blk);
1195 }
1196 }
1197
1198 // Note that we don't invoke the prefetcher at all in atomic mode.
1199 // It's not clear how to do it properly, particularly for
1200 // prefetchers that aggressively generate prefetch candidates and
1201 // rely on bandwidth contention to throttle them; these will tend
1202 // to pollute the cache in atomic mode since there is no bandwidth
1203 // contention. If we ever do want to enable prefetching in atomic
1204 // mode, though, this is the place to do it... see timingAccess()
1205 // for an example (though we'd want to issue the prefetch(es)
1206 // immediately rather than calling requestMemSideBus() as we do
1207 // there).
1208
1209 // do any writebacks resulting from the response handling
1210 doWritebacksAtomic(writebacks);
1211
1212 // if we used temp block, check to see if its valid and if so
1213 // clear it out, but only do so after the call to recvAtomic is
1214 // finished so that any downstream observers (such as a snoop
1215 // filter), first see the fill, and only then see the eviction
1216 if (blk == tempBlock && tempBlock->isValid()) {
1217 // the atomic CPU calls recvAtomic for fetch and load/store
1218 // sequentuially, and we may already have a tempBlock
1219 // writeback from the fetch that we have not yet sent
1220 if (tempBlockWriteback) {
1221 // if that is the case, write the prevoius one back, and
1222 // do not schedule any new event
1223 writebackTempBlockAtomic();
1224 } else {
1225 // the writeback/clean eviction happens after the call to
1226 // recvAtomic has finished (but before any successive
1227 // calls), so that the response handling from the fill is
1228 // allowed to happen first
1229 schedule(writebackTempBlockAtomicEvent, curTick());
1230 }
1231
1232 tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1233 writebackBlk(blk) : cleanEvictBlk(blk);
1234 invalidateBlock(blk);
1235 }
1236
1237 if (pkt->needsResponse()) {
1238 pkt->makeAtomicResponse();
1239 }
1240
1241 return lat * clockPeriod();
1242}
1243
1244
1245void
1246Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1247{
1248 if (system->bypassCaches()) {
1249 // Packets from the memory side are snoop request and
1250 // shouldn't happen in bypass mode.
1251 assert(fromCpuSide);
1252
1253 // The cache should be flushed if we are in cache bypass mode,
1254 // so we don't need to check if we need to update anything.
1255 memSidePort->sendFunctional(pkt);
1256 return;
1257 }
1258
1259 Addr blk_addr = pkt->getBlockAddr(blkSize);
1260 bool is_secure = pkt->isSecure();
1261 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1262 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1263
1264 pkt->pushLabel(name());
1265
1266 CacheBlkPrintWrapper cbpw(blk);
1267
1268 // Note that just because an L2/L3 has valid data doesn't mean an
1269 // L1 doesn't have a more up-to-date modified copy that still
1270 // needs to be found. As a result we always update the request if
1271 // we have it, but only declare it satisfied if we are the owner.
1272
1273 // see if we have data at all (owned or otherwise)
1274 bool have_data = blk && blk->isValid()
1275 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1276 blk->data);
1277
1278 // data we have is dirty if marked as such or if we have an
1279 // in-service MSHR that is pending a modified line
1280 bool have_dirty =
1281 have_data && (blk->isDirty() ||
1282 (mshr && mshr->inService && mshr->isPendingModified()));
1283
1284 bool done = have_dirty
1285 || cpuSidePort->checkFunctional(pkt)
1286 || mshrQueue.checkFunctional(pkt, blk_addr)
1287 || writeBuffer.checkFunctional(pkt, blk_addr)
1288 || memSidePort->checkFunctional(pkt);
1289
1290 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
1291 (blk && blk->isValid()) ? "valid " : "",
1292 have_data ? "data " : "", done ? "done " : "");
1293
1294 // We're leaving the cache, so pop cache->name() label
1295 pkt->popLabel();
1296
1297 if (done) {
1298 pkt->makeResponse();
1299 } else {
1300 // if it came as a request from the CPU side then make sure it
1301 // continues towards the memory side
1302 if (fromCpuSide) {
1303 memSidePort->sendFunctional(pkt);
1304 } else if (cpuSidePort->isSnooping()) {
1305 // if it came from the memory side, it must be a snoop request
1306 // and we should only forward it if we are forwarding snoops
1307 cpuSidePort->sendFunctionalSnoop(pkt);
1308 }
1309 }
1310}
1311
1312
1313/////////////////////////////////////////////////////
1314//
1315// Response handling: responses from the memory side
1316//
1317/////////////////////////////////////////////////////
1318
1319
1320void
1321Cache::handleUncacheableWriteResp(PacketPtr pkt)
1322{
1323 Tick completion_time = clockEdge(responseLatency) +
1324 pkt->headerDelay + pkt->payloadDelay;
1325
1326 // Reset the bus additional time as it is now accounted for
1327 pkt->headerDelay = pkt->payloadDelay = 0;
1328
1329 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1330}
1331
1332void
1333Cache::recvTimingResp(PacketPtr pkt)
1334{
1335 assert(pkt->isResponse());
1336
1337 // all header delay should be paid for by the crossbar, unless
1338 // this is a prefetch response from above
1339 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1340 "%s saw a non-zero packet delay\n", name());
1341
1342 bool is_error = pkt->isError();
1343
1344 if (is_error) {
1345 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1346 pkt->print());
1347 }
1348
1349 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1350 pkt->print());
1351
1352 // if this is a write, we should be looking at an uncacheable
1353 // write
1354 if (pkt->isWrite()) {
1355 assert(pkt->req->isUncacheable());
1356 handleUncacheableWriteResp(pkt);
1357 return;
1358 }
1359
1360 // we have dealt with any (uncacheable) writes above, from here on
1361 // we know we are dealing with an MSHR due to a miss or a prefetch
1362 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1363 assert(mshr);
1364
1365 if (mshr == noTargetMSHR) {
1366 // we always clear at least one target
1367 clearBlocked(Blocked_NoTargets);
1368 noTargetMSHR = nullptr;
1369 }
1370
1371 // Initial target is used just for stats
1372 MSHR::Target *initial_tgt = mshr->getTarget();
1373 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1374 Tick miss_latency = curTick() - initial_tgt->recvTime;
1375
1376 if (pkt->req->isUncacheable()) {
1377 assert(pkt->req->masterId() < system->maxMasters());
1378 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1379 miss_latency;
1380 } else {
1381 assert(pkt->req->masterId() < system->maxMasters());
1382 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1383 miss_latency;
1384 }
1385
1386 bool wasFull = mshrQueue.isFull();
1387
1388 PacketList writebacks;
1389
1390 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1391
1392 bool is_fill = !mshr->isForward &&
1393 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1394
1395 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1396 const bool valid_blk = blk && blk->isValid();
1397 // If the response indicates that there are no sharers and we
1398 // either had the block already or the response is filling we can
1399 // promote our copy to writable
1400 if (!pkt->hasSharers() &&
1401 (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1402 mshr->promoteWritable();
1403 }
1404
1405 if (is_fill && !is_error) {
1406 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1407 pkt->getAddr());
1408
1409 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1410 assert(blk != nullptr);
1411 }
1412
1413 // allow invalidation responses originating from write-line
1414 // requests to be discarded
1415 bool is_invalidate = pkt->isInvalidate();
1416
1417 // The block was marked as not readable while there was a pending
1418 // cache maintenance operation, restore its flag.
1419 if (pkt->isClean() && !is_invalidate && valid_blk) {
1420 blk->status |= BlkReadable;
1421 }
1422
1423 // First offset for critical word first calculations
1424 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1425
1426 bool from_cache = false;
1427 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1428 for (auto &target: targets) {
1429 Packet *tgt_pkt = target.pkt;
1430 switch (target.source) {
1431 case MSHR::Target::FromCPU:
1432 Tick completion_time;
1433 // Here we charge on completion_time the delay of the xbar if the
1434 // packet comes from it, charged on headerDelay.
1435 completion_time = pkt->headerDelay;
1436
1437 // Software prefetch handling for cache closest to core
1438 if (tgt_pkt->cmd.isSWPrefetch()) {
1439 // a software prefetch would have already been ack'd
1440 // immediately with dummy data so the core would be able to
1441 // retire it. This request completes right here, so we
1442 // deallocate it.
1443 delete tgt_pkt->req;
1444 delete tgt_pkt;
1445 break; // skip response
1446 }
1447
1448 // keep track of whether we have responded to another
1449 // cache
1450 from_cache = from_cache || tgt_pkt->fromCache();
1451
1452 // unlike the other packet flows, where data is found in other
1453 // caches or memory and brought back, write-line requests always
1454 // have the data right away, so the above check for "is fill?"
1455 // cannot actually be determined until examining the stored MSHR
1456 // state. We "catch up" with that logic here, which is duplicated
1457 // from above.
1458 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1459 assert(!is_error);
1460 // we got the block in a writable state, so promote
1461 // any deferred targets if possible
1462 mshr->promoteWritable();
1463 // NB: we use the original packet here and not the response!
1464 blk = handleFill(tgt_pkt, blk, writebacks,
1465 targets.allocOnFill);
1466 assert(blk != nullptr);
1467
1468 // treat as a fill, and discard the invalidation
1469 // response
1470 is_fill = true;
1471 is_invalidate = false;
1472 }
1473
1474 if (is_fill) {
1475 satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1476
1477 // How many bytes past the first request is this one
1478 int transfer_offset =
1479 tgt_pkt->getOffset(blkSize) - initial_offset;
1480 if (transfer_offset < 0) {
1481 transfer_offset += blkSize;
1482 }
1483
1484 // If not critical word (offset) return payloadDelay.
1485 // responseLatency is the latency of the return path
1486 // from lower level caches/memory to an upper level cache or
1487 // the core.
1488 completion_time += clockEdge(responseLatency) +
1489 (transfer_offset ? pkt->payloadDelay : 0);
1490
1491 assert(!tgt_pkt->req->isUncacheable());
1492
1493 assert(tgt_pkt->req->masterId() < system->maxMasters());
1494 missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1495 completion_time - target.recvTime;
1496 } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1497 // failed StoreCond upgrade
1498 assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1499 tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1500 tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1501 // responseLatency is the latency of the return path
1502 // from lower level caches/memory to an upper level cache or
1503 // the core.
1504 completion_time += clockEdge(responseLatency) +
1505 pkt->payloadDelay;
1506 tgt_pkt->req->setExtraData(0);
1507 } else {
1508 // We are about to send a response to a cache above
1509 // that asked for an invalidation; we need to
1510 // invalidate our copy immediately as the most
1511 // up-to-date copy of the block will now be in the
1512 // cache above. It will also prevent this cache from
1513 // responding (if the block was previously dirty) to
1514 // snoops as they should snoop the caches above where
1515 // they will get the response from.
1516 if (is_invalidate && blk && blk->isValid()) {
1517 invalidateBlock(blk);
1518 }
1519 // not a cache fill, just forwarding response
1520 // responseLatency is the latency of the return path
1521 // from lower level cahces/memory to the core.
1522 completion_time += clockEdge(responseLatency) +
1523 pkt->payloadDelay;
1524 if (pkt->isRead() && !is_error) {
1525 // sanity check
1526 assert(pkt->getAddr() == tgt_pkt->getAddr());
1527 assert(pkt->getSize() >= tgt_pkt->getSize());
1528
1529 tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1530 }
1531 }
1532 tgt_pkt->makeTimingResponse();
1533 // if this packet is an error copy that to the new packet
1534 if (is_error)
1535 tgt_pkt->copyError(pkt);
1536 if (tgt_pkt->cmd == MemCmd::ReadResp &&
1537 (is_invalidate || mshr->hasPostInvalidate())) {
1538 // If intermediate cache got ReadRespWithInvalidate,
1539 // propagate that. Response should not have
1540 // isInvalidate() set otherwise.
1541 tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1542 DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1543 tgt_pkt->print());
1544 }
1545 // Reset the bus additional time as it is now accounted for
1546 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1547 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1548 break;
1549
1550 case MSHR::Target::FromPrefetcher:
1551 assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1552 if (blk)
1553 blk->status |= BlkHWPrefetched;
1554 delete tgt_pkt->req;
1555 delete tgt_pkt;
1556 break;
1557
1558 case MSHR::Target::FromSnoop:
1559 // I don't believe that a snoop can be in an error state
1560 assert(!is_error);
1561 // response to snoop request
1562 DPRINTF(Cache, "processing deferred snoop...\n");
1563 // If the response is invalidating, a snooping target can
1564 // be satisfied if it is also invalidating. If the reponse is, not
1565 // only invalidating, but more specifically an InvalidateResp and
1566 // the MSHR was created due to an InvalidateReq then a cache above
1567 // is waiting to satisfy a WriteLineReq. In this case even an
1568 // non-invalidating snoop is added as a target here since this is
1569 // the ordering point. When the InvalidateResp reaches this cache,
1570 // the snooping target will snoop further the cache above with the
1571 // WriteLineReq.
1572 assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
1573 pkt->req->isCacheMaintenance() ||
1574 mshr->hasPostInvalidate());
1575 handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1576 break;
1577
1578 default:
1579 panic("Illegal target->source enum %d\n", target.source);
1580 }
1581 }
1582
1583 maintainClusivity(from_cache, blk);
1584
1585 if (blk && blk->isValid()) {
1586 // an invalidate response stemming from a write line request
1587 // should not invalidate the block, so check if the
1588 // invalidation should be discarded
1589 if (is_invalidate || mshr->hasPostInvalidate()) {
1590 invalidateBlock(blk);
1591 } else if (mshr->hasPostDowngrade()) {
1592 blk->status &= ~BlkWritable;
1593 }
1594 }
1595
1596 if (mshr->promoteDeferredTargets()) {
1597 // avoid later read getting stale data while write miss is
1598 // outstanding.. see comment in timingAccess()
1599 if (blk) {
1600 blk->status &= ~BlkReadable;
1601 }
1602 mshrQueue.markPending(mshr);
1603 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1604 } else {
1605 mshrQueue.deallocate(mshr);
1606 if (wasFull && !mshrQueue.isFull()) {
1607 clearBlocked(Blocked_NoMSHRs);
1608 }
1609
1610 // Request the bus for a prefetch if this deallocation freed enough
1611 // MSHRs for a prefetch to take place
1612 if (prefetcher && mshrQueue.canPrefetch()) {
1613 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1614 clockEdge());
1615 if (next_pf_time != MaxTick)
1616 schedMemSideSendEvent(next_pf_time);
1617 }
1618 }
1619 // reset the xbar additional timinig as it is now accounted for
1620 pkt->headerDelay = pkt->payloadDelay = 0;
1621
1622 // copy writebacks to write buffer
1623 doWritebacks(writebacks, forward_time);
1624
1625 // if we used temp block, check to see if its valid and then clear it out
1626 if (blk == tempBlock && tempBlock->isValid()) {
1627 // We use forwardLatency here because we are copying
1628 // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1629 // allocate an internal buffer and to schedule an event to the
1630 // queued port.
1631 if (blk->isDirty() || writebackClean) {
1632 PacketPtr wbPkt = writebackBlk(blk);
1633 allocateWriteBuffer(wbPkt, forward_time);
1634 // Set BLOCK_CACHED flag if cached above.
1635 if (isCachedAbove(wbPkt))
1636 wbPkt->setBlockCached();
1637 } else {
1638 PacketPtr wcPkt = cleanEvictBlk(blk);
1639 // Check to see if block is cached above. If not allocate
1640 // write buffer
1641 if (isCachedAbove(wcPkt))
1642 delete wcPkt;
1643 else
1644 allocateWriteBuffer(wcPkt, forward_time);
1645 }
1646 invalidateBlock(blk);
1647 }
1648
1649 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1650 delete pkt;
1651}
1652
1653PacketPtr
1654Cache::writebackBlk(CacheBlk *blk)
1655{
1656 chatty_assert(!isReadOnly || writebackClean,
1657 "Writeback from read-only cache");
1658 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1659
1660 writebacks[Request::wbMasterId]++;
1661
1662 Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1663 Request::wbMasterId);
1664 if (blk->isSecure())
1665 req->setFlags(Request::SECURE);
1666
1667 req->taskId(blk->task_id);
1668
1669 PacketPtr pkt =
1670 new Packet(req, blk->isDirty() ?
1671 MemCmd::WritebackDirty : MemCmd::WritebackClean);
1672
1673 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1674 pkt->print(), blk->isWritable(), blk->isDirty());
1675
1676 if (blk->isWritable()) {
1677 // not asserting shared means we pass the block in modified
1678 // state, mark our own block non-writeable
1679 blk->status &= ~BlkWritable;
1680 } else {
1681 // we are in the Owned state, tell the receiver
1682 pkt->setHasSharers();
1683 }
1684
1685 // make sure the block is not marked dirty
1686 blk->status &= ~BlkDirty;
1687
1688 pkt->allocate();
1689 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1690
1691 return pkt;
1692}
1693
1694PacketPtr
1695Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1696{
1697 Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1698 Request::wbMasterId);
1699 if (blk->isSecure()) {
1700 req->setFlags(Request::SECURE);
1701 }
1702 req->taskId(blk->task_id);
1703
1704 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1705
1706 if (dest) {
1707 req->setFlags(dest);
1708 pkt->setWriteThrough();
1709 }
1710
1711 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1712 blk->isWritable(), blk->isDirty());
1713
1714 if (blk->isWritable()) {
1715 // not asserting shared means we pass the block in modified
1716 // state, mark our own block non-writeable
1717 blk->status &= ~BlkWritable;
1718 } else {
1719 // we are in the Owned state, tell the receiver
1720 pkt->setHasSharers();
1721 }
1722
1723 // make sure the block is not marked dirty
1724 blk->status &= ~BlkDirty;
1725
1726 pkt->allocate();
1727 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1728
1729 return pkt;
1730}
1731
1732
1733PacketPtr
1734Cache::cleanEvictBlk(CacheBlk *blk)
1735{
1736 assert(!writebackClean);
1737 assert(blk && blk->isValid() && !blk->isDirty());
1738 // Creating a zero sized write, a message to the snoop filter
1739 Request *req =
1740 new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1741 Request::wbMasterId);
1742 if (blk->isSecure())
1743 req->setFlags(Request::SECURE);
1744
1745 req->taskId(blk->task_id);
1746
1747 PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1748 pkt->allocate();
1749 DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1750
1751 return pkt;
1752}
1753
1754void
1755Cache::memWriteback()
1756{
1757 CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1758 tags->forEachBlk(visitor);
1759}
1760
1761void
1762Cache::memInvalidate()
1763{
1764 CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1765 tags->forEachBlk(visitor);
1766}
1767
1768bool
1769Cache::isDirty() const
1770{
1771 CacheBlkIsDirtyVisitor visitor;
1772 tags->forEachBlk(visitor);
1773
1774 return visitor.isDirty();
1775}
1776
1777bool
1778Cache::writebackVisitor(CacheBlk &blk)
1779{
1780 if (blk.isDirty()) {
1781 assert(blk.isValid());
1782
1783 Request request(tags->regenerateBlkAddr(&blk), blkSize, 0,
1784 Request::funcMasterId);
1785 request.taskId(blk.task_id);
1786 if (blk.isSecure()) {
1787 request.setFlags(Request::SECURE);
1788 }
1789
1790 Packet packet(&request, MemCmd::WriteReq);
1791 packet.dataStatic(blk.data);
1792
1793 memSidePort->sendFunctional(&packet);
1794
1795 blk.status &= ~BlkDirty;
1796 }
1797
1798 return true;
1799}
1800
1801bool
1802Cache::invalidateVisitor(CacheBlk &blk)
1803{
1804
1805 if (blk.isDirty())
1806 warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1807
1808 if (blk.isValid()) {
1809 assert(!blk.isDirty());
1810 invalidateBlock(&blk);
1811 }
1812
1813 return true;
1814}
1815
1816CacheBlk*
1817Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1818{
1819 // Find replacement victim
1820 CacheBlk *blk = tags->findVictim(addr);
1821
1822 // It is valid to return nullptr if there is no victim
1823 if (!blk)
1824 return nullptr;
1825
1826 if (blk->isValid()) {
1827 Addr repl_addr = tags->regenerateBlkAddr(blk);
1828 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1829 if (repl_mshr) {
1830 // must be an outstanding upgrade or clean request
1831 // on a block we're about to replace...
1832 assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
1833 repl_mshr->isCleaning());
1834 // too hard to replace block with transient state
1835 // allocation failed, block not inserted
1836 return nullptr;
1837 } else {
1838 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1839 "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1840 addr, is_secure ? "s" : "ns",
1841 blk->isDirty() ? "writeback" : "clean");
1842
1843 if (blk->wasPrefetched()) {
1844 unusedPrefetches++;
1845 }
1846 // Will send up Writeback/CleanEvict snoops via isCachedAbove
1847 // when pushing this writeback list into the write buffer.
1848 if (blk->isDirty() || writebackClean) {
1849 // Save writeback packet for handling by caller
1850 writebacks.push_back(writebackBlk(blk));
1851 } else {
1852 writebacks.push_back(cleanEvictBlk(blk));
1853 }
1854 }
1855 }
1856
1857 return blk;
1858}
1859
1860void
1861Cache::invalidateBlock(CacheBlk *blk)
1862{
1863 if (blk != tempBlock)
1864 tags->invalidate(blk);
1865 blk->invalidate();
1866}
1867
1868// Note that the reason we return a list of writebacks rather than
1869// inserting them directly in the write buffer is that this function
1870// is called by both atomic and timing-mode accesses, and in atomic
1871// mode we don't mess with the write buffer (we just perform the
1872// writebacks atomically once the original request is complete).
1873CacheBlk*
1874Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1875 bool allocate)
1876{
1877 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1878 Addr addr = pkt->getAddr();
1879 bool is_secure = pkt->isSecure();
1880#if TRACING_ON
1881 CacheBlk::State old_state = blk ? blk->status : 0;
1882#endif
1883
1884 // When handling a fill, we should have no writes to this line.
1885 assert(addr == pkt->getBlockAddr(blkSize));
1886 assert(!writeBuffer.findMatch(addr, is_secure));
1887
1888 if (blk == nullptr) {
1889 // better have read new data...
1890 assert(pkt->hasData());
1891
1892 // only read responses and write-line requests have data;
1893 // note that we don't write the data here for write-line - that
1894 // happens in the subsequent call to satisfyRequest
1895 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1896
1897 // need to do a replacement if allocating, otherwise we stick
1898 // with the temporary storage
1899 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1900
1901 if (blk == nullptr) {
1902 // No replaceable block or a mostly exclusive
1903 // cache... just use temporary storage to complete the
1904 // current request and then get rid of it
1905 assert(!tempBlock->isValid());
1906 blk = tempBlock;
1907 tempBlock->set = tags->extractSet(addr);
1908 tempBlock->tag = tags->extractTag(addr);
1909 if (is_secure) {
1910 tempBlock->status |= BlkSecure;
1911 }
1912 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1913 is_secure ? "s" : "ns");
1914 } else {
1915 tags->insertBlock(pkt, blk);
1916 }
1917
1918 // we should never be overwriting a valid block
1919 assert(!blk->isValid());
1920 } else {
1921 // existing block... probably an upgrade
1922 assert(blk->tag == tags->extractTag(addr));
1923 // either we're getting new data or the block should already be valid
1924 assert(pkt->hasData() || blk->isValid());
1925 // don't clear block status... if block is already dirty we
1926 // don't want to lose that
1927 }
1928
1929 if (is_secure)
1930 blk->status |= BlkSecure;
1931 blk->status |= BlkValid | BlkReadable;
1932
1933 // sanity check for whole-line writes, which should always be
1934 // marked as writable as part of the fill, and then later marked
1935 // dirty as part of satisfyRequest
1936 if (pkt->cmd == MemCmd::WriteLineReq) {
1937 assert(!pkt->hasSharers());
1938 }
1939
1940 // here we deal with setting the appropriate state of the line,
1941 // and we start by looking at the hasSharers flag, and ignore the
1942 // cacheResponding flag (normally signalling dirty data) if the
1943 // packet has sharers, thus the line is never allocated as Owned
1944 // (dirty but not writable), and always ends up being either
1945 // Shared, Exclusive or Modified, see Packet::setCacheResponding
1946 // for more details
1947 if (!pkt->hasSharers()) {
1948 // we could get a writable line from memory (rather than a
1949 // cache) even in a read-only cache, note that we set this bit
1950 // even for a read-only cache, possibly revisit this decision
1951 blk->status |= BlkWritable;
1952
1953 // check if we got this via cache-to-cache transfer (i.e., from a
1954 // cache that had the block in Modified or Owned state)
1955 if (pkt->cacheResponding()) {
1956 // we got the block in Modified state, and invalidated the
1957 // owners copy
1958 blk->status |= BlkDirty;
1959
1960 chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1961 "in read-only cache %s\n", name());
1962 }
1963 }
1964
1965 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1966 addr, is_secure ? "s" : "ns", old_state, blk->print());
1967
1968 // if we got new data, copy it in (checking for a read response
1969 // and a response that has data is the same in the end)
1970 if (pkt->isRead()) {
1971 // sanity checks
1972 assert(pkt->hasData());
1973 assert(pkt->getSize() == blkSize);
1974
1975 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
1976 }
1977 // We pay for fillLatency here.
1978 blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1979 pkt->payloadDelay;
1980
1981 return blk;
1982}
1983
1984
1985/////////////////////////////////////////////////////
1986//
1987// Snoop path: requests coming in from the memory side
1988//
1989/////////////////////////////////////////////////////
1990
1991void
1992Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1993 bool already_copied, bool pending_inval)
1994{
1995 // sanity check
1996 assert(req_pkt->isRequest());
1997 assert(req_pkt->needsResponse());
1998
1999 DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
2000 // timing-mode snoop responses require a new packet, unless we
2001 // already made a copy...
2002 PacketPtr pkt = req_pkt;
2003 if (!already_copied)
2004 // do not clear flags, and allocate space for data if the
2005 // packet needs it (the only packets that carry data are read
2006 // responses)
2007 pkt = new Packet(req_pkt, false, req_pkt->isRead());
2008
2009 assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
2010 pkt->hasSharers());
2011 pkt->makeTimingResponse();
2012 if (pkt->isRead()) {
2013 pkt->setDataFromBlock(blk_data, blkSize);
2014 }
2015 if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
2016 // Assume we defer a response to a read from a far-away cache
2017 // A, then later defer a ReadExcl from a cache B on the same
2018 // bus as us. We'll assert cacheResponding in both cases, but
2019 // in the latter case cacheResponding will keep the
2020 // invalidation from reaching cache A. This special response
2021 // tells cache A that it gets the block to satisfy its read,
2022 // but must immediately invalidate it.
2023 pkt->cmd = MemCmd::ReadRespWithInvalidate;
2024 }
2025 // Here we consider forward_time, paying for just forward latency and
2026 // also charging the delay provided by the xbar.
2027 // forward_time is used as send_time in next allocateWriteBuffer().
2028 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
2029 // Here we reset the timing of the packet.
2030 pkt->headerDelay = pkt->payloadDelay = 0;
2031 DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
2032 pkt->print(), forward_time);
2033 memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
2034}
2035
2036uint32_t
2037Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
2038 bool is_deferred, bool pending_inval)
2039{
2040 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2041 // deferred snoops can only happen in timing mode
2042 assert(!(is_deferred && !is_timing));
2043 // pending_inval only makes sense on deferred snoops
2044 assert(!(pending_inval && !is_deferred));
2045 assert(pkt->isRequest());
2046
2047 // the packet may get modified if we or a forwarded snooper
2048 // responds in atomic mode, so remember a few things about the
2049 // original packet up front
2050 bool invalidate = pkt->isInvalidate();
2051 bool M5_VAR_USED needs_writable = pkt->needsWritable();
2052
2053 // at the moment we could get an uncacheable write which does not
2054 // have the invalidate flag, and we need a suitable way of dealing
2055 // with this case
2056 panic_if(invalidate && pkt->req->isUncacheable(),
2057 "%s got an invalidating uncacheable snoop request %s",
2058 name(), pkt->print());
2059
2060 uint32_t snoop_delay = 0;
2061
2062 if (forwardSnoops) {
2063 // first propagate snoop upward to see if anyone above us wants to
2064 // handle it. save & restore packet src since it will get
2065 // rewritten to be relative to cpu-side bus (if any)
2066 bool alreadyResponded = pkt->cacheResponding();
2067 if (is_timing) {
2068 // copy the packet so that we can clear any flags before
2069 // forwarding it upwards, we also allocate data (passing
2070 // the pointer along in case of static data), in case
2071 // there is a snoop hit in upper levels
2072 Packet snoopPkt(pkt, true, true);
2073 snoopPkt.setExpressSnoop();
2074 // the snoop packet does not need to wait any additional
2075 // time
2076 snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2077 cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2078
2079 // add the header delay (including crossbar and snoop
2080 // delays) of the upward snoop to the snoop delay for this
2081 // cache
2082 snoop_delay += snoopPkt.headerDelay;
2083
2084 if (snoopPkt.cacheResponding()) {
2085 // cache-to-cache response from some upper cache
2086 assert(!alreadyResponded);
2087 pkt->setCacheResponding();
2088 }
2089 // upstream cache has the block, or has an outstanding
2090 // MSHR, pass the flag on
2091 if (snoopPkt.hasSharers()) {
2092 pkt->setHasSharers();
2093 }
2094 // If this request is a prefetch or clean evict and an upper level
2095 // signals block present, make sure to propagate the block
2096 // presence to the requester.
2097 if (snoopPkt.isBlockCached()) {
2098 pkt->setBlockCached();
2099 }
2100 // If the request was satisfied by snooping the cache
2101 // above, mark the original packet as satisfied too.
2102 if (snoopPkt.satisfied()) {
2103 pkt->setSatisfied();
2104 }
2105 } else {
2106 cpuSidePort->sendAtomicSnoop(pkt);
2107 if (!alreadyResponded && pkt->cacheResponding()) {
2108 // cache-to-cache response from some upper cache:
2109 // forward response to original requester
2110 assert(pkt->isResponse());
2111 }
2112 }
2113 }
2114
2115 bool respond = false;
2116 bool blk_valid = blk && blk->isValid();
2117 if (pkt->isClean()) {
2118 if (blk_valid && blk->isDirty()) {
2119 DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
2120 __func__, pkt->print(), blk->print());
2121 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
2122 PacketList writebacks;
2123 writebacks.push_back(wb_pkt);
2124
2125 if (is_timing) {
2126 // anything that is merely forwarded pays for the forward
2127 // latency and the delay provided by the crossbar
2128 Tick forward_time = clockEdge(forwardLatency) +
2129 pkt->headerDelay;
2130 doWritebacks(writebacks, forward_time);
2131 } else {
2132 doWritebacksAtomic(writebacks);
2133 }
2134 pkt->setSatisfied();
2135 }
2136 } else if (!blk_valid) {
2137 DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2138 pkt->print());
2139 if (is_deferred) {
2140 // we no longer have the block, and will not respond, but a
2141 // packet was allocated in MSHR::handleSnoop and we have
2142 // to delete it
2143 assert(pkt->needsResponse());
2144
2145 // we have passed the block to a cache upstream, that
2146 // cache should be responding
2147 assert(pkt->cacheResponding());
2148
2149 delete pkt;
2150 }
2151 return snoop_delay;
2152 } else {
2153 DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2154 pkt->print(), blk->print());
2155
2156 // We may end up modifying both the block state and the packet (if
2157 // we respond in atomic mode), so just figure out what to do now
2158 // and then do it later. We respond to all snoops that need
2159 // responses provided we have the block in dirty state. The
2160 // invalidation itself is taken care of below. We don't respond to
2161 // cache maintenance operations as this is done by the destination
2162 // xbar.
2163 respond = blk->isDirty() && pkt->needsResponse();
2164
2165 chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
2166 "a dirty block in a read-only cache %s\n", name());
2167 }
2168
2169 // Invalidate any prefetch's from below that would strip write permissions
2170 // MemCmd::HardPFReq is only observed by upstream caches. After missing
2171 // above and in it's own cache, a new MemCmd::ReadReq is created that
2172 // downstream caches observe.
2173 if (pkt->mustCheckAbove()) {
2174 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2175 "from lower cache\n", pkt->getAddr(), pkt->print());
2176 pkt->setBlockCached();
2177 return snoop_delay;
2178 }
2179
2180 if (pkt->isRead() && !invalidate) {
2181 // reading without requiring the line in a writable state
2182 assert(!needs_writable);
2183 pkt->setHasSharers();
2184
2185 // if the requesting packet is uncacheable, retain the line in
2186 // the current state, otherwhise unset the writable flag,
2187 // which means we go from Modified to Owned (and will respond
2188 // below), remain in Owned (and will respond below), from
2189 // Exclusive to Shared, or remain in Shared
2190 if (!pkt->req->isUncacheable())
2191 blk->status &= ~BlkWritable;
2192 DPRINTF(Cache, "new state is %s\n", blk->print());
2193 }
2194
2195 if (respond) {
2196 // prevent anyone else from responding, cache as well as
2197 // memory, and also prevent any memory from even seeing the
2198 // request
2199 pkt->setCacheResponding();
2200 if (!pkt->isClean() && blk->isWritable()) {
2201 // inform the cache hierarchy that this cache had the line
2202 // in the Modified state so that we avoid unnecessary
2203 // invalidations (see Packet::setResponderHadWritable)
2204 pkt->setResponderHadWritable();
2205
2206 // in the case of an uncacheable request there is no point
2207 // in setting the responderHadWritable flag, but since the
2208 // recipient does not care there is no harm in doing so
2209 } else {
2210 // if the packet has needsWritable set we invalidate our
2211 // copy below and all other copies will be invalidates
2212 // through express snoops, and if needsWritable is not set
2213 // we already called setHasSharers above
2214 }
2215
2216 // if we are returning a writable and dirty (Modified) line,
2217 // we should be invalidating the line
2218 panic_if(!invalidate && !pkt->hasSharers(),
2219 "%s is passing a Modified line through %s, "
2220 "but keeping the block", name(), pkt->print());
2221
2222 if (is_timing) {
2223 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2224 } else {
2225 pkt->makeAtomicResponse();
2226 // packets such as upgrades do not actually have any data
2227 // payload
2228 if (pkt->hasData())
2229 pkt->setDataFromBlock(blk->data, blkSize);
2230 }
2231 }
2232
2233 if (!respond && is_deferred) {
2234 assert(pkt->needsResponse());
2235
2236 // if we copied the deferred packet with the intention to
2237 // respond, but are not responding, then a cache above us must
2238 // be, and we can use this as the indication of whether this
2239 // is a packet where we created a copy of the request or not
2240 if (!pkt->cacheResponding()) {
2241 delete pkt->req;
2242 }
2243
2244 delete pkt;
2245 }
2246
2247 // Do this last in case it deallocates block data or something
2248 // like that
2249 if (blk_valid && invalidate) {
2250 invalidateBlock(blk);
2251 DPRINTF(Cache, "new state is %s\n", blk->print());
2252 }
2253
2254 return snoop_delay;
2255}
2256
2257
2258void
2259Cache::recvTimingSnoopReq(PacketPtr pkt)
2260{
2261 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2262
2263 // Snoops shouldn't happen when bypassing caches
2264 assert(!system->bypassCaches());
2265
2266 // no need to snoop requests that are not in range
2267 if (!inRange(pkt->getAddr())) {
2268 return;
2269 }
2270
2271 bool is_secure = pkt->isSecure();
2272 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2273
2274 Addr blk_addr = pkt->getBlockAddr(blkSize);
2275 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2276
2277 // Update the latency cost of the snoop so that the crossbar can
2278 // account for it. Do not overwrite what other neighbouring caches
2279 // have already done, rather take the maximum. The update is
2280 // tentative, for cases where we return before an upward snoop
2281 // happens below.
2282 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2283 lookupLatency * clockPeriod());
2284
2285 // Inform request(Prefetch, CleanEvict or Writeback) from below of
2286 // MSHR hit, set setBlockCached.
2287 if (mshr && pkt->mustCheckAbove()) {
2288 DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2289 "mshr hit\n", pkt->print());
2290 pkt->setBlockCached();
2291 return;
2292 }
2293
2294 // Bypass any existing cache maintenance requests if the request
2295 // has been satisfied already (i.e., the dirty block has been
2296 // found).
2297 if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
2298 return;
2299 }
2300
2301 // Let the MSHR itself track the snoop and decide whether we want
2302 // to go ahead and do the regular cache snoop
2303 if (mshr && mshr->handleSnoop(pkt, order++)) {
2304 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2305 "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2306 mshr->print());
2307
2308 if (mshr->getNumTargets() > numTarget)
2309 warn("allocating bonus target for snoop"); //handle later
2310 return;
2311 }
2312
2313 //We also need to check the writeback buffers and handle those
2314 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2315 if (wb_entry) {
2316 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2317 pkt->getAddr(), is_secure ? "s" : "ns");
2318 // Expect to see only Writebacks and/or CleanEvicts here, both of
2319 // which should not be generated for uncacheable data.
2320 assert(!wb_entry->isUncacheable());
2321 // There should only be a single request responsible for generating
2322 // Writebacks/CleanEvicts.
2323 assert(wb_entry->getNumTargets() == 1);
2324 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2325 assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2326
2327 if (pkt->isEviction()) {
2328 // if the block is found in the write queue, set the BLOCK_CACHED
2329 // flag for Writeback/CleanEvict snoop. On return the snoop will
2330 // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2331 // any CleanEvicts from travelling down the memory hierarchy.
2332 pkt->setBlockCached();
2333 DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2334 "hit\n", __func__, pkt->print());
2335 return;
2336 }
2337
2338 // conceptually writebacks are no different to other blocks in
2339 // this cache, so the behaviour is modelled after handleSnoop,
2340 // the difference being that instead of querying the block
2341 // state to determine if it is dirty and writable, we use the
2342 // command and fields of the writeback packet
2343 bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2344 pkt->needsResponse();
2345 bool have_writable = !wb_pkt->hasSharers();
2346 bool invalidate = pkt->isInvalidate();
2347
2348 if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2349 assert(!pkt->needsWritable());
2350 pkt->setHasSharers();
2351 wb_pkt->setHasSharers();
2352 }
2353
2354 if (respond) {
2355 pkt->setCacheResponding();
2356
2357 if (have_writable) {
2358 pkt->setResponderHadWritable();
2359 }
2360
2361 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2362 false, false);
2363 }
2364
2365 if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2366 // Invalidation trumps our writeback... discard here
2367 // Note: markInService will remove entry from writeback buffer.
2368 markInService(wb_entry);
2369 delete wb_pkt;
2370 }
2371 }
2372
2373 // If this was a shared writeback, there may still be
2374 // other shared copies above that require invalidation.
2375 // We could be more selective and return here if the
2376 // request is non-exclusive or if the writeback is
2377 // exclusive.
2378 uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2379
2380 // Override what we did when we first saw the snoop, as we now
2381 // also have the cost of the upwards snoops to account for
2382 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2383 lookupLatency * clockPeriod());
2384}
2385
2386bool
2387Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2388{
2389 // Express snoop responses from master to slave, e.g., from L1 to L2
2390 cache->recvTimingSnoopResp(pkt);
2391 return true;
2392}
2393
2394Tick
2395Cache::recvAtomicSnoop(PacketPtr pkt)
2396{
2397 // Snoops shouldn't happen when bypassing caches
2398 assert(!system->bypassCaches());
2399
2400 // no need to snoop requests that are not in range.
2401 if (!inRange(pkt->getAddr())) {
2402 return 0;
2403 }
2404
2405 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2406 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2407 return snoop_delay + lookupLatency * clockPeriod();
2408}
2409
2410
2411QueueEntry*
2412Cache::getNextQueueEntry()
2413{
2414 // Check both MSHR queue and write buffer for potential requests,
2415 // note that null does not mean there is no request, it could
2416 // simply be that it is not ready
2417 MSHR *miss_mshr = mshrQueue.getNext();
2418 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2419
2420 // If we got a write buffer request ready, first priority is a
2421 // full write buffer, otherwise we favour the miss requests
2422 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2423 // need to search MSHR queue for conflicting earlier miss.
2424 MSHR *conflict_mshr =
2425 mshrQueue.findPending(wq_entry->blkAddr,
2426 wq_entry->isSecure);
2427
2428 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2429 // Service misses in order until conflict is cleared.
2430 return conflict_mshr;
2431
2432 // @todo Note that we ignore the ready time of the conflict here
2433 }
2434
2435 // No conflicts; issue write
2436 return wq_entry;
2437 } else if (miss_mshr) {
2438 // need to check for conflicting earlier writeback
2439 WriteQueueEntry *conflict_mshr =
2440 writeBuffer.findPending(miss_mshr->blkAddr,
2441 miss_mshr->isSecure);
2442 if (conflict_mshr) {
2443 // not sure why we don't check order here... it was in the
2444 // original code but commented out.
2445
2446 // The only way this happens is if we are
2447 // doing a write and we didn't have permissions
2448 // then subsequently saw a writeback (owned got evicted)
2449 // We need to make sure to perform the writeback first
2450 // To preserve the dirty data, then we can issue the write
2451
2452 // should we return wq_entry here instead? I.e. do we
2453 // have to flush writes in order? I don't think so... not
2454 // for Alpha anyway. Maybe for x86?
2455 return conflict_mshr;
2456
2457 // @todo Note that we ignore the ready time of the conflict here
2458 }
2459
2460 // No conflicts; issue read
2461 return miss_mshr;
2462 }
2463
2464 // fall through... no pending requests. Try a prefetch.
2465 assert(!miss_mshr && !wq_entry);
2466 if (prefetcher && mshrQueue.canPrefetch()) {
2467 // If we have a miss queue slot, we can try a prefetch
2468 PacketPtr pkt = prefetcher->getPacket();
2469 if (pkt) {
2470 Addr pf_addr = pkt->getBlockAddr(blkSize);
2471 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2472 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2473 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2474 // Update statistic on number of prefetches issued
2475 // (hwpf_mshr_misses)
2476 assert(pkt->req->masterId() < system->maxMasters());
2477 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2478
2479 // allocate an MSHR and return it, note
2480 // that we send the packet straight away, so do not
2481 // schedule the send
2482 return allocateMissBuffer(pkt, curTick(), false);
2483 } else {
2484 // free the request and packet
2485 delete pkt->req;
2486 delete pkt;
2487 }
2488 }
2489 }
2490
2491 return nullptr;
2492}
2493
2494bool
2495Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2496{
2497 if (!forwardSnoops)
2498 return false;
2499 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2500 // Writeback snoops into upper level caches to check for copies of the
2501 // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2502 // packet, the cache can inform the crossbar below of presence or absence
2503 // of the block.
2504 if (is_timing) {
2505 Packet snoop_pkt(pkt, true, false);
2506 snoop_pkt.setExpressSnoop();
2507 // Assert that packet is either Writeback or CleanEvict and not a
2508 // prefetch request because prefetch requests need an MSHR and may
2509 // generate a snoop response.
2510 assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2511 snoop_pkt.senderState = nullptr;
2512 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2513 // Writeback/CleanEvict snoops do not generate a snoop response.
2514 assert(!(snoop_pkt.cacheResponding()));
2515 return snoop_pkt.isBlockCached();
2516 } else {
2517 cpuSidePort->sendAtomicSnoop(pkt);
2518 return pkt->isBlockCached();
2519 }
2520}
2521
2522Tick
2523Cache::nextQueueReadyTime() const
2524{
2525 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2526 writeBuffer.nextReadyTime());
2527
2528 // Don't signal prefetch ready time if no MSHRs available
2529 // Will signal once enoguh MSHRs are deallocated
2530 if (prefetcher && mshrQueue.canPrefetch()) {
2531 nextReady = std::min(nextReady,
2532 prefetcher->nextPrefetchReadyTime());
2533 }
2534
2535 return nextReady;
2536}
2537
2538bool
2539Cache::sendMSHRQueuePacket(MSHR* mshr)
2540{
2541 assert(mshr);
2542
2543 // use request from 1st target
2544 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2545
2546 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2547
2548 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2549
2550 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2551 // we should never have hardware prefetches to allocated
2552 // blocks
2553 assert(blk == nullptr);
2554
2555 // We need to check the caches above us to verify that
2556 // they don't have a copy of this block in the dirty state
2557 // at the moment. Without this check we could get a stale
2558 // copy from memory that might get used in place of the
2559 // dirty one.
2560 Packet snoop_pkt(tgt_pkt, true, false);
2561 snoop_pkt.setExpressSnoop();
2562 // We are sending this packet upwards, but if it hits we will
2563 // get a snoop response that we end up treating just like a
2564 // normal response, hence it needs the MSHR as its sender
2565 // state
2566 snoop_pkt.senderState = mshr;
2567 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2568
2569 // Check to see if the prefetch was squashed by an upper cache (to
2570 // prevent us from grabbing the line) or if a Check to see if a
2571 // writeback arrived between the time the prefetch was placed in
2572 // the MSHRs and when it was selected to be sent or if the
2573 // prefetch was squashed by an upper cache.
2574
2575 // It is important to check cacheResponding before
2576 // prefetchSquashed. If another cache has committed to
2577 // responding, it will be sending a dirty response which will
2578 // arrive at the MSHR allocated for this request. Checking the
2579 // prefetchSquash first may result in the MSHR being
2580 // prematurely deallocated.
2581 if (snoop_pkt.cacheResponding()) {
2582 auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2583 assert(r.second);
2584
2585 // if we are getting a snoop response with no sharers it
2586 // will be allocated as Modified
2587 bool pending_modified_resp = !snoop_pkt.hasSharers();
2588 markInService(mshr, pending_modified_resp);
2589
2590 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2591 " %#x (%s) hit\n",
2592 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2593 return false;
2594 }
2595
2596 if (snoop_pkt.isBlockCached()) {
2597 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2598 "Deallocating mshr target %#x.\n",
2599 mshr->blkAddr);
2600
2601 // Deallocate the mshr target
2602 if (mshrQueue.forceDeallocateTarget(mshr)) {
2603 // Clear block if this deallocation resulted freed an
2604 // mshr when all had previously been utilized
2605 clearBlocked(Blocked_NoMSHRs);
2606 }
2607
2608 // given that no response is expected, delete Request and Packet
2609 delete tgt_pkt->req;
2610 delete tgt_pkt;
2611
2612 return false;
2613 }
2614 }
2615
2616 // either a prefetch that is not present upstream, or a normal
2617 // MSHR request, proceed to get the packet to send downstream
2618 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2619
2620 mshr->isForward = (pkt == nullptr);
2621
2622 if (mshr->isForward) {
2623 // not a cache block request, but a response is expected
2624 // make copy of current packet to forward, keep current
2625 // copy for response handling
2626 pkt = new Packet(tgt_pkt, false, true);
2627 assert(!pkt->isWrite());
2628 }
2629
2630 // play it safe and append (rather than set) the sender state,
2631 // as forwarded packets may already have existing state
2632 pkt->pushSenderState(mshr);
2633
2634 if (pkt->isClean() && blk && blk->isDirty()) {
2635 // A cache clean opearation is looking for a dirty block. Mark
2636 // the packet so that the destination xbar can determine that
2637 // there will be a follow-up write packet as well.
2638 pkt->setSatisfied();
2639 }
2640
2641 if (!memSidePort->sendTimingReq(pkt)) {
2642 // we are awaiting a retry, but we
2643 // delete the packet and will be creating a new packet
2644 // when we get the opportunity
2645 delete pkt;
2646
2647 // note that we have now masked any requestBus and
2648 // schedSendEvent (we will wait for a retry before
2649 // doing anything), and this is so even if we do not
2650 // care about this packet and might override it before
2651 // it gets retried
2652 return true;
2653 } else {
2654 // As part of the call to sendTimingReq the packet is
2655 // forwarded to all neighbouring caches (and any caches
2656 // above them) as a snoop. Thus at this point we know if
2657 // any of the neighbouring caches are responding, and if
2658 // so, we know it is dirty, and we can determine if it is
2659 // being passed as Modified, making our MSHR the ordering
2660 // point
2661 bool pending_modified_resp = !pkt->hasSharers() &&
2662 pkt->cacheResponding();
2663 markInService(mshr, pending_modified_resp);
2664 if (pkt->isClean() && blk && blk->isDirty()) {
2665 // A cache clean opearation is looking for a dirty
2666 // block. If a dirty block is encountered a WriteClean
2667 // will update any copies to the path to the memory
2668 // until the point of reference.
2669 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
2670 __func__, pkt->print(), blk->print());
2671 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
2672 pkt->id);
2673 PacketList writebacks;
2674 writebacks.push_back(wb_pkt);
2675 doWritebacks(writebacks, 0);
2676 }
2677
2678 return false;
2679 }
2680}
2681
2682bool
2683Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2684{
2685 assert(wq_entry);
2686
2687 // always a single target for write queue entries
2688 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2689
2690 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2691
2692 // forward as is, both for evictions and uncacheable writes
2693 if (!memSidePort->sendTimingReq(tgt_pkt)) {
2694 // note that we have now masked any requestBus and
2695 // schedSendEvent (we will wait for a retry before
2696 // doing anything), and this is so even if we do not
2697 // care about this packet and might override it before
2698 // it gets retried
2699 return true;
2700 } else {
2701 markInService(wq_entry);
2702 return false;
2703 }
2704}
2705
2706void
2707Cache::serialize(CheckpointOut &cp) const
2708{
2709 bool dirty(isDirty());
2710
2711 if (dirty) {
2712 warn("*** The cache still contains dirty data. ***\n");
2713 warn(" Make sure to drain the system using the correct flags.\n");
2714 warn(" This checkpoint will not restore correctly and dirty data "
2715 " in the cache will be lost!\n");
2716 }
2717
2718 // Since we don't checkpoint the data in the cache, any dirty data
2719 // will be lost when restoring from a checkpoint of a system that
2720 // wasn't drained properly. Flag the checkpoint as invalid if the
2721 // cache contains dirty data.
2722 bool bad_checkpoint(dirty);
2723 SERIALIZE_SCALAR(bad_checkpoint);
2724}
2725
2726void
2727Cache::unserialize(CheckpointIn &cp)
2728{
2729 bool bad_checkpoint;
2730 UNSERIALIZE_SCALAR(bad_checkpoint);
2731 if (bad_checkpoint) {
2732 fatal("Restoring from checkpoints with dirty caches is not supported "
2733 "in the classic memory system. Please remove any caches or "
2734 " drain them properly before taking checkpoints.\n");
2735 }
2736}
2737
2738///////////////
2739//
2740// CpuSidePort
2741//
2742///////////////
2743
2744AddrRangeList
2745Cache::CpuSidePort::getAddrRanges() const
2746{
2747 return cache->getAddrRanges();
2748}
2749
2750bool
2751Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2752{
2753 assert(!cache->system->bypassCaches());
2754
2755 // always let express snoop packets through if even if blocked
2756 if (pkt->isExpressSnoop()) {
2757 return true;
2758 } else if (isBlocked() || mustSendRetry) {
2759 // either already committed to send a retry, or blocked
2760 mustSendRetry = true;
2761 return false;
2762 }
2763 mustSendRetry = false;
2764 return true;
2765}
2766
2767bool
2768Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2769{
2770 assert(!cache->system->bypassCaches());
2771
2772 // always let express snoop packets through if even if blocked
| 979}
980
981PacketPtr
982Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
983 bool needsWritable) const
984{
985 // should never see evictions here
986 assert(!cpu_pkt->isEviction());
987
988 bool blkValid = blk && blk->isValid();
989
990 if (cpu_pkt->req->isUncacheable() ||
991 (!blkValid && cpu_pkt->isUpgrade()) ||
992 cpu_pkt->cmd == MemCmd::InvalidateReq || cpu_pkt->isClean()) {
993 // uncacheable requests and upgrades from upper-level caches
994 // that missed completely just go through as is
995 return nullptr;
996 }
997
998 assert(cpu_pkt->needsResponse());
999
1000 MemCmd cmd;
1001 // @TODO make useUpgrades a parameter.
1002 // Note that ownership protocols require upgrade, otherwise a
1003 // write miss on a shared owned block will generate a ReadExcl,
1004 // which will clobber the owned copy.
1005 const bool useUpgrades = true;
1006 if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
1007 assert(!blkValid || !blk->isWritable());
1008 // forward as invalidate to all other caches, this gives us
1009 // the line in Exclusive state, and invalidates all other
1010 // copies
1011 cmd = MemCmd::InvalidateReq;
1012 } else if (blkValid && useUpgrades) {
1013 // only reason to be here is that blk is read only and we need
1014 // it to be writable
1015 assert(needsWritable);
1016 assert(!blk->isWritable());
1017 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
1018 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
1019 cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
1020 // Even though this SC will fail, we still need to send out the
1021 // request and get the data to supply it to other snoopers in the case
1022 // where the determination the StoreCond fails is delayed due to
1023 // all caches not being on the same local bus.
1024 cmd = MemCmd::SCUpgradeFailReq;
1025 } else {
1026 // block is invalid
1027
1028 // If the request does not need a writable there are two cases
1029 // where we need to ensure the response will not fetch the
1030 // block in dirty state:
1031 // * this cache is read only and it does not perform
1032 // writebacks,
1033 // * this cache is mostly exclusive and will not fill (since
1034 // it does not fill it will have to writeback the dirty data
1035 // immediately which generates uneccesary writebacks).
1036 bool force_clean_rsp = isReadOnly || clusivity == Enums::mostly_excl;
1037 cmd = needsWritable ? MemCmd::ReadExReq :
1038 (force_clean_rsp ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1039 }
1040 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1041
1042 // if there are upstream caches that have already marked the
1043 // packet as having sharers (not passing writable), pass that info
1044 // downstream
1045 if (cpu_pkt->hasSharers() && !needsWritable) {
1046 // note that cpu_pkt may have spent a considerable time in the
1047 // MSHR queue and that the information could possibly be out
1048 // of date, however, there is no harm in conservatively
1049 // assuming the block has sharers
1050 pkt->setHasSharers();
1051 DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1052 __func__, cpu_pkt->print(), pkt->print());
1053 }
1054
1055 // the packet should be block aligned
1056 assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1057
1058 pkt->allocate();
1059 DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1060 cpu_pkt->print());
1061 return pkt;
1062}
1063
1064
1065Tick
1066Cache::recvAtomic(PacketPtr pkt)
1067{
1068 // We are in atomic mode so we pay just for lookupLatency here.
1069 Cycles lat = lookupLatency;
1070
1071 // Forward the request if the system is in cache bypass mode.
1072 if (system->bypassCaches())
1073 return ticksToCycles(memSidePort->sendAtomic(pkt));
1074
1075 promoteWholeLineWrites(pkt);
1076
1077 // follow the same flow as in recvTimingReq, and check if a cache
1078 // above us is responding
1079 if (pkt->cacheResponding() && !pkt->isClean()) {
1080 assert(!pkt->req->isCacheInvalidate());
1081 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1082 pkt->print());
1083
1084 // if a cache is responding, and it had the line in Owned
1085 // rather than Modified state, we need to invalidate any
1086 // copies that are not on the same path to memory
1087 assert(pkt->needsWritable() && !pkt->responderHadWritable());
1088 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1089
1090 return lat * clockPeriod();
1091 }
1092
1093 // should assert here that there are no outstanding MSHRs or
1094 // writebacks... that would mean that someone used an atomic
1095 // access in timing mode
1096
1097 CacheBlk *blk = nullptr;
1098 PacketList writebacks;
1099 bool satisfied = access(pkt, blk, lat, writebacks);
1100
1101 if (pkt->isClean() && blk && blk->isDirty()) {
1102 // A cache clean opearation is looking for a dirty
1103 // block. If a dirty block is encountered a WriteClean
1104 // will update any copies to the path to the memory
1105 // until the point of reference.
1106 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
1107 __func__, pkt->print(), blk->print());
1108 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
1109 writebacks.push_back(wb_pkt);
1110 pkt->setSatisfied();
1111 }
1112
1113 // handle writebacks resulting from the access here to ensure they
1114 // logically proceed anything happening below
1115 doWritebacksAtomic(writebacks);
1116
1117 if (!satisfied) {
1118 // MISS
1119
1120 // deal with the packets that go through the write path of
1121 // the cache, i.e. any evictions and writes
1122 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1123 (pkt->req->isUncacheable() && pkt->isWrite())) {
1124 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1125 return lat * clockPeriod();
1126 }
1127 // only misses left
1128
1129 PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1130
1131 bool is_forward = (bus_pkt == nullptr);
1132
1133 if (is_forward) {
1134 // just forwarding the same request to the next level
1135 // no local cache operation involved
1136 bus_pkt = pkt;
1137 }
1138
1139 DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1140 bus_pkt->print());
1141
1142#if TRACING_ON
1143 CacheBlk::State old_state = blk ? blk->status : 0;
1144#endif
1145
1146 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1147
1148 bool is_invalidate = bus_pkt->isInvalidate();
1149
1150 // We are now dealing with the response handling
1151 DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1152 bus_pkt->print(), old_state);
1153
1154 // If packet was a forward, the response (if any) is already
1155 // in place in the bus_pkt == pkt structure, so we don't need
1156 // to do anything. Otherwise, use the separate bus_pkt to
1157 // generate response to pkt and then delete it.
1158 if (!is_forward) {
1159 if (pkt->needsResponse()) {
1160 assert(bus_pkt->isResponse());
1161 if (bus_pkt->isError()) {
1162 pkt->makeAtomicResponse();
1163 pkt->copyError(bus_pkt);
1164 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1165 // note the use of pkt, not bus_pkt here.
1166
1167 // write-line request to the cache that promoted
1168 // the write to a whole line
1169 blk = handleFill(pkt, blk, writebacks,
1170 allocOnFill(pkt->cmd));
1171 assert(blk != NULL);
1172 is_invalidate = false;
1173 satisfyRequest(pkt, blk);
1174 } else if (bus_pkt->isRead() ||
1175 bus_pkt->cmd == MemCmd::UpgradeResp) {
1176 // we're updating cache state to allow us to
1177 // satisfy the upstream request from the cache
1178 blk = handleFill(bus_pkt, blk, writebacks,
1179 allocOnFill(pkt->cmd));
1180 satisfyRequest(pkt, blk);
1181 maintainClusivity(pkt->fromCache(), blk);
1182 } else {
1183 // we're satisfying the upstream request without
1184 // modifying cache state, e.g., a write-through
1185 pkt->makeAtomicResponse();
1186 }
1187 }
1188 delete bus_pkt;
1189 }
1190
1191 if (is_invalidate && blk && blk->isValid()) {
1192 invalidateBlock(blk);
1193 }
1194 }
1195
1196 // Note that we don't invoke the prefetcher at all in atomic mode.
1197 // It's not clear how to do it properly, particularly for
1198 // prefetchers that aggressively generate prefetch candidates and
1199 // rely on bandwidth contention to throttle them; these will tend
1200 // to pollute the cache in atomic mode since there is no bandwidth
1201 // contention. If we ever do want to enable prefetching in atomic
1202 // mode, though, this is the place to do it... see timingAccess()
1203 // for an example (though we'd want to issue the prefetch(es)
1204 // immediately rather than calling requestMemSideBus() as we do
1205 // there).
1206
1207 // do any writebacks resulting from the response handling
1208 doWritebacksAtomic(writebacks);
1209
1210 // if we used temp block, check to see if its valid and if so
1211 // clear it out, but only do so after the call to recvAtomic is
1212 // finished so that any downstream observers (such as a snoop
1213 // filter), first see the fill, and only then see the eviction
1214 if (blk == tempBlock && tempBlock->isValid()) {
1215 // the atomic CPU calls recvAtomic for fetch and load/store
1216 // sequentuially, and we may already have a tempBlock
1217 // writeback from the fetch that we have not yet sent
1218 if (tempBlockWriteback) {
1219 // if that is the case, write the prevoius one back, and
1220 // do not schedule any new event
1221 writebackTempBlockAtomic();
1222 } else {
1223 // the writeback/clean eviction happens after the call to
1224 // recvAtomic has finished (but before any successive
1225 // calls), so that the response handling from the fill is
1226 // allowed to happen first
1227 schedule(writebackTempBlockAtomicEvent, curTick());
1228 }
1229
1230 tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1231 writebackBlk(blk) : cleanEvictBlk(blk);
1232 invalidateBlock(blk);
1233 }
1234
1235 if (pkt->needsResponse()) {
1236 pkt->makeAtomicResponse();
1237 }
1238
1239 return lat * clockPeriod();
1240}
1241
1242
1243void
1244Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1245{
1246 if (system->bypassCaches()) {
1247 // Packets from the memory side are snoop request and
1248 // shouldn't happen in bypass mode.
1249 assert(fromCpuSide);
1250
1251 // The cache should be flushed if we are in cache bypass mode,
1252 // so we don't need to check if we need to update anything.
1253 memSidePort->sendFunctional(pkt);
1254 return;
1255 }
1256
1257 Addr blk_addr = pkt->getBlockAddr(blkSize);
1258 bool is_secure = pkt->isSecure();
1259 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1260 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1261
1262 pkt->pushLabel(name());
1263
1264 CacheBlkPrintWrapper cbpw(blk);
1265
1266 // Note that just because an L2/L3 has valid data doesn't mean an
1267 // L1 doesn't have a more up-to-date modified copy that still
1268 // needs to be found. As a result we always update the request if
1269 // we have it, but only declare it satisfied if we are the owner.
1270
1271 // see if we have data at all (owned or otherwise)
1272 bool have_data = blk && blk->isValid()
1273 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1274 blk->data);
1275
1276 // data we have is dirty if marked as such or if we have an
1277 // in-service MSHR that is pending a modified line
1278 bool have_dirty =
1279 have_data && (blk->isDirty() ||
1280 (mshr && mshr->inService && mshr->isPendingModified()));
1281
1282 bool done = have_dirty
1283 || cpuSidePort->checkFunctional(pkt)
1284 || mshrQueue.checkFunctional(pkt, blk_addr)
1285 || writeBuffer.checkFunctional(pkt, blk_addr)
1286 || memSidePort->checkFunctional(pkt);
1287
1288 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
1289 (blk && blk->isValid()) ? "valid " : "",
1290 have_data ? "data " : "", done ? "done " : "");
1291
1292 // We're leaving the cache, so pop cache->name() label
1293 pkt->popLabel();
1294
1295 if (done) {
1296 pkt->makeResponse();
1297 } else {
1298 // if it came as a request from the CPU side then make sure it
1299 // continues towards the memory side
1300 if (fromCpuSide) {
1301 memSidePort->sendFunctional(pkt);
1302 } else if (cpuSidePort->isSnooping()) {
1303 // if it came from the memory side, it must be a snoop request
1304 // and we should only forward it if we are forwarding snoops
1305 cpuSidePort->sendFunctionalSnoop(pkt);
1306 }
1307 }
1308}
1309
1310
1311/////////////////////////////////////////////////////
1312//
1313// Response handling: responses from the memory side
1314//
1315/////////////////////////////////////////////////////
1316
1317
1318void
1319Cache::handleUncacheableWriteResp(PacketPtr pkt)
1320{
1321 Tick completion_time = clockEdge(responseLatency) +
1322 pkt->headerDelay + pkt->payloadDelay;
1323
1324 // Reset the bus additional time as it is now accounted for
1325 pkt->headerDelay = pkt->payloadDelay = 0;
1326
1327 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1328}
1329
1330void
1331Cache::recvTimingResp(PacketPtr pkt)
1332{
1333 assert(pkt->isResponse());
1334
1335 // all header delay should be paid for by the crossbar, unless
1336 // this is a prefetch response from above
1337 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1338 "%s saw a non-zero packet delay\n", name());
1339
1340 bool is_error = pkt->isError();
1341
1342 if (is_error) {
1343 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1344 pkt->print());
1345 }
1346
1347 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1348 pkt->print());
1349
1350 // if this is a write, we should be looking at an uncacheable
1351 // write
1352 if (pkt->isWrite()) {
1353 assert(pkt->req->isUncacheable());
1354 handleUncacheableWriteResp(pkt);
1355 return;
1356 }
1357
1358 // we have dealt with any (uncacheable) writes above, from here on
1359 // we know we are dealing with an MSHR due to a miss or a prefetch
1360 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1361 assert(mshr);
1362
1363 if (mshr == noTargetMSHR) {
1364 // we always clear at least one target
1365 clearBlocked(Blocked_NoTargets);
1366 noTargetMSHR = nullptr;
1367 }
1368
1369 // Initial target is used just for stats
1370 MSHR::Target *initial_tgt = mshr->getTarget();
1371 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1372 Tick miss_latency = curTick() - initial_tgt->recvTime;
1373
1374 if (pkt->req->isUncacheable()) {
1375 assert(pkt->req->masterId() < system->maxMasters());
1376 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1377 miss_latency;
1378 } else {
1379 assert(pkt->req->masterId() < system->maxMasters());
1380 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1381 miss_latency;
1382 }
1383
1384 bool wasFull = mshrQueue.isFull();
1385
1386 PacketList writebacks;
1387
1388 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1389
1390 bool is_fill = !mshr->isForward &&
1391 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1392
1393 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1394 const bool valid_blk = blk && blk->isValid();
1395 // If the response indicates that there are no sharers and we
1396 // either had the block already or the response is filling we can
1397 // promote our copy to writable
1398 if (!pkt->hasSharers() &&
1399 (is_fill || (valid_blk && !pkt->req->isCacheInvalidate()))) {
1400 mshr->promoteWritable();
1401 }
1402
1403 if (is_fill && !is_error) {
1404 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1405 pkt->getAddr());
1406
1407 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1408 assert(blk != nullptr);
1409 }
1410
1411 // allow invalidation responses originating from write-line
1412 // requests to be discarded
1413 bool is_invalidate = pkt->isInvalidate();
1414
1415 // The block was marked as not readable while there was a pending
1416 // cache maintenance operation, restore its flag.
1417 if (pkt->isClean() && !is_invalidate && valid_blk) {
1418 blk->status |= BlkReadable;
1419 }
1420
1421 // First offset for critical word first calculations
1422 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1423
1424 bool from_cache = false;
1425 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1426 for (auto &target: targets) {
1427 Packet *tgt_pkt = target.pkt;
1428 switch (target.source) {
1429 case MSHR::Target::FromCPU:
1430 Tick completion_time;
1431 // Here we charge on completion_time the delay of the xbar if the
1432 // packet comes from it, charged on headerDelay.
1433 completion_time = pkt->headerDelay;
1434
1435 // Software prefetch handling for cache closest to core
1436 if (tgt_pkt->cmd.isSWPrefetch()) {
1437 // a software prefetch would have already been ack'd
1438 // immediately with dummy data so the core would be able to
1439 // retire it. This request completes right here, so we
1440 // deallocate it.
1441 delete tgt_pkt->req;
1442 delete tgt_pkt;
1443 break; // skip response
1444 }
1445
1446 // keep track of whether we have responded to another
1447 // cache
1448 from_cache = from_cache || tgt_pkt->fromCache();
1449
1450 // unlike the other packet flows, where data is found in other
1451 // caches or memory and brought back, write-line requests always
1452 // have the data right away, so the above check for "is fill?"
1453 // cannot actually be determined until examining the stored MSHR
1454 // state. We "catch up" with that logic here, which is duplicated
1455 // from above.
1456 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1457 assert(!is_error);
1458 // we got the block in a writable state, so promote
1459 // any deferred targets if possible
1460 mshr->promoteWritable();
1461 // NB: we use the original packet here and not the response!
1462 blk = handleFill(tgt_pkt, blk, writebacks,
1463 targets.allocOnFill);
1464 assert(blk != nullptr);
1465
1466 // treat as a fill, and discard the invalidation
1467 // response
1468 is_fill = true;
1469 is_invalidate = false;
1470 }
1471
1472 if (is_fill) {
1473 satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1474
1475 // How many bytes past the first request is this one
1476 int transfer_offset =
1477 tgt_pkt->getOffset(blkSize) - initial_offset;
1478 if (transfer_offset < 0) {
1479 transfer_offset += blkSize;
1480 }
1481
1482 // If not critical word (offset) return payloadDelay.
1483 // responseLatency is the latency of the return path
1484 // from lower level caches/memory to an upper level cache or
1485 // the core.
1486 completion_time += clockEdge(responseLatency) +
1487 (transfer_offset ? pkt->payloadDelay : 0);
1488
1489 assert(!tgt_pkt->req->isUncacheable());
1490
1491 assert(tgt_pkt->req->masterId() < system->maxMasters());
1492 missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1493 completion_time - target.recvTime;
1494 } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1495 // failed StoreCond upgrade
1496 assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1497 tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1498 tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1499 // responseLatency is the latency of the return path
1500 // from lower level caches/memory to an upper level cache or
1501 // the core.
1502 completion_time += clockEdge(responseLatency) +
1503 pkt->payloadDelay;
1504 tgt_pkt->req->setExtraData(0);
1505 } else {
1506 // We are about to send a response to a cache above
1507 // that asked for an invalidation; we need to
1508 // invalidate our copy immediately as the most
1509 // up-to-date copy of the block will now be in the
1510 // cache above. It will also prevent this cache from
1511 // responding (if the block was previously dirty) to
1512 // snoops as they should snoop the caches above where
1513 // they will get the response from.
1514 if (is_invalidate && blk && blk->isValid()) {
1515 invalidateBlock(blk);
1516 }
1517 // not a cache fill, just forwarding response
1518 // responseLatency is the latency of the return path
1519 // from lower level cahces/memory to the core.
1520 completion_time += clockEdge(responseLatency) +
1521 pkt->payloadDelay;
1522 if (pkt->isRead() && !is_error) {
1523 // sanity check
1524 assert(pkt->getAddr() == tgt_pkt->getAddr());
1525 assert(pkt->getSize() >= tgt_pkt->getSize());
1526
1527 tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1528 }
1529 }
1530 tgt_pkt->makeTimingResponse();
1531 // if this packet is an error copy that to the new packet
1532 if (is_error)
1533 tgt_pkt->copyError(pkt);
1534 if (tgt_pkt->cmd == MemCmd::ReadResp &&
1535 (is_invalidate || mshr->hasPostInvalidate())) {
1536 // If intermediate cache got ReadRespWithInvalidate,
1537 // propagate that. Response should not have
1538 // isInvalidate() set otherwise.
1539 tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1540 DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1541 tgt_pkt->print());
1542 }
1543 // Reset the bus additional time as it is now accounted for
1544 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1545 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1546 break;
1547
1548 case MSHR::Target::FromPrefetcher:
1549 assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1550 if (blk)
1551 blk->status |= BlkHWPrefetched;
1552 delete tgt_pkt->req;
1553 delete tgt_pkt;
1554 break;
1555
1556 case MSHR::Target::FromSnoop:
1557 // I don't believe that a snoop can be in an error state
1558 assert(!is_error);
1559 // response to snoop request
1560 DPRINTF(Cache, "processing deferred snoop...\n");
1561 // If the response is invalidating, a snooping target can
1562 // be satisfied if it is also invalidating. If the reponse is, not
1563 // only invalidating, but more specifically an InvalidateResp and
1564 // the MSHR was created due to an InvalidateReq then a cache above
1565 // is waiting to satisfy a WriteLineReq. In this case even an
1566 // non-invalidating snoop is added as a target here since this is
1567 // the ordering point. When the InvalidateResp reaches this cache,
1568 // the snooping target will snoop further the cache above with the
1569 // WriteLineReq.
1570 assert(!is_invalidate || pkt->cmd == MemCmd::InvalidateResp ||
1571 pkt->req->isCacheMaintenance() ||
1572 mshr->hasPostInvalidate());
1573 handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1574 break;
1575
1576 default:
1577 panic("Illegal target->source enum %d\n", target.source);
1578 }
1579 }
1580
1581 maintainClusivity(from_cache, blk);
1582
1583 if (blk && blk->isValid()) {
1584 // an invalidate response stemming from a write line request
1585 // should not invalidate the block, so check if the
1586 // invalidation should be discarded
1587 if (is_invalidate || mshr->hasPostInvalidate()) {
1588 invalidateBlock(blk);
1589 } else if (mshr->hasPostDowngrade()) {
1590 blk->status &= ~BlkWritable;
1591 }
1592 }
1593
1594 if (mshr->promoteDeferredTargets()) {
1595 // avoid later read getting stale data while write miss is
1596 // outstanding.. see comment in timingAccess()
1597 if (blk) {
1598 blk->status &= ~BlkReadable;
1599 }
1600 mshrQueue.markPending(mshr);
1601 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1602 } else {
1603 mshrQueue.deallocate(mshr);
1604 if (wasFull && !mshrQueue.isFull()) {
1605 clearBlocked(Blocked_NoMSHRs);
1606 }
1607
1608 // Request the bus for a prefetch if this deallocation freed enough
1609 // MSHRs for a prefetch to take place
1610 if (prefetcher && mshrQueue.canPrefetch()) {
1611 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1612 clockEdge());
1613 if (next_pf_time != MaxTick)
1614 schedMemSideSendEvent(next_pf_time);
1615 }
1616 }
1617 // reset the xbar additional timinig as it is now accounted for
1618 pkt->headerDelay = pkt->payloadDelay = 0;
1619
1620 // copy writebacks to write buffer
1621 doWritebacks(writebacks, forward_time);
1622
1623 // if we used temp block, check to see if its valid and then clear it out
1624 if (blk == tempBlock && tempBlock->isValid()) {
1625 // We use forwardLatency here because we are copying
1626 // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1627 // allocate an internal buffer and to schedule an event to the
1628 // queued port.
1629 if (blk->isDirty() || writebackClean) {
1630 PacketPtr wbPkt = writebackBlk(blk);
1631 allocateWriteBuffer(wbPkt, forward_time);
1632 // Set BLOCK_CACHED flag if cached above.
1633 if (isCachedAbove(wbPkt))
1634 wbPkt->setBlockCached();
1635 } else {
1636 PacketPtr wcPkt = cleanEvictBlk(blk);
1637 // Check to see if block is cached above. If not allocate
1638 // write buffer
1639 if (isCachedAbove(wcPkt))
1640 delete wcPkt;
1641 else
1642 allocateWriteBuffer(wcPkt, forward_time);
1643 }
1644 invalidateBlock(blk);
1645 }
1646
1647 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1648 delete pkt;
1649}
1650
1651PacketPtr
1652Cache::writebackBlk(CacheBlk *blk)
1653{
1654 chatty_assert(!isReadOnly || writebackClean,
1655 "Writeback from read-only cache");
1656 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1657
1658 writebacks[Request::wbMasterId]++;
1659
1660 Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1661 Request::wbMasterId);
1662 if (blk->isSecure())
1663 req->setFlags(Request::SECURE);
1664
1665 req->taskId(blk->task_id);
1666
1667 PacketPtr pkt =
1668 new Packet(req, blk->isDirty() ?
1669 MemCmd::WritebackDirty : MemCmd::WritebackClean);
1670
1671 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1672 pkt->print(), blk->isWritable(), blk->isDirty());
1673
1674 if (blk->isWritable()) {
1675 // not asserting shared means we pass the block in modified
1676 // state, mark our own block non-writeable
1677 blk->status &= ~BlkWritable;
1678 } else {
1679 // we are in the Owned state, tell the receiver
1680 pkt->setHasSharers();
1681 }
1682
1683 // make sure the block is not marked dirty
1684 blk->status &= ~BlkDirty;
1685
1686 pkt->allocate();
1687 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1688
1689 return pkt;
1690}
1691
1692PacketPtr
1693Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest, PacketId id)
1694{
1695 Request *req = new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1696 Request::wbMasterId);
1697 if (blk->isSecure()) {
1698 req->setFlags(Request::SECURE);
1699 }
1700 req->taskId(blk->task_id);
1701
1702 PacketPtr pkt = new Packet(req, MemCmd::WriteClean, blkSize, id);
1703
1704 if (dest) {
1705 req->setFlags(dest);
1706 pkt->setWriteThrough();
1707 }
1708
1709 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1710 blk->isWritable(), blk->isDirty());
1711
1712 if (blk->isWritable()) {
1713 // not asserting shared means we pass the block in modified
1714 // state, mark our own block non-writeable
1715 blk->status &= ~BlkWritable;
1716 } else {
1717 // we are in the Owned state, tell the receiver
1718 pkt->setHasSharers();
1719 }
1720
1721 // make sure the block is not marked dirty
1722 blk->status &= ~BlkDirty;
1723
1724 pkt->allocate();
1725 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1726
1727 return pkt;
1728}
1729
1730
1731PacketPtr
1732Cache::cleanEvictBlk(CacheBlk *blk)
1733{
1734 assert(!writebackClean);
1735 assert(blk && blk->isValid() && !blk->isDirty());
1736 // Creating a zero sized write, a message to the snoop filter
1737 Request *req =
1738 new Request(tags->regenerateBlkAddr(blk), blkSize, 0,
1739 Request::wbMasterId);
1740 if (blk->isSecure())
1741 req->setFlags(Request::SECURE);
1742
1743 req->taskId(blk->task_id);
1744
1745 PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1746 pkt->allocate();
1747 DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1748
1749 return pkt;
1750}
1751
1752void
1753Cache::memWriteback()
1754{
1755 CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1756 tags->forEachBlk(visitor);
1757}
1758
1759void
1760Cache::memInvalidate()
1761{
1762 CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1763 tags->forEachBlk(visitor);
1764}
1765
1766bool
1767Cache::isDirty() const
1768{
1769 CacheBlkIsDirtyVisitor visitor;
1770 tags->forEachBlk(visitor);
1771
1772 return visitor.isDirty();
1773}
1774
1775bool
1776Cache::writebackVisitor(CacheBlk &blk)
1777{
1778 if (blk.isDirty()) {
1779 assert(blk.isValid());
1780
1781 Request request(tags->regenerateBlkAddr(&blk), blkSize, 0,
1782 Request::funcMasterId);
1783 request.taskId(blk.task_id);
1784 if (blk.isSecure()) {
1785 request.setFlags(Request::SECURE);
1786 }
1787
1788 Packet packet(&request, MemCmd::WriteReq);
1789 packet.dataStatic(blk.data);
1790
1791 memSidePort->sendFunctional(&packet);
1792
1793 blk.status &= ~BlkDirty;
1794 }
1795
1796 return true;
1797}
1798
1799bool
1800Cache::invalidateVisitor(CacheBlk &blk)
1801{
1802
1803 if (blk.isDirty())
1804 warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1805
1806 if (blk.isValid()) {
1807 assert(!blk.isDirty());
1808 invalidateBlock(&blk);
1809 }
1810
1811 return true;
1812}
1813
1814CacheBlk*
1815Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1816{
1817 // Find replacement victim
1818 CacheBlk *blk = tags->findVictim(addr);
1819
1820 // It is valid to return nullptr if there is no victim
1821 if (!blk)
1822 return nullptr;
1823
1824 if (blk->isValid()) {
1825 Addr repl_addr = tags->regenerateBlkAddr(blk);
1826 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1827 if (repl_mshr) {
1828 // must be an outstanding upgrade or clean request
1829 // on a block we're about to replace...
1830 assert((!blk->isWritable() && repl_mshr->needsWritable()) ||
1831 repl_mshr->isCleaning());
1832 // too hard to replace block with transient state
1833 // allocation failed, block not inserted
1834 return nullptr;
1835 } else {
1836 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1837 "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1838 addr, is_secure ? "s" : "ns",
1839 blk->isDirty() ? "writeback" : "clean");
1840
1841 if (blk->wasPrefetched()) {
1842 unusedPrefetches++;
1843 }
1844 // Will send up Writeback/CleanEvict snoops via isCachedAbove
1845 // when pushing this writeback list into the write buffer.
1846 if (blk->isDirty() || writebackClean) {
1847 // Save writeback packet for handling by caller
1848 writebacks.push_back(writebackBlk(blk));
1849 } else {
1850 writebacks.push_back(cleanEvictBlk(blk));
1851 }
1852 }
1853 }
1854
1855 return blk;
1856}
1857
1858void
1859Cache::invalidateBlock(CacheBlk *blk)
1860{
1861 if (blk != tempBlock)
1862 tags->invalidate(blk);
1863 blk->invalidate();
1864}
1865
1866// Note that the reason we return a list of writebacks rather than
1867// inserting them directly in the write buffer is that this function
1868// is called by both atomic and timing-mode accesses, and in atomic
1869// mode we don't mess with the write buffer (we just perform the
1870// writebacks atomically once the original request is complete).
1871CacheBlk*
1872Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1873 bool allocate)
1874{
1875 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1876 Addr addr = pkt->getAddr();
1877 bool is_secure = pkt->isSecure();
1878#if TRACING_ON
1879 CacheBlk::State old_state = blk ? blk->status : 0;
1880#endif
1881
1882 // When handling a fill, we should have no writes to this line.
1883 assert(addr == pkt->getBlockAddr(blkSize));
1884 assert(!writeBuffer.findMatch(addr, is_secure));
1885
1886 if (blk == nullptr) {
1887 // better have read new data...
1888 assert(pkt->hasData());
1889
1890 // only read responses and write-line requests have data;
1891 // note that we don't write the data here for write-line - that
1892 // happens in the subsequent call to satisfyRequest
1893 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1894
1895 // need to do a replacement if allocating, otherwise we stick
1896 // with the temporary storage
1897 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1898
1899 if (blk == nullptr) {
1900 // No replaceable block or a mostly exclusive
1901 // cache... just use temporary storage to complete the
1902 // current request and then get rid of it
1903 assert(!tempBlock->isValid());
1904 blk = tempBlock;
1905 tempBlock->set = tags->extractSet(addr);
1906 tempBlock->tag = tags->extractTag(addr);
1907 if (is_secure) {
1908 tempBlock->status |= BlkSecure;
1909 }
1910 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1911 is_secure ? "s" : "ns");
1912 } else {
1913 tags->insertBlock(pkt, blk);
1914 }
1915
1916 // we should never be overwriting a valid block
1917 assert(!blk->isValid());
1918 } else {
1919 // existing block... probably an upgrade
1920 assert(blk->tag == tags->extractTag(addr));
1921 // either we're getting new data or the block should already be valid
1922 assert(pkt->hasData() || blk->isValid());
1923 // don't clear block status... if block is already dirty we
1924 // don't want to lose that
1925 }
1926
1927 if (is_secure)
1928 blk->status |= BlkSecure;
1929 blk->status |= BlkValid | BlkReadable;
1930
1931 // sanity check for whole-line writes, which should always be
1932 // marked as writable as part of the fill, and then later marked
1933 // dirty as part of satisfyRequest
1934 if (pkt->cmd == MemCmd::WriteLineReq) {
1935 assert(!pkt->hasSharers());
1936 }
1937
1938 // here we deal with setting the appropriate state of the line,
1939 // and we start by looking at the hasSharers flag, and ignore the
1940 // cacheResponding flag (normally signalling dirty data) if the
1941 // packet has sharers, thus the line is never allocated as Owned
1942 // (dirty but not writable), and always ends up being either
1943 // Shared, Exclusive or Modified, see Packet::setCacheResponding
1944 // for more details
1945 if (!pkt->hasSharers()) {
1946 // we could get a writable line from memory (rather than a
1947 // cache) even in a read-only cache, note that we set this bit
1948 // even for a read-only cache, possibly revisit this decision
1949 blk->status |= BlkWritable;
1950
1951 // check if we got this via cache-to-cache transfer (i.e., from a
1952 // cache that had the block in Modified or Owned state)
1953 if (pkt->cacheResponding()) {
1954 // we got the block in Modified state, and invalidated the
1955 // owners copy
1956 blk->status |= BlkDirty;
1957
1958 chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1959 "in read-only cache %s\n", name());
1960 }
1961 }
1962
1963 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1964 addr, is_secure ? "s" : "ns", old_state, blk->print());
1965
1966 // if we got new data, copy it in (checking for a read response
1967 // and a response that has data is the same in the end)
1968 if (pkt->isRead()) {
1969 // sanity checks
1970 assert(pkt->hasData());
1971 assert(pkt->getSize() == blkSize);
1972
1973 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
1974 }
1975 // We pay for fillLatency here.
1976 blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1977 pkt->payloadDelay;
1978
1979 return blk;
1980}
1981
1982
1983/////////////////////////////////////////////////////
1984//
1985// Snoop path: requests coming in from the memory side
1986//
1987/////////////////////////////////////////////////////
1988
1989void
1990Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1991 bool already_copied, bool pending_inval)
1992{
1993 // sanity check
1994 assert(req_pkt->isRequest());
1995 assert(req_pkt->needsResponse());
1996
1997 DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1998 // timing-mode snoop responses require a new packet, unless we
1999 // already made a copy...
2000 PacketPtr pkt = req_pkt;
2001 if (!already_copied)
2002 // do not clear flags, and allocate space for data if the
2003 // packet needs it (the only packets that carry data are read
2004 // responses)
2005 pkt = new Packet(req_pkt, false, req_pkt->isRead());
2006
2007 assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
2008 pkt->hasSharers());
2009 pkt->makeTimingResponse();
2010 if (pkt->isRead()) {
2011 pkt->setDataFromBlock(blk_data, blkSize);
2012 }
2013 if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
2014 // Assume we defer a response to a read from a far-away cache
2015 // A, then later defer a ReadExcl from a cache B on the same
2016 // bus as us. We'll assert cacheResponding in both cases, but
2017 // in the latter case cacheResponding will keep the
2018 // invalidation from reaching cache A. This special response
2019 // tells cache A that it gets the block to satisfy its read,
2020 // but must immediately invalidate it.
2021 pkt->cmd = MemCmd::ReadRespWithInvalidate;
2022 }
2023 // Here we consider forward_time, paying for just forward latency and
2024 // also charging the delay provided by the xbar.
2025 // forward_time is used as send_time in next allocateWriteBuffer().
2026 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
2027 // Here we reset the timing of the packet.
2028 pkt->headerDelay = pkt->payloadDelay = 0;
2029 DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
2030 pkt->print(), forward_time);
2031 memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
2032}
2033
2034uint32_t
2035Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
2036 bool is_deferred, bool pending_inval)
2037{
2038 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2039 // deferred snoops can only happen in timing mode
2040 assert(!(is_deferred && !is_timing));
2041 // pending_inval only makes sense on deferred snoops
2042 assert(!(pending_inval && !is_deferred));
2043 assert(pkt->isRequest());
2044
2045 // the packet may get modified if we or a forwarded snooper
2046 // responds in atomic mode, so remember a few things about the
2047 // original packet up front
2048 bool invalidate = pkt->isInvalidate();
2049 bool M5_VAR_USED needs_writable = pkt->needsWritable();
2050
2051 // at the moment we could get an uncacheable write which does not
2052 // have the invalidate flag, and we need a suitable way of dealing
2053 // with this case
2054 panic_if(invalidate && pkt->req->isUncacheable(),
2055 "%s got an invalidating uncacheable snoop request %s",
2056 name(), pkt->print());
2057
2058 uint32_t snoop_delay = 0;
2059
2060 if (forwardSnoops) {
2061 // first propagate snoop upward to see if anyone above us wants to
2062 // handle it. save & restore packet src since it will get
2063 // rewritten to be relative to cpu-side bus (if any)
2064 bool alreadyResponded = pkt->cacheResponding();
2065 if (is_timing) {
2066 // copy the packet so that we can clear any flags before
2067 // forwarding it upwards, we also allocate data (passing
2068 // the pointer along in case of static data), in case
2069 // there is a snoop hit in upper levels
2070 Packet snoopPkt(pkt, true, true);
2071 snoopPkt.setExpressSnoop();
2072 // the snoop packet does not need to wait any additional
2073 // time
2074 snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2075 cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2076
2077 // add the header delay (including crossbar and snoop
2078 // delays) of the upward snoop to the snoop delay for this
2079 // cache
2080 snoop_delay += snoopPkt.headerDelay;
2081
2082 if (snoopPkt.cacheResponding()) {
2083 // cache-to-cache response from some upper cache
2084 assert(!alreadyResponded);
2085 pkt->setCacheResponding();
2086 }
2087 // upstream cache has the block, or has an outstanding
2088 // MSHR, pass the flag on
2089 if (snoopPkt.hasSharers()) {
2090 pkt->setHasSharers();
2091 }
2092 // If this request is a prefetch or clean evict and an upper level
2093 // signals block present, make sure to propagate the block
2094 // presence to the requester.
2095 if (snoopPkt.isBlockCached()) {
2096 pkt->setBlockCached();
2097 }
2098 // If the request was satisfied by snooping the cache
2099 // above, mark the original packet as satisfied too.
2100 if (snoopPkt.satisfied()) {
2101 pkt->setSatisfied();
2102 }
2103 } else {
2104 cpuSidePort->sendAtomicSnoop(pkt);
2105 if (!alreadyResponded && pkt->cacheResponding()) {
2106 // cache-to-cache response from some upper cache:
2107 // forward response to original requester
2108 assert(pkt->isResponse());
2109 }
2110 }
2111 }
2112
2113 bool respond = false;
2114 bool blk_valid = blk && blk->isValid();
2115 if (pkt->isClean()) {
2116 if (blk_valid && blk->isDirty()) {
2117 DPRINTF(CacheVerbose, "%s: packet (snoop) %s found block: %s\n",
2118 __func__, pkt->print(), blk->print());
2119 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(), pkt->id);
2120 PacketList writebacks;
2121 writebacks.push_back(wb_pkt);
2122
2123 if (is_timing) {
2124 // anything that is merely forwarded pays for the forward
2125 // latency and the delay provided by the crossbar
2126 Tick forward_time = clockEdge(forwardLatency) +
2127 pkt->headerDelay;
2128 doWritebacks(writebacks, forward_time);
2129 } else {
2130 doWritebacksAtomic(writebacks);
2131 }
2132 pkt->setSatisfied();
2133 }
2134 } else if (!blk_valid) {
2135 DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2136 pkt->print());
2137 if (is_deferred) {
2138 // we no longer have the block, and will not respond, but a
2139 // packet was allocated in MSHR::handleSnoop and we have
2140 // to delete it
2141 assert(pkt->needsResponse());
2142
2143 // we have passed the block to a cache upstream, that
2144 // cache should be responding
2145 assert(pkt->cacheResponding());
2146
2147 delete pkt;
2148 }
2149 return snoop_delay;
2150 } else {
2151 DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2152 pkt->print(), blk->print());
2153
2154 // We may end up modifying both the block state and the packet (if
2155 // we respond in atomic mode), so just figure out what to do now
2156 // and then do it later. We respond to all snoops that need
2157 // responses provided we have the block in dirty state. The
2158 // invalidation itself is taken care of below. We don't respond to
2159 // cache maintenance operations as this is done by the destination
2160 // xbar.
2161 respond = blk->isDirty() && pkt->needsResponse();
2162
2163 chatty_assert(!(isReadOnly && blk->isDirty()), "Should never have "
2164 "a dirty block in a read-only cache %s\n", name());
2165 }
2166
2167 // Invalidate any prefetch's from below that would strip write permissions
2168 // MemCmd::HardPFReq is only observed by upstream caches. After missing
2169 // above and in it's own cache, a new MemCmd::ReadReq is created that
2170 // downstream caches observe.
2171 if (pkt->mustCheckAbove()) {
2172 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2173 "from lower cache\n", pkt->getAddr(), pkt->print());
2174 pkt->setBlockCached();
2175 return snoop_delay;
2176 }
2177
2178 if (pkt->isRead() && !invalidate) {
2179 // reading without requiring the line in a writable state
2180 assert(!needs_writable);
2181 pkt->setHasSharers();
2182
2183 // if the requesting packet is uncacheable, retain the line in
2184 // the current state, otherwhise unset the writable flag,
2185 // which means we go from Modified to Owned (and will respond
2186 // below), remain in Owned (and will respond below), from
2187 // Exclusive to Shared, or remain in Shared
2188 if (!pkt->req->isUncacheable())
2189 blk->status &= ~BlkWritable;
2190 DPRINTF(Cache, "new state is %s\n", blk->print());
2191 }
2192
2193 if (respond) {
2194 // prevent anyone else from responding, cache as well as
2195 // memory, and also prevent any memory from even seeing the
2196 // request
2197 pkt->setCacheResponding();
2198 if (!pkt->isClean() && blk->isWritable()) {
2199 // inform the cache hierarchy that this cache had the line
2200 // in the Modified state so that we avoid unnecessary
2201 // invalidations (see Packet::setResponderHadWritable)
2202 pkt->setResponderHadWritable();
2203
2204 // in the case of an uncacheable request there is no point
2205 // in setting the responderHadWritable flag, but since the
2206 // recipient does not care there is no harm in doing so
2207 } else {
2208 // if the packet has needsWritable set we invalidate our
2209 // copy below and all other copies will be invalidates
2210 // through express snoops, and if needsWritable is not set
2211 // we already called setHasSharers above
2212 }
2213
2214 // if we are returning a writable and dirty (Modified) line,
2215 // we should be invalidating the line
2216 panic_if(!invalidate && !pkt->hasSharers(),
2217 "%s is passing a Modified line through %s, "
2218 "but keeping the block", name(), pkt->print());
2219
2220 if (is_timing) {
2221 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2222 } else {
2223 pkt->makeAtomicResponse();
2224 // packets such as upgrades do not actually have any data
2225 // payload
2226 if (pkt->hasData())
2227 pkt->setDataFromBlock(blk->data, blkSize);
2228 }
2229 }
2230
2231 if (!respond && is_deferred) {
2232 assert(pkt->needsResponse());
2233
2234 // if we copied the deferred packet with the intention to
2235 // respond, but are not responding, then a cache above us must
2236 // be, and we can use this as the indication of whether this
2237 // is a packet where we created a copy of the request or not
2238 if (!pkt->cacheResponding()) {
2239 delete pkt->req;
2240 }
2241
2242 delete pkt;
2243 }
2244
2245 // Do this last in case it deallocates block data or something
2246 // like that
2247 if (blk_valid && invalidate) {
2248 invalidateBlock(blk);
2249 DPRINTF(Cache, "new state is %s\n", blk->print());
2250 }
2251
2252 return snoop_delay;
2253}
2254
2255
2256void
2257Cache::recvTimingSnoopReq(PacketPtr pkt)
2258{
2259 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2260
2261 // Snoops shouldn't happen when bypassing caches
2262 assert(!system->bypassCaches());
2263
2264 // no need to snoop requests that are not in range
2265 if (!inRange(pkt->getAddr())) {
2266 return;
2267 }
2268
2269 bool is_secure = pkt->isSecure();
2270 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2271
2272 Addr blk_addr = pkt->getBlockAddr(blkSize);
2273 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2274
2275 // Update the latency cost of the snoop so that the crossbar can
2276 // account for it. Do not overwrite what other neighbouring caches
2277 // have already done, rather take the maximum. The update is
2278 // tentative, for cases where we return before an upward snoop
2279 // happens below.
2280 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2281 lookupLatency * clockPeriod());
2282
2283 // Inform request(Prefetch, CleanEvict or Writeback) from below of
2284 // MSHR hit, set setBlockCached.
2285 if (mshr && pkt->mustCheckAbove()) {
2286 DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2287 "mshr hit\n", pkt->print());
2288 pkt->setBlockCached();
2289 return;
2290 }
2291
2292 // Bypass any existing cache maintenance requests if the request
2293 // has been satisfied already (i.e., the dirty block has been
2294 // found).
2295 if (mshr && pkt->req->isCacheMaintenance() && pkt->satisfied()) {
2296 return;
2297 }
2298
2299 // Let the MSHR itself track the snoop and decide whether we want
2300 // to go ahead and do the regular cache snoop
2301 if (mshr && mshr->handleSnoop(pkt, order++)) {
2302 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2303 "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2304 mshr->print());
2305
2306 if (mshr->getNumTargets() > numTarget)
2307 warn("allocating bonus target for snoop"); //handle later
2308 return;
2309 }
2310
2311 //We also need to check the writeback buffers and handle those
2312 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2313 if (wb_entry) {
2314 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2315 pkt->getAddr(), is_secure ? "s" : "ns");
2316 // Expect to see only Writebacks and/or CleanEvicts here, both of
2317 // which should not be generated for uncacheable data.
2318 assert(!wb_entry->isUncacheable());
2319 // There should only be a single request responsible for generating
2320 // Writebacks/CleanEvicts.
2321 assert(wb_entry->getNumTargets() == 1);
2322 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2323 assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2324
2325 if (pkt->isEviction()) {
2326 // if the block is found in the write queue, set the BLOCK_CACHED
2327 // flag for Writeback/CleanEvict snoop. On return the snoop will
2328 // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2329 // any CleanEvicts from travelling down the memory hierarchy.
2330 pkt->setBlockCached();
2331 DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2332 "hit\n", __func__, pkt->print());
2333 return;
2334 }
2335
2336 // conceptually writebacks are no different to other blocks in
2337 // this cache, so the behaviour is modelled after handleSnoop,
2338 // the difference being that instead of querying the block
2339 // state to determine if it is dirty and writable, we use the
2340 // command and fields of the writeback packet
2341 bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2342 pkt->needsResponse();
2343 bool have_writable = !wb_pkt->hasSharers();
2344 bool invalidate = pkt->isInvalidate();
2345
2346 if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2347 assert(!pkt->needsWritable());
2348 pkt->setHasSharers();
2349 wb_pkt->setHasSharers();
2350 }
2351
2352 if (respond) {
2353 pkt->setCacheResponding();
2354
2355 if (have_writable) {
2356 pkt->setResponderHadWritable();
2357 }
2358
2359 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2360 false, false);
2361 }
2362
2363 if (invalidate && wb_pkt->cmd != MemCmd::WriteClean) {
2364 // Invalidation trumps our writeback... discard here
2365 // Note: markInService will remove entry from writeback buffer.
2366 markInService(wb_entry);
2367 delete wb_pkt;
2368 }
2369 }
2370
2371 // If this was a shared writeback, there may still be
2372 // other shared copies above that require invalidation.
2373 // We could be more selective and return here if the
2374 // request is non-exclusive or if the writeback is
2375 // exclusive.
2376 uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2377
2378 // Override what we did when we first saw the snoop, as we now
2379 // also have the cost of the upwards snoops to account for
2380 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2381 lookupLatency * clockPeriod());
2382}
2383
2384bool
2385Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2386{
2387 // Express snoop responses from master to slave, e.g., from L1 to L2
2388 cache->recvTimingSnoopResp(pkt);
2389 return true;
2390}
2391
2392Tick
2393Cache::recvAtomicSnoop(PacketPtr pkt)
2394{
2395 // Snoops shouldn't happen when bypassing caches
2396 assert(!system->bypassCaches());
2397
2398 // no need to snoop requests that are not in range.
2399 if (!inRange(pkt->getAddr())) {
2400 return 0;
2401 }
2402
2403 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2404 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2405 return snoop_delay + lookupLatency * clockPeriod();
2406}
2407
2408
2409QueueEntry*
2410Cache::getNextQueueEntry()
2411{
2412 // Check both MSHR queue and write buffer for potential requests,
2413 // note that null does not mean there is no request, it could
2414 // simply be that it is not ready
2415 MSHR *miss_mshr = mshrQueue.getNext();
2416 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2417
2418 // If we got a write buffer request ready, first priority is a
2419 // full write buffer, otherwise we favour the miss requests
2420 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2421 // need to search MSHR queue for conflicting earlier miss.
2422 MSHR *conflict_mshr =
2423 mshrQueue.findPending(wq_entry->blkAddr,
2424 wq_entry->isSecure);
2425
2426 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2427 // Service misses in order until conflict is cleared.
2428 return conflict_mshr;
2429
2430 // @todo Note that we ignore the ready time of the conflict here
2431 }
2432
2433 // No conflicts; issue write
2434 return wq_entry;
2435 } else if (miss_mshr) {
2436 // need to check for conflicting earlier writeback
2437 WriteQueueEntry *conflict_mshr =
2438 writeBuffer.findPending(miss_mshr->blkAddr,
2439 miss_mshr->isSecure);
2440 if (conflict_mshr) {
2441 // not sure why we don't check order here... it was in the
2442 // original code but commented out.
2443
2444 // The only way this happens is if we are
2445 // doing a write and we didn't have permissions
2446 // then subsequently saw a writeback (owned got evicted)
2447 // We need to make sure to perform the writeback first
2448 // To preserve the dirty data, then we can issue the write
2449
2450 // should we return wq_entry here instead? I.e. do we
2451 // have to flush writes in order? I don't think so... not
2452 // for Alpha anyway. Maybe for x86?
2453 return conflict_mshr;
2454
2455 // @todo Note that we ignore the ready time of the conflict here
2456 }
2457
2458 // No conflicts; issue read
2459 return miss_mshr;
2460 }
2461
2462 // fall through... no pending requests. Try a prefetch.
2463 assert(!miss_mshr && !wq_entry);
2464 if (prefetcher && mshrQueue.canPrefetch()) {
2465 // If we have a miss queue slot, we can try a prefetch
2466 PacketPtr pkt = prefetcher->getPacket();
2467 if (pkt) {
2468 Addr pf_addr = pkt->getBlockAddr(blkSize);
2469 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2470 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2471 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2472 // Update statistic on number of prefetches issued
2473 // (hwpf_mshr_misses)
2474 assert(pkt->req->masterId() < system->maxMasters());
2475 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2476
2477 // allocate an MSHR and return it, note
2478 // that we send the packet straight away, so do not
2479 // schedule the send
2480 return allocateMissBuffer(pkt, curTick(), false);
2481 } else {
2482 // free the request and packet
2483 delete pkt->req;
2484 delete pkt;
2485 }
2486 }
2487 }
2488
2489 return nullptr;
2490}
2491
2492bool
2493Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2494{
2495 if (!forwardSnoops)
2496 return false;
2497 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2498 // Writeback snoops into upper level caches to check for copies of the
2499 // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2500 // packet, the cache can inform the crossbar below of presence or absence
2501 // of the block.
2502 if (is_timing) {
2503 Packet snoop_pkt(pkt, true, false);
2504 snoop_pkt.setExpressSnoop();
2505 // Assert that packet is either Writeback or CleanEvict and not a
2506 // prefetch request because prefetch requests need an MSHR and may
2507 // generate a snoop response.
2508 assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2509 snoop_pkt.senderState = nullptr;
2510 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2511 // Writeback/CleanEvict snoops do not generate a snoop response.
2512 assert(!(snoop_pkt.cacheResponding()));
2513 return snoop_pkt.isBlockCached();
2514 } else {
2515 cpuSidePort->sendAtomicSnoop(pkt);
2516 return pkt->isBlockCached();
2517 }
2518}
2519
2520Tick
2521Cache::nextQueueReadyTime() const
2522{
2523 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2524 writeBuffer.nextReadyTime());
2525
2526 // Don't signal prefetch ready time if no MSHRs available
2527 // Will signal once enoguh MSHRs are deallocated
2528 if (prefetcher && mshrQueue.canPrefetch()) {
2529 nextReady = std::min(nextReady,
2530 prefetcher->nextPrefetchReadyTime());
2531 }
2532
2533 return nextReady;
2534}
2535
2536bool
2537Cache::sendMSHRQueuePacket(MSHR* mshr)
2538{
2539 assert(mshr);
2540
2541 // use request from 1st target
2542 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2543
2544 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2545
2546 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2547
2548 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2549 // we should never have hardware prefetches to allocated
2550 // blocks
2551 assert(blk == nullptr);
2552
2553 // We need to check the caches above us to verify that
2554 // they don't have a copy of this block in the dirty state
2555 // at the moment. Without this check we could get a stale
2556 // copy from memory that might get used in place of the
2557 // dirty one.
2558 Packet snoop_pkt(tgt_pkt, true, false);
2559 snoop_pkt.setExpressSnoop();
2560 // We are sending this packet upwards, but if it hits we will
2561 // get a snoop response that we end up treating just like a
2562 // normal response, hence it needs the MSHR as its sender
2563 // state
2564 snoop_pkt.senderState = mshr;
2565 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2566
2567 // Check to see if the prefetch was squashed by an upper cache (to
2568 // prevent us from grabbing the line) or if a Check to see if a
2569 // writeback arrived between the time the prefetch was placed in
2570 // the MSHRs and when it was selected to be sent or if the
2571 // prefetch was squashed by an upper cache.
2572
2573 // It is important to check cacheResponding before
2574 // prefetchSquashed. If another cache has committed to
2575 // responding, it will be sending a dirty response which will
2576 // arrive at the MSHR allocated for this request. Checking the
2577 // prefetchSquash first may result in the MSHR being
2578 // prematurely deallocated.
2579 if (snoop_pkt.cacheResponding()) {
2580 auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2581 assert(r.second);
2582
2583 // if we are getting a snoop response with no sharers it
2584 // will be allocated as Modified
2585 bool pending_modified_resp = !snoop_pkt.hasSharers();
2586 markInService(mshr, pending_modified_resp);
2587
2588 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2589 " %#x (%s) hit\n",
2590 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2591 return false;
2592 }
2593
2594 if (snoop_pkt.isBlockCached()) {
2595 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2596 "Deallocating mshr target %#x.\n",
2597 mshr->blkAddr);
2598
2599 // Deallocate the mshr target
2600 if (mshrQueue.forceDeallocateTarget(mshr)) {
2601 // Clear block if this deallocation resulted freed an
2602 // mshr when all had previously been utilized
2603 clearBlocked(Blocked_NoMSHRs);
2604 }
2605
2606 // given that no response is expected, delete Request and Packet
2607 delete tgt_pkt->req;
2608 delete tgt_pkt;
2609
2610 return false;
2611 }
2612 }
2613
2614 // either a prefetch that is not present upstream, or a normal
2615 // MSHR request, proceed to get the packet to send downstream
2616 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2617
2618 mshr->isForward = (pkt == nullptr);
2619
2620 if (mshr->isForward) {
2621 // not a cache block request, but a response is expected
2622 // make copy of current packet to forward, keep current
2623 // copy for response handling
2624 pkt = new Packet(tgt_pkt, false, true);
2625 assert(!pkt->isWrite());
2626 }
2627
2628 // play it safe and append (rather than set) the sender state,
2629 // as forwarded packets may already have existing state
2630 pkt->pushSenderState(mshr);
2631
2632 if (pkt->isClean() && blk && blk->isDirty()) {
2633 // A cache clean opearation is looking for a dirty block. Mark
2634 // the packet so that the destination xbar can determine that
2635 // there will be a follow-up write packet as well.
2636 pkt->setSatisfied();
2637 }
2638
2639 if (!memSidePort->sendTimingReq(pkt)) {
2640 // we are awaiting a retry, but we
2641 // delete the packet and will be creating a new packet
2642 // when we get the opportunity
2643 delete pkt;
2644
2645 // note that we have now masked any requestBus and
2646 // schedSendEvent (we will wait for a retry before
2647 // doing anything), and this is so even if we do not
2648 // care about this packet and might override it before
2649 // it gets retried
2650 return true;
2651 } else {
2652 // As part of the call to sendTimingReq the packet is
2653 // forwarded to all neighbouring caches (and any caches
2654 // above them) as a snoop. Thus at this point we know if
2655 // any of the neighbouring caches are responding, and if
2656 // so, we know it is dirty, and we can determine if it is
2657 // being passed as Modified, making our MSHR the ordering
2658 // point
2659 bool pending_modified_resp = !pkt->hasSharers() &&
2660 pkt->cacheResponding();
2661 markInService(mshr, pending_modified_resp);
2662 if (pkt->isClean() && blk && blk->isDirty()) {
2663 // A cache clean opearation is looking for a dirty
2664 // block. If a dirty block is encountered a WriteClean
2665 // will update any copies to the path to the memory
2666 // until the point of reference.
2667 DPRINTF(CacheVerbose, "%s: packet %s found block: %s\n",
2668 __func__, pkt->print(), blk->print());
2669 PacketPtr wb_pkt = writecleanBlk(blk, pkt->req->getDest(),
2670 pkt->id);
2671 PacketList writebacks;
2672 writebacks.push_back(wb_pkt);
2673 doWritebacks(writebacks, 0);
2674 }
2675
2676 return false;
2677 }
2678}
2679
2680bool
2681Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2682{
2683 assert(wq_entry);
2684
2685 // always a single target for write queue entries
2686 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2687
2688 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2689
2690 // forward as is, both for evictions and uncacheable writes
2691 if (!memSidePort->sendTimingReq(tgt_pkt)) {
2692 // note that we have now masked any requestBus and
2693 // schedSendEvent (we will wait for a retry before
2694 // doing anything), and this is so even if we do not
2695 // care about this packet and might override it before
2696 // it gets retried
2697 return true;
2698 } else {
2699 markInService(wq_entry);
2700 return false;
2701 }
2702}
2703
2704void
2705Cache::serialize(CheckpointOut &cp) const
2706{
2707 bool dirty(isDirty());
2708
2709 if (dirty) {
2710 warn("*** The cache still contains dirty data. ***\n");
2711 warn(" Make sure to drain the system using the correct flags.\n");
2712 warn(" This checkpoint will not restore correctly and dirty data "
2713 " in the cache will be lost!\n");
2714 }
2715
2716 // Since we don't checkpoint the data in the cache, any dirty data
2717 // will be lost when restoring from a checkpoint of a system that
2718 // wasn't drained properly. Flag the checkpoint as invalid if the
2719 // cache contains dirty data.
2720 bool bad_checkpoint(dirty);
2721 SERIALIZE_SCALAR(bad_checkpoint);
2722}
2723
2724void
2725Cache::unserialize(CheckpointIn &cp)
2726{
2727 bool bad_checkpoint;
2728 UNSERIALIZE_SCALAR(bad_checkpoint);
2729 if (bad_checkpoint) {
2730 fatal("Restoring from checkpoints with dirty caches is not supported "
2731 "in the classic memory system. Please remove any caches or "
2732 " drain them properly before taking checkpoints.\n");
2733 }
2734}
2735
2736///////////////
2737//
2738// CpuSidePort
2739//
2740///////////////
2741
2742AddrRangeList
2743Cache::CpuSidePort::getAddrRanges() const
2744{
2745 return cache->getAddrRanges();
2746}
2747
2748bool
2749Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2750{
2751 assert(!cache->system->bypassCaches());
2752
2753 // always let express snoop packets through if even if blocked
2754 if (pkt->isExpressSnoop()) {
2755 return true;
2756 } else if (isBlocked() || mustSendRetry) {
2757 // either already committed to send a retry, or blocked
2758 mustSendRetry = true;
2759 return false;
2760 }
2761 mustSendRetry = false;
2762 return true;
2763}
2764
2765bool
2766Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2767{
2768 assert(!cache->system->bypassCaches());
2769
2770 // always let express snoop packets through if even if blocked
|