1/*
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 * Dave Greene
43 * Nathan Binkert
44 * Steve Reinhardt
45 * Ron Dreslinski
46 * Andreas Sandberg
47 */
48
49/**
50 * @file
51 * Cache definitions.
52 */
53
54#include "mem/cache/cache.hh"
55
56#include "base/logging.hh"
57#include "base/types.hh"
58#include "debug/Cache.hh"
59#include "debug/CachePort.hh"
60#include "debug/CacheTags.hh"
61#include "debug/CacheVerbose.hh"
62#include "mem/cache/blk.hh"
63#include "mem/cache/mshr.hh"
64#include "mem/cache/prefetch/base.hh"
65#include "sim/sim_exit.hh"
66
67Cache::Cache(const CacheParams *p)
68 : BaseCache(p, p->system->cacheLineSize()),
69 tags(p->tags),
70 prefetcher(p->prefetcher),
71 doFastWrites(true),
72 prefetchOnAccess(p->prefetch_on_access),
73 clusivity(p->clusivity),
74 writebackClean(p->writeback_clean),
75 tempBlockWriteback(nullptr),
76 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
77 name(), false,
78 EventBase::Delayed_Writeback_Pri)
79{
80 tempBlock = new CacheBlk();
81 tempBlock->data = new uint8_t[blkSize];
82
83 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
84 "CpuSidePort");
85 memSidePort = new MemSidePort(p->name + ".mem_side", this,
86 "MemSidePort");
87
88 tags->setCache(this);
89 if (prefetcher)
90 prefetcher->setCache(this);
91}
92
93Cache::~Cache()
94{
95 delete [] tempBlock->data;
96 delete tempBlock;
97
98 delete cpuSidePort;
99 delete memSidePort;
100}
101
102void
103Cache::regStats()
104{
105 BaseCache::regStats();
106}
107
108void
109Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
110{
111 assert(pkt->isRequest());
112
113 uint64_t overwrite_val;
114 bool overwrite_mem;
115 uint64_t condition_val64;
116 uint32_t condition_val32;
117
118 int offset = tags->extractBlkOffset(pkt->getAddr());
119 uint8_t *blk_data = blk->data + offset;
120
121 assert(sizeof(uint64_t) >= pkt->getSize());
122
123 overwrite_mem = true;
124 // keep a copy of our possible write value, and copy what is at the
125 // memory address into the packet
126 pkt->writeData((uint8_t *)&overwrite_val);
127 pkt->setData(blk_data);
128
129 if (pkt->req->isCondSwap()) {
130 if (pkt->getSize() == sizeof(uint64_t)) {
131 condition_val64 = pkt->req->getExtraData();
132 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
133 sizeof(uint64_t));
134 } else if (pkt->getSize() == sizeof(uint32_t)) {
135 condition_val32 = (uint32_t)pkt->req->getExtraData();
136 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
137 sizeof(uint32_t));
138 } else
139 panic("Invalid size for conditional read/write\n");
140 }
141
142 if (overwrite_mem) {
143 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
144 blk->status |= BlkDirty;
145 }
146}
147
148
149void
150Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
151 bool deferred_response, bool pending_downgrade)
152{
153 assert(pkt->isRequest());
154
155 assert(blk && blk->isValid());
156 // Occasionally this is not true... if we are a lower-level cache
157 // satisfying a string of Read and ReadEx requests from
158 // upper-level caches, a Read will mark the block as shared but we
159 // can satisfy a following ReadEx anyway since we can rely on the
160 // Read requester(s) to have buffered the ReadEx snoop and to
161 // invalidate their blocks after receiving them.
162 // assert(!pkt->needsWritable() || blk->isWritable());
163 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
164
165 // Check RMW operations first since both isRead() and
166 // isWrite() will be true for them
167 if (pkt->cmd == MemCmd::SwapReq) {
168 cmpAndSwap(blk, pkt);
169 } else if (pkt->isWrite()) {
170 // we have the block in a writable state and can go ahead,
171 // note that the line may be also be considered writable in
172 // downstream caches along the path to memory, but always
173 // Exclusive, and never Modified
174 assert(blk->isWritable());
175 // Write or WriteLine at the first cache with block in writable state
176 if (blk->checkWrite(pkt)) {
177 pkt->writeDataToBlock(blk->data, blkSize);
178 }
179 // Always mark the line as dirty (and thus transition to the
180 // Modified state) even if we are a failed StoreCond so we
181 // supply data to any snoops that have appended themselves to
182 // this cache before knowing the store will fail.
183 blk->status |= BlkDirty;
184 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
185 } else if (pkt->isRead()) {
186 if (pkt->isLLSC()) {
187 blk->trackLoadLocked(pkt);
188 }
189
190 // all read responses have a data payload
191 assert(pkt->hasRespData());
192 pkt->setDataFromBlock(blk->data, blkSize);
193
194 // determine if this read is from a (coherent) cache or not
195 if (pkt->fromCache()) {
196 assert(pkt->getSize() == blkSize);
197 // special handling for coherent block requests from
198 // upper-level caches
199 if (pkt->needsWritable()) {
200 // sanity check
201 assert(pkt->cmd == MemCmd::ReadExReq ||
202 pkt->cmd == MemCmd::SCUpgradeFailReq);
203 assert(!pkt->hasSharers());
204
205 // if we have a dirty copy, make sure the recipient
206 // keeps it marked dirty (in the modified state)
207 if (blk->isDirty()) {
208 pkt->setCacheResponding();
209 blk->status &= ~BlkDirty;
210 }
211 } else if (blk->isWritable() && !pending_downgrade &&
212 !pkt->hasSharers() &&
213 pkt->cmd != MemCmd::ReadCleanReq) {
214 // we can give the requester a writable copy on a read
215 // request if:
216 // - we have a writable copy at this level (& below)
217 // - we don't have a pending snoop from below
218 // signaling another read request
219 // - no other cache above has a copy (otherwise it
220 // would have set hasSharers flag when
221 // snooping the packet)
222 // - the read has explicitly asked for a clean
223 // copy of the line
224 if (blk->isDirty()) {
225 // special considerations if we're owner:
226 if (!deferred_response) {
227 // respond with the line in Modified state
228 // (cacheResponding set, hasSharers not set)
229 pkt->setCacheResponding();
230
231 // if this cache is mostly inclusive, we
232 // keep the block in the Exclusive state,
233 // and pass it upwards as Modified
234 // (writable and dirty), hence we have
235 // multiple caches, all on the same path
236 // towards memory, all considering the
237 // same block writable, but only one
238 // considering it Modified
239
240 // we get away with multiple caches (on
241 // the same path to memory) considering
242 // the block writeable as we always enter
243 // the cache hierarchy through a cache,
244 // and first snoop upwards in all other
245 // branches
246 blk->status &= ~BlkDirty;
247 } else {
248 // if we're responding after our own miss,
249 // there's a window where the recipient didn't
250 // know it was getting ownership and may not
251 // have responded to snoops correctly, so we
252 // have to respond with a shared line
253 pkt->setHasSharers();
254 }
255 }
256 } else {
257 // otherwise only respond with a shared copy
258 pkt->setHasSharers();
259 }
260 }
261 } else if (pkt->isUpgrade()) {
262 // sanity check
263 assert(!pkt->hasSharers());
264
265 if (blk->isDirty()) {
266 // we were in the Owned state, and a cache above us that
267 // has the line in Shared state needs to be made aware
268 // that the data it already has is in fact dirty
269 pkt->setCacheResponding();
270 blk->status &= ~BlkDirty;
271 }
272 } else {
273 assert(pkt->isInvalidate());
274 invalidateBlock(blk);
275 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
276 pkt->print());
277 }
278}
279
280/////////////////////////////////////////////////////
281//
282// Access path: requests coming in from the CPU side
283//
284/////////////////////////////////////////////////////
285
286bool
287Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
288 PacketList &writebacks)
289{
290 // sanity check
291 assert(pkt->isRequest());
292
293 chatty_assert(!(isReadOnly && pkt->isWrite()),
294 "Should never see a write in a read-only cache %s\n",
295 name());
296
297 DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
298
299 if (pkt->req->isUncacheable()) {
300 DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
301
302 // flush and invalidate any existing block
303 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
304 if (old_blk && old_blk->isValid()) {
305 if (old_blk->isDirty() || writebackClean)
306 writebacks.push_back(writebackBlk(old_blk));
307 else
308 writebacks.push_back(cleanEvictBlk(old_blk));
309 invalidateBlock(old_blk);
310 }
311
312 blk = nullptr;
313 // lookupLatency is the latency in case the request is uncacheable.
314 lat = lookupLatency;
315 return false;
316 }
317
318 // Here lat is the value passed as parameter to accessBlock() function
319 // that can modify its value.
320 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
321
322 DPRINTF(Cache, "%s %s\n", pkt->print(),
323 blk ? "hit " + blk->print() : "miss");
324
325
326 if (pkt->isEviction()) {
327 // We check for presence of block in above caches before issuing
328 // Writeback or CleanEvict to write buffer. Therefore the only
329 // possible cases can be of a CleanEvict packet coming from above
330 // encountering a Writeback generated in this cache peer cache and
331 // waiting in the write buffer. Cases of upper level peer caches
332 // generating CleanEvict and Writeback or simply CleanEvict and
333 // CleanEvict almost simultaneously will be caught by snoops sent out
334 // by crossbar.
335 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
336 pkt->isSecure());
337 if (wb_entry) {
338 assert(wb_entry->getNumTargets() == 1);
339 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
340 assert(wbPkt->isWriteback());
341
342 if (pkt->isCleanEviction()) {
343 // The CleanEvict and WritebackClean snoops into other
344 // peer caches of the same level while traversing the
345 // crossbar. If a copy of the block is found, the
346 // packet is deleted in the crossbar. Hence, none of
347 // the other upper level caches connected to this
348 // cache have the block, so we can clear the
349 // BLOCK_CACHED flag in the Writeback if set and
350 // discard the CleanEvict by returning true.
351 wbPkt->clearBlockCached();
352 return true;
353 } else {
354 assert(pkt->cmd == MemCmd::WritebackDirty);
355 // Dirty writeback from above trumps our clean
356 // writeback... discard here
357 // Note: markInService will remove entry from writeback buffer.
358 markInService(wb_entry);
359 delete wbPkt;
360 }
361 }
362 }
363
364 // Writeback handling is special case. We can write the block into
365 // the cache without having a writeable copy (or any copy at all).
366 if (pkt->isWriteback()) {
367 assert(blkSize == pkt->getSize());
368
369 // we could get a clean writeback while we are having
370 // outstanding accesses to a block, do the simple thing for
371 // now and drop the clean writeback so that we do not upset
372 // any ordering/decisions about ownership already taken
373 if (pkt->cmd == MemCmd::WritebackClean &&
374 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
375 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
376 "dropping\n", pkt->getAddr());
377 return true;
378 }
379
380 if (blk == nullptr) {
381 // need to do a replacement
382 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
383 if (blk == nullptr) {
384 // no replaceable block available: give up, fwd to next level.
385 incMissCount(pkt);
386 return false;
387 }
388 tags->insertBlock(pkt, blk);
389
390 blk->status = (BlkValid | BlkReadable);
391 if (pkt->isSecure()) {
392 blk->status |= BlkSecure;
393 }
394 }
395 // only mark the block dirty if we got a writeback command,
396 // and leave it as is for a clean writeback
397 if (pkt->cmd == MemCmd::WritebackDirty) {
398 blk->status |= BlkDirty;
399 }
400 // if the packet does not have sharers, it is passing
401 // writable, and we got the writeback in Modified or Exclusive
402 // state, if not we are in the Owned or Shared state
403 if (!pkt->hasSharers()) {
404 blk->status |= BlkWritable;
405 }
406 // nothing else to do; writeback doesn't expect response
407 assert(!pkt->needsResponse());
408 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
409 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
410 incHitCount(pkt);
411 return true;
412 } else if (pkt->cmd == MemCmd::CleanEvict) {
413 if (blk != nullptr) {
414 // Found the block in the tags, need to stop CleanEvict from
415 // propagating further down the hierarchy. Returning true will
416 // treat the CleanEvict like a satisfied write request and delete
417 // it.
418 return true;
419 }
420 // We didn't find the block here, propagate the CleanEvict further
421 // down the memory hierarchy. Returning false will treat the CleanEvict
422 // like a Writeback which could not find a replaceable block so has to
423 // go to next level.
424 return false;
425 } else if (pkt->cmd == MemCmd::WriteClean) {
426 // WriteClean handling is a special case. We can allocate a
427 // block directly if it doesn't exist and we can update the
428 // block immediately. The WriteClean transfers the ownership
429 // of the block as well.
430 assert(blkSize == pkt->getSize());
431
432 if (!blk) {
| 1/*
2 * Copyright (c) 2010-2016 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2002-2005 The Regents of The University of Michigan
15 * Copyright (c) 2010,2015 Advanced Micro Devices, Inc.
16 * All rights reserved.
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are
20 * met: redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer;
22 * redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution;
25 * neither the name of the copyright holders nor the names of its
26 * contributors may be used to endorse or promote products derived from
27 * this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
30 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
31 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
32 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
33 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
35 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
36 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
37 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
38 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 *
41 * Authors: Erik Hallnor
42 * Dave Greene
43 * Nathan Binkert
44 * Steve Reinhardt
45 * Ron Dreslinski
46 * Andreas Sandberg
47 */
48
49/**
50 * @file
51 * Cache definitions.
52 */
53
54#include "mem/cache/cache.hh"
55
56#include "base/logging.hh"
57#include "base/types.hh"
58#include "debug/Cache.hh"
59#include "debug/CachePort.hh"
60#include "debug/CacheTags.hh"
61#include "debug/CacheVerbose.hh"
62#include "mem/cache/blk.hh"
63#include "mem/cache/mshr.hh"
64#include "mem/cache/prefetch/base.hh"
65#include "sim/sim_exit.hh"
66
67Cache::Cache(const CacheParams *p)
68 : BaseCache(p, p->system->cacheLineSize()),
69 tags(p->tags),
70 prefetcher(p->prefetcher),
71 doFastWrites(true),
72 prefetchOnAccess(p->prefetch_on_access),
73 clusivity(p->clusivity),
74 writebackClean(p->writeback_clean),
75 tempBlockWriteback(nullptr),
76 writebackTempBlockAtomicEvent([this]{ writebackTempBlockAtomic(); },
77 name(), false,
78 EventBase::Delayed_Writeback_Pri)
79{
80 tempBlock = new CacheBlk();
81 tempBlock->data = new uint8_t[blkSize];
82
83 cpuSidePort = new CpuSidePort(p->name + ".cpu_side", this,
84 "CpuSidePort");
85 memSidePort = new MemSidePort(p->name + ".mem_side", this,
86 "MemSidePort");
87
88 tags->setCache(this);
89 if (prefetcher)
90 prefetcher->setCache(this);
91}
92
93Cache::~Cache()
94{
95 delete [] tempBlock->data;
96 delete tempBlock;
97
98 delete cpuSidePort;
99 delete memSidePort;
100}
101
102void
103Cache::regStats()
104{
105 BaseCache::regStats();
106}
107
108void
109Cache::cmpAndSwap(CacheBlk *blk, PacketPtr pkt)
110{
111 assert(pkt->isRequest());
112
113 uint64_t overwrite_val;
114 bool overwrite_mem;
115 uint64_t condition_val64;
116 uint32_t condition_val32;
117
118 int offset = tags->extractBlkOffset(pkt->getAddr());
119 uint8_t *blk_data = blk->data + offset;
120
121 assert(sizeof(uint64_t) >= pkt->getSize());
122
123 overwrite_mem = true;
124 // keep a copy of our possible write value, and copy what is at the
125 // memory address into the packet
126 pkt->writeData((uint8_t *)&overwrite_val);
127 pkt->setData(blk_data);
128
129 if (pkt->req->isCondSwap()) {
130 if (pkt->getSize() == sizeof(uint64_t)) {
131 condition_val64 = pkt->req->getExtraData();
132 overwrite_mem = !std::memcmp(&condition_val64, blk_data,
133 sizeof(uint64_t));
134 } else if (pkt->getSize() == sizeof(uint32_t)) {
135 condition_val32 = (uint32_t)pkt->req->getExtraData();
136 overwrite_mem = !std::memcmp(&condition_val32, blk_data,
137 sizeof(uint32_t));
138 } else
139 panic("Invalid size for conditional read/write\n");
140 }
141
142 if (overwrite_mem) {
143 std::memcpy(blk_data, &overwrite_val, pkt->getSize());
144 blk->status |= BlkDirty;
145 }
146}
147
148
149void
150Cache::satisfyRequest(PacketPtr pkt, CacheBlk *blk,
151 bool deferred_response, bool pending_downgrade)
152{
153 assert(pkt->isRequest());
154
155 assert(blk && blk->isValid());
156 // Occasionally this is not true... if we are a lower-level cache
157 // satisfying a string of Read and ReadEx requests from
158 // upper-level caches, a Read will mark the block as shared but we
159 // can satisfy a following ReadEx anyway since we can rely on the
160 // Read requester(s) to have buffered the ReadEx snoop and to
161 // invalidate their blocks after receiving them.
162 // assert(!pkt->needsWritable() || blk->isWritable());
163 assert(pkt->getOffset(blkSize) + pkt->getSize() <= blkSize);
164
165 // Check RMW operations first since both isRead() and
166 // isWrite() will be true for them
167 if (pkt->cmd == MemCmd::SwapReq) {
168 cmpAndSwap(blk, pkt);
169 } else if (pkt->isWrite()) {
170 // we have the block in a writable state and can go ahead,
171 // note that the line may be also be considered writable in
172 // downstream caches along the path to memory, but always
173 // Exclusive, and never Modified
174 assert(blk->isWritable());
175 // Write or WriteLine at the first cache with block in writable state
176 if (blk->checkWrite(pkt)) {
177 pkt->writeDataToBlock(blk->data, blkSize);
178 }
179 // Always mark the line as dirty (and thus transition to the
180 // Modified state) even if we are a failed StoreCond so we
181 // supply data to any snoops that have appended themselves to
182 // this cache before knowing the store will fail.
183 blk->status |= BlkDirty;
184 DPRINTF(CacheVerbose, "%s for %s (write)\n", __func__, pkt->print());
185 } else if (pkt->isRead()) {
186 if (pkt->isLLSC()) {
187 blk->trackLoadLocked(pkt);
188 }
189
190 // all read responses have a data payload
191 assert(pkt->hasRespData());
192 pkt->setDataFromBlock(blk->data, blkSize);
193
194 // determine if this read is from a (coherent) cache or not
195 if (pkt->fromCache()) {
196 assert(pkt->getSize() == blkSize);
197 // special handling for coherent block requests from
198 // upper-level caches
199 if (pkt->needsWritable()) {
200 // sanity check
201 assert(pkt->cmd == MemCmd::ReadExReq ||
202 pkt->cmd == MemCmd::SCUpgradeFailReq);
203 assert(!pkt->hasSharers());
204
205 // if we have a dirty copy, make sure the recipient
206 // keeps it marked dirty (in the modified state)
207 if (blk->isDirty()) {
208 pkt->setCacheResponding();
209 blk->status &= ~BlkDirty;
210 }
211 } else if (blk->isWritable() && !pending_downgrade &&
212 !pkt->hasSharers() &&
213 pkt->cmd != MemCmd::ReadCleanReq) {
214 // we can give the requester a writable copy on a read
215 // request if:
216 // - we have a writable copy at this level (& below)
217 // - we don't have a pending snoop from below
218 // signaling another read request
219 // - no other cache above has a copy (otherwise it
220 // would have set hasSharers flag when
221 // snooping the packet)
222 // - the read has explicitly asked for a clean
223 // copy of the line
224 if (blk->isDirty()) {
225 // special considerations if we're owner:
226 if (!deferred_response) {
227 // respond with the line in Modified state
228 // (cacheResponding set, hasSharers not set)
229 pkt->setCacheResponding();
230
231 // if this cache is mostly inclusive, we
232 // keep the block in the Exclusive state,
233 // and pass it upwards as Modified
234 // (writable and dirty), hence we have
235 // multiple caches, all on the same path
236 // towards memory, all considering the
237 // same block writable, but only one
238 // considering it Modified
239
240 // we get away with multiple caches (on
241 // the same path to memory) considering
242 // the block writeable as we always enter
243 // the cache hierarchy through a cache,
244 // and first snoop upwards in all other
245 // branches
246 blk->status &= ~BlkDirty;
247 } else {
248 // if we're responding after our own miss,
249 // there's a window where the recipient didn't
250 // know it was getting ownership and may not
251 // have responded to snoops correctly, so we
252 // have to respond with a shared line
253 pkt->setHasSharers();
254 }
255 }
256 } else {
257 // otherwise only respond with a shared copy
258 pkt->setHasSharers();
259 }
260 }
261 } else if (pkt->isUpgrade()) {
262 // sanity check
263 assert(!pkt->hasSharers());
264
265 if (blk->isDirty()) {
266 // we were in the Owned state, and a cache above us that
267 // has the line in Shared state needs to be made aware
268 // that the data it already has is in fact dirty
269 pkt->setCacheResponding();
270 blk->status &= ~BlkDirty;
271 }
272 } else {
273 assert(pkt->isInvalidate());
274 invalidateBlock(blk);
275 DPRINTF(CacheVerbose, "%s for %s (invalidation)\n", __func__,
276 pkt->print());
277 }
278}
279
280/////////////////////////////////////////////////////
281//
282// Access path: requests coming in from the CPU side
283//
284/////////////////////////////////////////////////////
285
286bool
287Cache::access(PacketPtr pkt, CacheBlk *&blk, Cycles &lat,
288 PacketList &writebacks)
289{
290 // sanity check
291 assert(pkt->isRequest());
292
293 chatty_assert(!(isReadOnly && pkt->isWrite()),
294 "Should never see a write in a read-only cache %s\n",
295 name());
296
297 DPRINTF(CacheVerbose, "%s for %s\n", __func__, pkt->print());
298
299 if (pkt->req->isUncacheable()) {
300 DPRINTF(Cache, "uncacheable: %s\n", pkt->print());
301
302 // flush and invalidate any existing block
303 CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure()));
304 if (old_blk && old_blk->isValid()) {
305 if (old_blk->isDirty() || writebackClean)
306 writebacks.push_back(writebackBlk(old_blk));
307 else
308 writebacks.push_back(cleanEvictBlk(old_blk));
309 invalidateBlock(old_blk);
310 }
311
312 blk = nullptr;
313 // lookupLatency is the latency in case the request is uncacheable.
314 lat = lookupLatency;
315 return false;
316 }
317
318 // Here lat is the value passed as parameter to accessBlock() function
319 // that can modify its value.
320 blk = tags->accessBlock(pkt->getAddr(), pkt->isSecure(), lat);
321
322 DPRINTF(Cache, "%s %s\n", pkt->print(),
323 blk ? "hit " + blk->print() : "miss");
324
325
326 if (pkt->isEviction()) {
327 // We check for presence of block in above caches before issuing
328 // Writeback or CleanEvict to write buffer. Therefore the only
329 // possible cases can be of a CleanEvict packet coming from above
330 // encountering a Writeback generated in this cache peer cache and
331 // waiting in the write buffer. Cases of upper level peer caches
332 // generating CleanEvict and Writeback or simply CleanEvict and
333 // CleanEvict almost simultaneously will be caught by snoops sent out
334 // by crossbar.
335 WriteQueueEntry *wb_entry = writeBuffer.findMatch(pkt->getAddr(),
336 pkt->isSecure());
337 if (wb_entry) {
338 assert(wb_entry->getNumTargets() == 1);
339 PacketPtr wbPkt = wb_entry->getTarget()->pkt;
340 assert(wbPkt->isWriteback());
341
342 if (pkt->isCleanEviction()) {
343 // The CleanEvict and WritebackClean snoops into other
344 // peer caches of the same level while traversing the
345 // crossbar. If a copy of the block is found, the
346 // packet is deleted in the crossbar. Hence, none of
347 // the other upper level caches connected to this
348 // cache have the block, so we can clear the
349 // BLOCK_CACHED flag in the Writeback if set and
350 // discard the CleanEvict by returning true.
351 wbPkt->clearBlockCached();
352 return true;
353 } else {
354 assert(pkt->cmd == MemCmd::WritebackDirty);
355 // Dirty writeback from above trumps our clean
356 // writeback... discard here
357 // Note: markInService will remove entry from writeback buffer.
358 markInService(wb_entry);
359 delete wbPkt;
360 }
361 }
362 }
363
364 // Writeback handling is special case. We can write the block into
365 // the cache without having a writeable copy (or any copy at all).
366 if (pkt->isWriteback()) {
367 assert(blkSize == pkt->getSize());
368
369 // we could get a clean writeback while we are having
370 // outstanding accesses to a block, do the simple thing for
371 // now and drop the clean writeback so that we do not upset
372 // any ordering/decisions about ownership already taken
373 if (pkt->cmd == MemCmd::WritebackClean &&
374 mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) {
375 DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, "
376 "dropping\n", pkt->getAddr());
377 return true;
378 }
379
380 if (blk == nullptr) {
381 // need to do a replacement
382 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks);
383 if (blk == nullptr) {
384 // no replaceable block available: give up, fwd to next level.
385 incMissCount(pkt);
386 return false;
387 }
388 tags->insertBlock(pkt, blk);
389
390 blk->status = (BlkValid | BlkReadable);
391 if (pkt->isSecure()) {
392 blk->status |= BlkSecure;
393 }
394 }
395 // only mark the block dirty if we got a writeback command,
396 // and leave it as is for a clean writeback
397 if (pkt->cmd == MemCmd::WritebackDirty) {
398 blk->status |= BlkDirty;
399 }
400 // if the packet does not have sharers, it is passing
401 // writable, and we got the writeback in Modified or Exclusive
402 // state, if not we are in the Owned or Shared state
403 if (!pkt->hasSharers()) {
404 blk->status |= BlkWritable;
405 }
406 // nothing else to do; writeback doesn't expect response
407 assert(!pkt->needsResponse());
408 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
409 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
410 incHitCount(pkt);
411 return true;
412 } else if (pkt->cmd == MemCmd::CleanEvict) {
413 if (blk != nullptr) {
414 // Found the block in the tags, need to stop CleanEvict from
415 // propagating further down the hierarchy. Returning true will
416 // treat the CleanEvict like a satisfied write request and delete
417 // it.
418 return true;
419 }
420 // We didn't find the block here, propagate the CleanEvict further
421 // down the memory hierarchy. Returning false will treat the CleanEvict
422 // like a Writeback which could not find a replaceable block so has to
423 // go to next level.
424 return false;
425 } else if (pkt->cmd == MemCmd::WriteClean) {
426 // WriteClean handling is a special case. We can allocate a
427 // block directly if it doesn't exist and we can update the
428 // block immediately. The WriteClean transfers the ownership
429 // of the block as well.
430 assert(blkSize == pkt->getSize());
431
432 if (!blk) {
|
433 // a writeback that misses needs to allocate a new block
434 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
435 writebacks);
436 if (!blk) {
437 // no replaceable block available: give up, fwd to
438 // next level.
439 incMissCount(pkt);
| 433 if (pkt->writeThrough()) {
434 // if this is a write through packet, we don't try to
435 // allocate if the block is not present
|
440 return false;
| 436 return false;
|
441 }
442 tags->insertBlock(pkt, blk);
| 437 } else {
438 // a writeback that misses needs to allocate a new block
439 blk = allocateBlock(pkt->getAddr(), pkt->isSecure(),
440 writebacks);
441 if (!blk) {
442 // no replaceable block available: give up, fwd to
443 // next level.
444 incMissCount(pkt);
445 return false;
446 }
447 tags->insertBlock(pkt, blk);
|
443
| 448
|
444 blk->status = (BlkValid | BlkReadable);
445 if (pkt->isSecure()) {
446 blk->status |= BlkSecure;
| 449 blk->status = (BlkValid | BlkReadable);
450 if (pkt->isSecure()) {
451 blk->status |= BlkSecure;
452 }
|
447 }
448 }
449
450 // at this point either this is a writeback or a write-through
451 // write clean operation and the block is already in this
452 // cache, we need to update the data and the block flags
453 assert(blk);
| 453 }
454 }
455
456 // at this point either this is a writeback or a write-through
457 // write clean operation and the block is already in this
458 // cache, we need to update the data and the block flags
459 assert(blk);
|
454 blk->status |= BlkDirty;
| 460 if (!pkt->writeThrough()) {
461 blk->status |= BlkDirty;
462 }
|
455 // nothing else to do; writeback doesn't expect response
456 assert(!pkt->needsResponse());
457 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
458 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
459
460 incHitCount(pkt);
461 // populate the time when the block will be ready to access.
462 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
463 pkt->payloadDelay;
| 463 // nothing else to do; writeback doesn't expect response
464 assert(!pkt->needsResponse());
465 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
466 DPRINTF(Cache, "%s new state is %s\n", __func__, blk->print());
467
468 incHitCount(pkt);
469 // populate the time when the block will be ready to access.
470 blk->whenReady = clockEdge(fillLatency) + pkt->headerDelay +
471 pkt->payloadDelay;
|
464 return true;
| 472 // if this a write-through packet it will be sent to cache
473 // below
474 return !pkt->writeThrough();
|
465 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
466 blk->isReadable())) {
467 // OK to satisfy access
468 incHitCount(pkt);
469 satisfyRequest(pkt, blk);
470 maintainClusivity(pkt->fromCache(), blk);
471
472 return true;
473 }
474
475 // Can't satisfy access normally... either no block (blk == nullptr)
476 // or have block but need writable
477
478 incMissCount(pkt);
479
480 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
481 // complete miss on store conditional... just give up now
482 pkt->req->setExtraData(0);
483 return true;
484 }
485
486 return false;
487}
488
489void
490Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
491{
492 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
493 clusivity == Enums::mostly_excl) {
494 // if we have responded to a cache, and our block is still
495 // valid, but not dirty, and this cache is mostly exclusive
496 // with respect to the cache above, drop the block
497 invalidateBlock(blk);
498 }
499}
500
501void
502Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
503{
504 while (!writebacks.empty()) {
505 PacketPtr wbPkt = writebacks.front();
506 // We use forwardLatency here because we are copying writebacks to
507 // write buffer.
508
509 // Call isCachedAbove for Writebacks, CleanEvicts and
510 // WriteCleans to discover if the block is cached above.
511 if (isCachedAbove(wbPkt)) {
512 if (wbPkt->cmd == MemCmd::CleanEvict) {
513 // Delete CleanEvict because cached copies exist above. The
514 // packet destructor will delete the request object because
515 // this is a non-snoop request packet which does not require a
516 // response.
517 delete wbPkt;
518 } else if (wbPkt->cmd == MemCmd::WritebackClean) {
519 // clean writeback, do not send since the block is
520 // still cached above
521 assert(writebackClean);
522 delete wbPkt;
523 } else {
524 assert(wbPkt->cmd == MemCmd::WritebackDirty ||
525 wbPkt->cmd == MemCmd::WriteClean);
526 // Set BLOCK_CACHED flag in Writeback and send below, so that
527 // the Writeback does not reset the bit corresponding to this
528 // address in the snoop filter below.
529 wbPkt->setBlockCached();
530 allocateWriteBuffer(wbPkt, forward_time);
531 }
532 } else {
533 // If the block is not cached above, send packet below. Both
534 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
535 // reset the bit corresponding to this address in the snoop filter
536 // below.
537 allocateWriteBuffer(wbPkt, forward_time);
538 }
539 writebacks.pop_front();
540 }
541}
542
543void
544Cache::doWritebacksAtomic(PacketList& writebacks)
545{
546 while (!writebacks.empty()) {
547 PacketPtr wbPkt = writebacks.front();
548 // Call isCachedAbove for both Writebacks and CleanEvicts. If
549 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
550 // and discard CleanEvicts.
551 if (isCachedAbove(wbPkt, false)) {
552 if (wbPkt->cmd == MemCmd::WritebackDirty ||
553 wbPkt->cmd == MemCmd::WriteClean) {
554 // Set BLOCK_CACHED flag in Writeback and send below,
555 // so that the Writeback does not reset the bit
556 // corresponding to this address in the snoop filter
557 // below. We can discard CleanEvicts because cached
558 // copies exist above. Atomic mode isCachedAbove
559 // modifies packet to set BLOCK_CACHED flag
560 memSidePort->sendAtomic(wbPkt);
561 }
562 } else {
563 // If the block is not cached above, send packet below. Both
564 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
565 // reset the bit corresponding to this address in the snoop filter
566 // below.
567 memSidePort->sendAtomic(wbPkt);
568 }
569 writebacks.pop_front();
570 // In case of CleanEvicts, the packet destructor will delete the
571 // request object because this is a non-snoop request packet which
572 // does not require a response.
573 delete wbPkt;
574 }
575}
576
577
578void
579Cache::recvTimingSnoopResp(PacketPtr pkt)
580{
581 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
582
583 assert(pkt->isResponse());
584 assert(!system->bypassCaches());
585
586 // determine if the response is from a snoop request we created
587 // (in which case it should be in the outstandingSnoop), or if we
588 // merely forwarded someone else's snoop request
589 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
590 outstandingSnoop.end();
591
592 if (!forwardAsSnoop) {
593 // the packet came from this cache, so sink it here and do not
594 // forward it
595 assert(pkt->cmd == MemCmd::HardPFResp);
596
597 outstandingSnoop.erase(pkt->req);
598
599 DPRINTF(Cache, "Got prefetch response from above for addr "
600 "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
601 recvTimingResp(pkt);
602 return;
603 }
604
605 // forwardLatency is set here because there is a response from an
606 // upper level cache.
607 // To pay the delay that occurs if the packet comes from the bus,
608 // we charge also headerDelay.
609 Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
610 // Reset the timing of the packet.
611 pkt->headerDelay = pkt->payloadDelay = 0;
612 memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
613}
614
615void
616Cache::promoteWholeLineWrites(PacketPtr pkt)
617{
618 // Cache line clearing instructions
619 if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
620 (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
621 pkt->cmd = MemCmd::WriteLineReq;
622 DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
623 }
624}
625
626bool
627Cache::recvTimingReq(PacketPtr pkt)
628{
629 DPRINTF(CacheTags, "%s tags:\n%s\n", __func__, tags->print());
630
631 assert(pkt->isRequest());
632
633 // Just forward the packet if caches are disabled.
634 if (system->bypassCaches()) {
635 // @todo This should really enqueue the packet rather
636 bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
637 assert(success);
638 return true;
639 }
640
641 promoteWholeLineWrites(pkt);
642
643 if (pkt->cacheResponding()) {
644 // a cache above us (but not where the packet came from) is
645 // responding to the request, in other words it has the line
646 // in Modified or Owned state
647 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
648 pkt->print());
649
650 // if the packet needs the block to be writable, and the cache
651 // that has promised to respond (setting the cache responding
652 // flag) is not providing writable (it is in Owned rather than
653 // the Modified state), we know that there may be other Shared
654 // copies in the system; go out and invalidate them all
655 assert(pkt->needsWritable() && !pkt->responderHadWritable());
656
657 // an upstream cache that had the line in Owned state
658 // (dirty, but not writable), is responding and thus
659 // transferring the dirty line from one branch of the
660 // cache hierarchy to another
661
662 // send out an express snoop and invalidate all other
663 // copies (snooping a packet that needs writable is the
664 // same as an invalidation), thus turning the Owned line
665 // into a Modified line, note that we don't invalidate the
666 // block in the current cache or any other cache on the
667 // path to memory
668
669 // create a downstream express snoop with cleared packet
670 // flags, there is no need to allocate any data as the
671 // packet is merely used to co-ordinate state transitions
672 Packet *snoop_pkt = new Packet(pkt, true, false);
673
674 // also reset the bus time that the original packet has
675 // not yet paid for
676 snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
677
678 // make this an instantaneous express snoop, and let the
679 // other caches in the system know that the another cache
680 // is responding, because we have found the authorative
681 // copy (Modified or Owned) that will supply the right
682 // data
683 snoop_pkt->setExpressSnoop();
684 snoop_pkt->setCacheResponding();
685
686 // this express snoop travels towards the memory, and at
687 // every crossbar it is snooped upwards thus reaching
688 // every cache in the system
689 bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt);
690 // express snoops always succeed
691 assert(success);
692
693 // main memory will delete the snoop packet
694
695 // queue for deletion, as opposed to immediate deletion, as
696 // the sending cache is still relying on the packet
697 pendingDelete.reset(pkt);
698
699 // no need to take any further action in this particular cache
700 // as an upstram cache has already committed to responding,
701 // and we have already sent out any express snoops in the
702 // section above to ensure all other copies in the system are
703 // invalidated
704 return true;
705 }
706
707 // anything that is merely forwarded pays for the forward latency and
708 // the delay provided by the crossbar
709 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
710
711 // We use lookupLatency here because it is used to specify the latency
712 // to access.
713 Cycles lat = lookupLatency;
714 CacheBlk *blk = nullptr;
715 bool satisfied = false;
716 {
717 PacketList writebacks;
718 // Note that lat is passed by reference here. The function
719 // access() calls accessBlock() which can modify lat value.
720 satisfied = access(pkt, blk, lat, writebacks);
721
722 // copy writebacks to write buffer here to ensure they logically
723 // proceed anything happening below
724 doWritebacks(writebacks, forward_time);
725 }
726
727 // Here we charge the headerDelay that takes into account the latencies
728 // of the bus, if the packet comes from it.
729 // The latency charged it is just lat that is the value of lookupLatency
730 // modified by access() function, or if not just lookupLatency.
731 // In case of a hit we are neglecting response latency.
732 // In case of a miss we are neglecting forward latency.
733 Tick request_time = clockEdge(lat) + pkt->headerDelay;
734 // Here we reset the timing of the packet.
735 pkt->headerDelay = pkt->payloadDelay = 0;
736
737 // track time of availability of next prefetch, if any
738 Tick next_pf_time = MaxTick;
739
740 bool needsResponse = pkt->needsResponse();
741
742 if (satisfied) {
743 // should never be satisfying an uncacheable access as we
744 // flush and invalidate any existing block as part of the
745 // lookup
746 assert(!pkt->req->isUncacheable());
747
748 // hit (for all other request types)
749
750 if (prefetcher && (prefetchOnAccess ||
751 (blk && blk->wasPrefetched()))) {
752 if (blk)
753 blk->status &= ~BlkHWPrefetched;
754
755 // Don't notify on SWPrefetch
756 if (!pkt->cmd.isSWPrefetch())
757 next_pf_time = prefetcher->notify(pkt);
758 }
759
760 if (needsResponse) {
761 pkt->makeTimingResponse();
762 // @todo: Make someone pay for this
763 pkt->headerDelay = pkt->payloadDelay = 0;
764
765 // In this case we are considering request_time that takes
766 // into account the delay of the xbar, if any, and just
767 // lat, neglecting responseLatency, modelling hit latency
768 // just as lookupLatency or or the value of lat overriden
769 // by access(), that calls accessBlock() function.
770 cpuSidePort->schedTimingResp(pkt, request_time, true);
771 } else {
772 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
773 pkt->print());
774
775 // queue the packet for deletion, as the sending cache is
776 // still relying on it; if the block is found in access(),
777 // CleanEvict and Writeback messages will be deleted
778 // here as well
779 pendingDelete.reset(pkt);
780 }
781 } else {
782 // miss
783
784 Addr blk_addr = pkt->getBlockAddr(blkSize);
785
786 // ignore any existing MSHR if we are dealing with an
787 // uncacheable request
788 MSHR *mshr = pkt->req->isUncacheable() ? nullptr :
789 mshrQueue.findMatch(blk_addr, pkt->isSecure());
790
791 // Software prefetch handling:
792 // To keep the core from waiting on data it won't look at
793 // anyway, send back a response with dummy data. Miss handling
794 // will continue asynchronously. Unfortunately, the core will
795 // insist upon freeing original Packet/Request, so we have to
796 // create a new pair with a different lifecycle. Note that this
797 // processing happens before any MSHR munging on the behalf of
798 // this request because this new Request will be the one stored
799 // into the MSHRs, not the original.
800 if (pkt->cmd.isSWPrefetch()) {
801 assert(needsResponse);
802 assert(pkt->req->hasPaddr());
803 assert(!pkt->req->isUncacheable());
804
805 // There's no reason to add a prefetch as an additional target
806 // to an existing MSHR. If an outstanding request is already
807 // in progress, there is nothing for the prefetch to do.
808 // If this is the case, we don't even create a request at all.
809 PacketPtr pf = nullptr;
810
811 if (!mshr) {
812 // copy the request and create a new SoftPFReq packet
813 RequestPtr req = new Request(pkt->req->getPaddr(),
814 pkt->req->getSize(),
815 pkt->req->getFlags(),
816 pkt->req->masterId());
817 pf = new Packet(req, pkt->cmd);
818 pf->allocate();
819 assert(pf->getAddr() == pkt->getAddr());
820 assert(pf->getSize() == pkt->getSize());
821 }
822
823 pkt->makeTimingResponse();
824
825 // request_time is used here, taking into account lat and the delay
826 // charged if the packet comes from the xbar.
827 cpuSidePort->schedTimingResp(pkt, request_time, true);
828
829 // If an outstanding request is in progress (we found an
830 // MSHR) this is set to null
831 pkt = pf;
832 }
833
834 if (mshr) {
835 /// MSHR hit
836 /// @note writebacks will be checked in getNextMSHR()
837 /// for any conflicting requests to the same block
838
839 //@todo remove hw_pf here
840
841 // Coalesce unless it was a software prefetch (see above).
842 if (pkt) {
843 assert(!pkt->isWriteback());
844 // CleanEvicts corresponding to blocks which have
845 // outstanding requests in MSHRs are simply sunk here
846 if (pkt->cmd == MemCmd::CleanEvict) {
847 pendingDelete.reset(pkt);
848 } else {
849 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
850 pkt->print());
851
852 assert(pkt->req->masterId() < system->maxMasters());
853 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
854 // We use forward_time here because it is the same
855 // considering new targets. We have multiple
856 // requests for the same address here. It
857 // specifies the latency to allocate an internal
858 // buffer and to schedule an event to the queued
859 // port and also takes into account the additional
860 // delay of the xbar.
861 mshr->allocateTarget(pkt, forward_time, order++,
862 allocOnFill(pkt->cmd));
863 if (mshr->getNumTargets() == numTarget) {
864 noTargetMSHR = mshr;
865 setBlocked(Blocked_NoTargets);
866 // need to be careful with this... if this mshr isn't
867 // ready yet (i.e. time > curTick()), we don't want to
868 // move it ahead of mshrs that are ready
869 // mshrQueue.moveToFront(mshr);
870 }
871 }
872 // We should call the prefetcher reguardless if the request is
873 // satisfied or not, reguardless if the request is in the MSHR
874 // or not. The request could be a ReadReq hit, but still not
875 // satisfied (potentially because of a prior write to the same
876 // cache line. So, even when not satisfied, tehre is an MSHR
877 // already allocated for this, we need to let the prefetcher
878 // know about the request
879 if (prefetcher) {
880 // Don't notify on SWPrefetch
881 if (!pkt->cmd.isSWPrefetch())
882 next_pf_time = prefetcher->notify(pkt);
883 }
884 }
885 } else {
886 // no MSHR
887 assert(pkt->req->masterId() < system->maxMasters());
888 if (pkt->req->isUncacheable()) {
889 mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
890 } else {
891 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
892 }
893
894 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
895 (pkt->req->isUncacheable() && pkt->isWrite())) {
896 // We use forward_time here because there is an
897 // uncached memory write, forwarded to WriteBuffer.
898 allocateWriteBuffer(pkt, forward_time);
899 } else {
900 if (blk && blk->isValid()) {
901 // should have flushed and have no valid block
902 assert(!pkt->req->isUncacheable());
903
904 // If we have a write miss to a valid block, we
905 // need to mark the block non-readable. Otherwise
906 // if we allow reads while there's an outstanding
907 // write miss, the read could return stale data
908 // out of the cache block... a more aggressive
909 // system could detect the overlap (if any) and
910 // forward data out of the MSHRs, but we don't do
911 // that yet. Note that we do need to leave the
912 // block valid so that it stays in the cache, in
913 // case we get an upgrade response (and hence no
914 // new data) when the write miss completes.
915 // As long as CPUs do proper store/load forwarding
916 // internally, and have a sufficiently weak memory
917 // model, this is probably unnecessary, but at some
918 // point it must have seemed like we needed it...
919 assert(pkt->needsWritable());
920 assert(!blk->isWritable());
921 blk->status &= ~BlkReadable;
922 }
923 // Here we are using forward_time, modelling the latency of
924 // a miss (outbound) just as forwardLatency, neglecting the
925 // lookupLatency component.
926 allocateMissBuffer(pkt, forward_time);
927 }
928
929 if (prefetcher) {
930 // Don't notify on SWPrefetch
931 if (!pkt->cmd.isSWPrefetch())
932 next_pf_time = prefetcher->notify(pkt);
933 }
934 }
935 }
936
937 if (next_pf_time != MaxTick)
938 schedMemSideSendEvent(next_pf_time);
939
940 return true;
941}
942
943PacketPtr
944Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
945 bool needsWritable) const
946{
947 // should never see evictions here
948 assert(!cpu_pkt->isEviction());
949
950 bool blkValid = blk && blk->isValid();
951
952 if (cpu_pkt->req->isUncacheable() ||
953 (!blkValid && cpu_pkt->isUpgrade()) ||
954 cpu_pkt->cmd == MemCmd::InvalidateReq) {
955 // uncacheable requests and upgrades from upper-level caches
956 // that missed completely just go through as is
957 return nullptr;
958 }
959
960 assert(cpu_pkt->needsResponse());
961
962 MemCmd cmd;
963 // @TODO make useUpgrades a parameter.
964 // Note that ownership protocols require upgrade, otherwise a
965 // write miss on a shared owned block will generate a ReadExcl,
966 // which will clobber the owned copy.
967 const bool useUpgrades = true;
968 if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
969 assert(!blkValid || !blk->isWritable());
970 // forward as invalidate to all other caches, this gives us
971 // the line in Exclusive state, and invalidates all other
972 // copies
973 cmd = MemCmd::InvalidateReq;
974 } else if (blkValid && useUpgrades) {
975 // only reason to be here is that blk is read only and we need
976 // it to be writable
977 assert(needsWritable);
978 assert(!blk->isWritable());
979 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
980 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
981 cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
982 // Even though this SC will fail, we still need to send out the
983 // request and get the data to supply it to other snoopers in the case
984 // where the determination the StoreCond fails is delayed due to
985 // all caches not being on the same local bus.
986 cmd = MemCmd::SCUpgradeFailReq;
987 } else {
988 // block is invalid
989 cmd = needsWritable ? MemCmd::ReadExReq :
990 (isReadOnly ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
991 }
992 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
993
994 // if there are upstream caches that have already marked the
995 // packet as having sharers (not passing writable), pass that info
996 // downstream
997 if (cpu_pkt->hasSharers() && !needsWritable) {
998 // note that cpu_pkt may have spent a considerable time in the
999 // MSHR queue and that the information could possibly be out
1000 // of date, however, there is no harm in conservatively
1001 // assuming the block has sharers
1002 pkt->setHasSharers();
1003 DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1004 __func__, cpu_pkt->print(), pkt->print());
1005 }
1006
1007 // the packet should be block aligned
1008 assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1009
1010 pkt->allocate();
1011 DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1012 cpu_pkt->print());
1013 return pkt;
1014}
1015
1016
1017Tick
1018Cache::recvAtomic(PacketPtr pkt)
1019{
1020 // We are in atomic mode so we pay just for lookupLatency here.
1021 Cycles lat = lookupLatency;
1022
1023 // Forward the request if the system is in cache bypass mode.
1024 if (system->bypassCaches())
1025 return ticksToCycles(memSidePort->sendAtomic(pkt));
1026
1027 promoteWholeLineWrites(pkt);
1028
1029 // follow the same flow as in recvTimingReq, and check if a cache
1030 // above us is responding
1031 if (pkt->cacheResponding()) {
1032 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1033 pkt->print());
1034
1035 // if a cache is responding, and it had the line in Owned
1036 // rather than Modified state, we need to invalidate any
1037 // copies that are not on the same path to memory
1038 assert(pkt->needsWritable() && !pkt->responderHadWritable());
1039 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1040
1041 return lat * clockPeriod();
1042 }
1043
1044 // should assert here that there are no outstanding MSHRs or
1045 // writebacks... that would mean that someone used an atomic
1046 // access in timing mode
1047
1048 CacheBlk *blk = nullptr;
1049 PacketList writebacks;
1050 bool satisfied = access(pkt, blk, lat, writebacks);
1051
1052 // handle writebacks resulting from the access here to ensure they
1053 // logically proceed anything happening below
1054 doWritebacksAtomic(writebacks);
1055
1056 if (!satisfied) {
1057 // MISS
1058
1059 // deal with the packets that go through the write path of
1060 // the cache, i.e. any evictions and writes
1061 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1062 (pkt->req->isUncacheable() && pkt->isWrite())) {
1063 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1064 return lat * clockPeriod();
1065 }
1066 // only misses left
1067
1068 PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1069
1070 bool is_forward = (bus_pkt == nullptr);
1071
1072 if (is_forward) {
1073 // just forwarding the same request to the next level
1074 // no local cache operation involved
1075 bus_pkt = pkt;
1076 }
1077
1078 DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1079 bus_pkt->print());
1080
1081#if TRACING_ON
1082 CacheBlk::State old_state = blk ? blk->status : 0;
1083#endif
1084
1085 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1086
1087 bool is_invalidate = bus_pkt->isInvalidate();
1088
1089 // We are now dealing with the response handling
1090 DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1091 bus_pkt->print(), old_state);
1092
1093 // If packet was a forward, the response (if any) is already
1094 // in place in the bus_pkt == pkt structure, so we don't need
1095 // to do anything. Otherwise, use the separate bus_pkt to
1096 // generate response to pkt and then delete it.
1097 if (!is_forward) {
1098 if (pkt->needsResponse()) {
1099 assert(bus_pkt->isResponse());
1100 if (bus_pkt->isError()) {
1101 pkt->makeAtomicResponse();
1102 pkt->copyError(bus_pkt);
1103 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1104 // note the use of pkt, not bus_pkt here.
1105
1106 // write-line request to the cache that promoted
1107 // the write to a whole line
1108 blk = handleFill(pkt, blk, writebacks,
1109 allocOnFill(pkt->cmd));
1110 assert(blk != NULL);
1111 is_invalidate = false;
1112 satisfyRequest(pkt, blk);
1113 } else if (bus_pkt->isRead() ||
1114 bus_pkt->cmd == MemCmd::UpgradeResp) {
1115 // we're updating cache state to allow us to
1116 // satisfy the upstream request from the cache
1117 blk = handleFill(bus_pkt, blk, writebacks,
1118 allocOnFill(pkt->cmd));
1119 satisfyRequest(pkt, blk);
1120 maintainClusivity(pkt->fromCache(), blk);
1121 } else {
1122 // we're satisfying the upstream request without
1123 // modifying cache state, e.g., a write-through
1124 pkt->makeAtomicResponse();
1125 }
1126 }
1127 delete bus_pkt;
1128 }
1129
1130 if (is_invalidate && blk && blk->isValid()) {
1131 invalidateBlock(blk);
1132 }
1133 }
1134
1135 // Note that we don't invoke the prefetcher at all in atomic mode.
1136 // It's not clear how to do it properly, particularly for
1137 // prefetchers that aggressively generate prefetch candidates and
1138 // rely on bandwidth contention to throttle them; these will tend
1139 // to pollute the cache in atomic mode since there is no bandwidth
1140 // contention. If we ever do want to enable prefetching in atomic
1141 // mode, though, this is the place to do it... see timingAccess()
1142 // for an example (though we'd want to issue the prefetch(es)
1143 // immediately rather than calling requestMemSideBus() as we do
1144 // there).
1145
1146 // do any writebacks resulting from the response handling
1147 doWritebacksAtomic(writebacks);
1148
1149 // if we used temp block, check to see if its valid and if so
1150 // clear it out, but only do so after the call to recvAtomic is
1151 // finished so that any downstream observers (such as a snoop
1152 // filter), first see the fill, and only then see the eviction
1153 if (blk == tempBlock && tempBlock->isValid()) {
1154 // the atomic CPU calls recvAtomic for fetch and load/store
1155 // sequentuially, and we may already have a tempBlock
1156 // writeback from the fetch that we have not yet sent
1157 if (tempBlockWriteback) {
1158 // if that is the case, write the prevoius one back, and
1159 // do not schedule any new event
1160 writebackTempBlockAtomic();
1161 } else {
1162 // the writeback/clean eviction happens after the call to
1163 // recvAtomic has finished (but before any successive
1164 // calls), so that the response handling from the fill is
1165 // allowed to happen first
1166 schedule(writebackTempBlockAtomicEvent, curTick());
1167 }
1168
1169 tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1170 writebackBlk(blk) : cleanEvictBlk(blk);
1171 invalidateBlock(blk);
1172 }
1173
1174 if (pkt->needsResponse()) {
1175 pkt->makeAtomicResponse();
1176 }
1177
1178 return lat * clockPeriod();
1179}
1180
1181
1182void
1183Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1184{
1185 if (system->bypassCaches()) {
1186 // Packets from the memory side are snoop request and
1187 // shouldn't happen in bypass mode.
1188 assert(fromCpuSide);
1189
1190 // The cache should be flushed if we are in cache bypass mode,
1191 // so we don't need to check if we need to update anything.
1192 memSidePort->sendFunctional(pkt);
1193 return;
1194 }
1195
1196 Addr blk_addr = pkt->getBlockAddr(blkSize);
1197 bool is_secure = pkt->isSecure();
1198 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1199 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1200
1201 pkt->pushLabel(name());
1202
1203 CacheBlkPrintWrapper cbpw(blk);
1204
1205 // Note that just because an L2/L3 has valid data doesn't mean an
1206 // L1 doesn't have a more up-to-date modified copy that still
1207 // needs to be found. As a result we always update the request if
1208 // we have it, but only declare it satisfied if we are the owner.
1209
1210 // see if we have data at all (owned or otherwise)
1211 bool have_data = blk && blk->isValid()
1212 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1213 blk->data);
1214
1215 // data we have is dirty if marked as such or if we have an
1216 // in-service MSHR that is pending a modified line
1217 bool have_dirty =
1218 have_data && (blk->isDirty() ||
1219 (mshr && mshr->inService && mshr->isPendingModified()));
1220
1221 bool done = have_dirty
1222 || cpuSidePort->checkFunctional(pkt)
1223 || mshrQueue.checkFunctional(pkt, blk_addr)
1224 || writeBuffer.checkFunctional(pkt, blk_addr)
1225 || memSidePort->checkFunctional(pkt);
1226
1227 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
1228 (blk && blk->isValid()) ? "valid " : "",
1229 have_data ? "data " : "", done ? "done " : "");
1230
1231 // We're leaving the cache, so pop cache->name() label
1232 pkt->popLabel();
1233
1234 if (done) {
1235 pkt->makeResponse();
1236 } else {
1237 // if it came as a request from the CPU side then make sure it
1238 // continues towards the memory side
1239 if (fromCpuSide) {
1240 memSidePort->sendFunctional(pkt);
1241 } else if (cpuSidePort->isSnooping()) {
1242 // if it came from the memory side, it must be a snoop request
1243 // and we should only forward it if we are forwarding snoops
1244 cpuSidePort->sendFunctionalSnoop(pkt);
1245 }
1246 }
1247}
1248
1249
1250/////////////////////////////////////////////////////
1251//
1252// Response handling: responses from the memory side
1253//
1254/////////////////////////////////////////////////////
1255
1256
1257void
1258Cache::handleUncacheableWriteResp(PacketPtr pkt)
1259{
1260 Tick completion_time = clockEdge(responseLatency) +
1261 pkt->headerDelay + pkt->payloadDelay;
1262
1263 // Reset the bus additional time as it is now accounted for
1264 pkt->headerDelay = pkt->payloadDelay = 0;
1265
1266 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1267}
1268
1269void
1270Cache::recvTimingResp(PacketPtr pkt)
1271{
1272 assert(pkt->isResponse());
1273
1274 // all header delay should be paid for by the crossbar, unless
1275 // this is a prefetch response from above
1276 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1277 "%s saw a non-zero packet delay\n", name());
1278
1279 bool is_error = pkt->isError();
1280
1281 if (is_error) {
1282 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1283 pkt->print());
1284 }
1285
1286 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1287 pkt->print());
1288
1289 // if this is a write, we should be looking at an uncacheable
1290 // write
1291 if (pkt->isWrite()) {
1292 assert(pkt->req->isUncacheable());
1293 handleUncacheableWriteResp(pkt);
1294 return;
1295 }
1296
1297 // we have dealt with any (uncacheable) writes above, from here on
1298 // we know we are dealing with an MSHR due to a miss or a prefetch
1299 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1300 assert(mshr);
1301
1302 if (mshr == noTargetMSHR) {
1303 // we always clear at least one target
1304 clearBlocked(Blocked_NoTargets);
1305 noTargetMSHR = nullptr;
1306 }
1307
1308 // Initial target is used just for stats
1309 MSHR::Target *initial_tgt = mshr->getTarget();
1310 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1311 Tick miss_latency = curTick() - initial_tgt->recvTime;
1312
1313 if (pkt->req->isUncacheable()) {
1314 assert(pkt->req->masterId() < system->maxMasters());
1315 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1316 miss_latency;
1317 } else {
1318 assert(pkt->req->masterId() < system->maxMasters());
1319 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1320 miss_latency;
1321 }
1322
1323 bool wasFull = mshrQueue.isFull();
1324
1325 PacketList writebacks;
1326
1327 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1328
1329 // upgrade deferred targets if the response has no sharers, and is
1330 // thus passing writable
1331 if (!pkt->hasSharers()) {
1332 mshr->promoteWritable();
1333 }
1334
1335 bool is_fill = !mshr->isForward &&
1336 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1337
1338 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1339
1340 if (is_fill && !is_error) {
1341 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1342 pkt->getAddr());
1343
1344 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1345 assert(blk != nullptr);
1346 }
1347
1348 // allow invalidation responses originating from write-line
1349 // requests to be discarded
1350 bool is_invalidate = pkt->isInvalidate();
1351
1352 // First offset for critical word first calculations
1353 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1354
1355 bool from_cache = false;
1356 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1357 for (auto &target: targets) {
1358 Packet *tgt_pkt = target.pkt;
1359 switch (target.source) {
1360 case MSHR::Target::FromCPU:
1361 Tick completion_time;
1362 // Here we charge on completion_time the delay of the xbar if the
1363 // packet comes from it, charged on headerDelay.
1364 completion_time = pkt->headerDelay;
1365
1366 // Software prefetch handling for cache closest to core
1367 if (tgt_pkt->cmd.isSWPrefetch()) {
1368 // a software prefetch would have already been ack'd
1369 // immediately with dummy data so the core would be able to
1370 // retire it. This request completes right here, so we
1371 // deallocate it.
1372 delete tgt_pkt->req;
1373 delete tgt_pkt;
1374 break; // skip response
1375 }
1376
1377 // keep track of whether we have responded to another
1378 // cache
1379 from_cache = from_cache || tgt_pkt->fromCache();
1380
1381 // unlike the other packet flows, where data is found in other
1382 // caches or memory and brought back, write-line requests always
1383 // have the data right away, so the above check for "is fill?"
1384 // cannot actually be determined until examining the stored MSHR
1385 // state. We "catch up" with that logic here, which is duplicated
1386 // from above.
1387 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1388 assert(!is_error);
1389 // we got the block in a writable state, so promote
1390 // any deferred targets if possible
1391 mshr->promoteWritable();
1392 // NB: we use the original packet here and not the response!
1393 blk = handleFill(tgt_pkt, blk, writebacks,
1394 targets.allocOnFill);
1395 assert(blk != nullptr);
1396
1397 // treat as a fill, and discard the invalidation
1398 // response
1399 is_fill = true;
1400 is_invalidate = false;
1401 }
1402
1403 if (is_fill) {
1404 satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1405
1406 // How many bytes past the first request is this one
1407 int transfer_offset =
1408 tgt_pkt->getOffset(blkSize) - initial_offset;
1409 if (transfer_offset < 0) {
1410 transfer_offset += blkSize;
1411 }
1412
1413 // If not critical word (offset) return payloadDelay.
1414 // responseLatency is the latency of the return path
1415 // from lower level caches/memory to an upper level cache or
1416 // the core.
1417 completion_time += clockEdge(responseLatency) +
1418 (transfer_offset ? pkt->payloadDelay : 0);
1419
1420 assert(!tgt_pkt->req->isUncacheable());
1421
1422 assert(tgt_pkt->req->masterId() < system->maxMasters());
1423 missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1424 completion_time - target.recvTime;
1425 } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1426 // failed StoreCond upgrade
1427 assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1428 tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1429 tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1430 // responseLatency is the latency of the return path
1431 // from lower level caches/memory to an upper level cache or
1432 // the core.
1433 completion_time += clockEdge(responseLatency) +
1434 pkt->payloadDelay;
1435 tgt_pkt->req->setExtraData(0);
1436 } else {
1437 // We are about to send a response to a cache above
1438 // that asked for an invalidation; we need to
1439 // invalidate our copy immediately as the most
1440 // up-to-date copy of the block will now be in the
1441 // cache above. It will also prevent this cache from
1442 // responding (if the block was previously dirty) to
1443 // snoops as they should snoop the caches above where
1444 // they will get the response from.
1445 if (is_invalidate && blk && blk->isValid()) {
1446 invalidateBlock(blk);
1447 }
1448 // not a cache fill, just forwarding response
1449 // responseLatency is the latency of the return path
1450 // from lower level cahces/memory to the core.
1451 completion_time += clockEdge(responseLatency) +
1452 pkt->payloadDelay;
1453 if (pkt->isRead() && !is_error) {
1454 // sanity check
1455 assert(pkt->getAddr() == tgt_pkt->getAddr());
1456 assert(pkt->getSize() >= tgt_pkt->getSize());
1457
1458 tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1459 }
1460 }
1461 tgt_pkt->makeTimingResponse();
1462 // if this packet is an error copy that to the new packet
1463 if (is_error)
1464 tgt_pkt->copyError(pkt);
1465 if (tgt_pkt->cmd == MemCmd::ReadResp &&
1466 (is_invalidate || mshr->hasPostInvalidate())) {
1467 // If intermediate cache got ReadRespWithInvalidate,
1468 // propagate that. Response should not have
1469 // isInvalidate() set otherwise.
1470 tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1471 DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1472 tgt_pkt->print());
1473 }
1474 // Reset the bus additional time as it is now accounted for
1475 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1476 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1477 break;
1478
1479 case MSHR::Target::FromPrefetcher:
1480 assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1481 if (blk)
1482 blk->status |= BlkHWPrefetched;
1483 delete tgt_pkt->req;
1484 delete tgt_pkt;
1485 break;
1486
1487 case MSHR::Target::FromSnoop:
1488 // I don't believe that a snoop can be in an error state
1489 assert(!is_error);
1490 // response to snoop request
1491 DPRINTF(Cache, "processing deferred snoop...\n");
1492 // If the response is invalidating, a snooping target can
1493 // be satisfied if it is also invalidating. If the reponse is, not
1494 // only invalidating, but more specifically an InvalidateResp, the
1495 // MSHR was created due to an InvalidateReq and a cache above is
1496 // waiting to satisfy a WriteLineReq. In this case even an
1497 // non-invalidating snoop is added as a target here since this is
1498 // the ordering point. When the InvalidateResp reaches this cache,
1499 // the snooping target will snoop further the cache above with the
1500 // WriteLineReq.
1501 assert(!(is_invalidate &&
1502 pkt->cmd != MemCmd::InvalidateResp &&
1503 !mshr->hasPostInvalidate()));
1504 handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1505 break;
1506
1507 default:
1508 panic("Illegal target->source enum %d\n", target.source);
1509 }
1510 }
1511
1512 maintainClusivity(from_cache, blk);
1513
1514 if (blk && blk->isValid()) {
1515 // an invalidate response stemming from a write line request
1516 // should not invalidate the block, so check if the
1517 // invalidation should be discarded
1518 if (is_invalidate || mshr->hasPostInvalidate()) {
1519 invalidateBlock(blk);
1520 } else if (mshr->hasPostDowngrade()) {
1521 blk->status &= ~BlkWritable;
1522 }
1523 }
1524
1525 if (mshr->promoteDeferredTargets()) {
1526 // avoid later read getting stale data while write miss is
1527 // outstanding.. see comment in timingAccess()
1528 if (blk) {
1529 blk->status &= ~BlkReadable;
1530 }
1531 mshrQueue.markPending(mshr);
1532 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1533 } else {
1534 mshrQueue.deallocate(mshr);
1535 if (wasFull && !mshrQueue.isFull()) {
1536 clearBlocked(Blocked_NoMSHRs);
1537 }
1538
1539 // Request the bus for a prefetch if this deallocation freed enough
1540 // MSHRs for a prefetch to take place
1541 if (prefetcher && mshrQueue.canPrefetch()) {
1542 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1543 clockEdge());
1544 if (next_pf_time != MaxTick)
1545 schedMemSideSendEvent(next_pf_time);
1546 }
1547 }
1548 // reset the xbar additional timinig as it is now accounted for
1549 pkt->headerDelay = pkt->payloadDelay = 0;
1550
1551 // copy writebacks to write buffer
1552 doWritebacks(writebacks, forward_time);
1553
1554 // if we used temp block, check to see if its valid and then clear it out
1555 if (blk == tempBlock && tempBlock->isValid()) {
1556 // We use forwardLatency here because we are copying
1557 // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1558 // allocate an internal buffer and to schedule an event to the
1559 // queued port.
1560 if (blk->isDirty() || writebackClean) {
1561 PacketPtr wbPkt = writebackBlk(blk);
1562 allocateWriteBuffer(wbPkt, forward_time);
1563 // Set BLOCK_CACHED flag if cached above.
1564 if (isCachedAbove(wbPkt))
1565 wbPkt->setBlockCached();
1566 } else {
1567 PacketPtr wcPkt = cleanEvictBlk(blk);
1568 // Check to see if block is cached above. If not allocate
1569 // write buffer
1570 if (isCachedAbove(wcPkt))
1571 delete wcPkt;
1572 else
1573 allocateWriteBuffer(wcPkt, forward_time);
1574 }
1575 invalidateBlock(blk);
1576 }
1577
1578 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1579 delete pkt;
1580}
1581
1582PacketPtr
1583Cache::writebackBlk(CacheBlk *blk)
1584{
1585 chatty_assert(!isReadOnly || writebackClean,
1586 "Writeback from read-only cache");
1587 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1588
1589 writebacks[Request::wbMasterId]++;
1590
1591 Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1592 blkSize, 0, Request::wbMasterId);
1593 if (blk->isSecure())
1594 req->setFlags(Request::SECURE);
1595
1596 req->taskId(blk->task_id);
1597 blk->task_id= ContextSwitchTaskId::Unknown;
1598 blk->tickInserted = curTick();
1599
1600 PacketPtr pkt =
1601 new Packet(req, blk->isDirty() ?
1602 MemCmd::WritebackDirty : MemCmd::WritebackClean);
1603
1604 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1605 pkt->print(), blk->isWritable(), blk->isDirty());
1606
1607 if (blk->isWritable()) {
1608 // not asserting shared means we pass the block in modified
1609 // state, mark our own block non-writeable
1610 blk->status &= ~BlkWritable;
1611 } else {
1612 // we are in the Owned state, tell the receiver
1613 pkt->setHasSharers();
1614 }
1615
1616 // make sure the block is not marked dirty
1617 blk->status &= ~BlkDirty;
1618
1619 pkt->allocate();
1620 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1621
1622 return pkt;
1623}
1624
1625PacketPtr
| 475 } else if (blk && (pkt->needsWritable() ? blk->isWritable() :
476 blk->isReadable())) {
477 // OK to satisfy access
478 incHitCount(pkt);
479 satisfyRequest(pkt, blk);
480 maintainClusivity(pkt->fromCache(), blk);
481
482 return true;
483 }
484
485 // Can't satisfy access normally... either no block (blk == nullptr)
486 // or have block but need writable
487
488 incMissCount(pkt);
489
490 if (blk == nullptr && pkt->isLLSC() && pkt->isWrite()) {
491 // complete miss on store conditional... just give up now
492 pkt->req->setExtraData(0);
493 return true;
494 }
495
496 return false;
497}
498
499void
500Cache::maintainClusivity(bool from_cache, CacheBlk *blk)
501{
502 if (from_cache && blk && blk->isValid() && !blk->isDirty() &&
503 clusivity == Enums::mostly_excl) {
504 // if we have responded to a cache, and our block is still
505 // valid, but not dirty, and this cache is mostly exclusive
506 // with respect to the cache above, drop the block
507 invalidateBlock(blk);
508 }
509}
510
511void
512Cache::doWritebacks(PacketList& writebacks, Tick forward_time)
513{
514 while (!writebacks.empty()) {
515 PacketPtr wbPkt = writebacks.front();
516 // We use forwardLatency here because we are copying writebacks to
517 // write buffer.
518
519 // Call isCachedAbove for Writebacks, CleanEvicts and
520 // WriteCleans to discover if the block is cached above.
521 if (isCachedAbove(wbPkt)) {
522 if (wbPkt->cmd == MemCmd::CleanEvict) {
523 // Delete CleanEvict because cached copies exist above. The
524 // packet destructor will delete the request object because
525 // this is a non-snoop request packet which does not require a
526 // response.
527 delete wbPkt;
528 } else if (wbPkt->cmd == MemCmd::WritebackClean) {
529 // clean writeback, do not send since the block is
530 // still cached above
531 assert(writebackClean);
532 delete wbPkt;
533 } else {
534 assert(wbPkt->cmd == MemCmd::WritebackDirty ||
535 wbPkt->cmd == MemCmd::WriteClean);
536 // Set BLOCK_CACHED flag in Writeback and send below, so that
537 // the Writeback does not reset the bit corresponding to this
538 // address in the snoop filter below.
539 wbPkt->setBlockCached();
540 allocateWriteBuffer(wbPkt, forward_time);
541 }
542 } else {
543 // If the block is not cached above, send packet below. Both
544 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
545 // reset the bit corresponding to this address in the snoop filter
546 // below.
547 allocateWriteBuffer(wbPkt, forward_time);
548 }
549 writebacks.pop_front();
550 }
551}
552
553void
554Cache::doWritebacksAtomic(PacketList& writebacks)
555{
556 while (!writebacks.empty()) {
557 PacketPtr wbPkt = writebacks.front();
558 // Call isCachedAbove for both Writebacks and CleanEvicts. If
559 // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks
560 // and discard CleanEvicts.
561 if (isCachedAbove(wbPkt, false)) {
562 if (wbPkt->cmd == MemCmd::WritebackDirty ||
563 wbPkt->cmd == MemCmd::WriteClean) {
564 // Set BLOCK_CACHED flag in Writeback and send below,
565 // so that the Writeback does not reset the bit
566 // corresponding to this address in the snoop filter
567 // below. We can discard CleanEvicts because cached
568 // copies exist above. Atomic mode isCachedAbove
569 // modifies packet to set BLOCK_CACHED flag
570 memSidePort->sendAtomic(wbPkt);
571 }
572 } else {
573 // If the block is not cached above, send packet below. Both
574 // CleanEvict and Writeback with BLOCK_CACHED flag cleared will
575 // reset the bit corresponding to this address in the snoop filter
576 // below.
577 memSidePort->sendAtomic(wbPkt);
578 }
579 writebacks.pop_front();
580 // In case of CleanEvicts, the packet destructor will delete the
581 // request object because this is a non-snoop request packet which
582 // does not require a response.
583 delete wbPkt;
584 }
585}
586
587
588void
589Cache::recvTimingSnoopResp(PacketPtr pkt)
590{
591 DPRINTF(Cache, "%s for %s\n", __func__, pkt->print());
592
593 assert(pkt->isResponse());
594 assert(!system->bypassCaches());
595
596 // determine if the response is from a snoop request we created
597 // (in which case it should be in the outstandingSnoop), or if we
598 // merely forwarded someone else's snoop request
599 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
600 outstandingSnoop.end();
601
602 if (!forwardAsSnoop) {
603 // the packet came from this cache, so sink it here and do not
604 // forward it
605 assert(pkt->cmd == MemCmd::HardPFResp);
606
607 outstandingSnoop.erase(pkt->req);
608
609 DPRINTF(Cache, "Got prefetch response from above for addr "
610 "%#llx (%s)\n", pkt->getAddr(), pkt->isSecure() ? "s" : "ns");
611 recvTimingResp(pkt);
612 return;
613 }
614
615 // forwardLatency is set here because there is a response from an
616 // upper level cache.
617 // To pay the delay that occurs if the packet comes from the bus,
618 // we charge also headerDelay.
619 Tick snoop_resp_time = clockEdge(forwardLatency) + pkt->headerDelay;
620 // Reset the timing of the packet.
621 pkt->headerDelay = pkt->payloadDelay = 0;
622 memSidePort->schedTimingSnoopResp(pkt, snoop_resp_time);
623}
624
625void
626Cache::promoteWholeLineWrites(PacketPtr pkt)
627{
628 // Cache line clearing instructions
629 if (doFastWrites && (pkt->cmd == MemCmd::WriteReq) &&
630 (pkt->getSize() == blkSize) && (pkt->getOffset(blkSize) == 0)) {
631 pkt->cmd = MemCmd::WriteLineReq;
632 DPRINTF(Cache, "packet promoted from Write to WriteLineReq\n");
633 }
634}
635
636bool
637Cache::recvTimingReq(PacketPtr pkt)
638{
639 DPRINTF(CacheTags, "%s tags:\n%s\n", __func__, tags->print());
640
641 assert(pkt->isRequest());
642
643 // Just forward the packet if caches are disabled.
644 if (system->bypassCaches()) {
645 // @todo This should really enqueue the packet rather
646 bool M5_VAR_USED success = memSidePort->sendTimingReq(pkt);
647 assert(success);
648 return true;
649 }
650
651 promoteWholeLineWrites(pkt);
652
653 if (pkt->cacheResponding()) {
654 // a cache above us (but not where the packet came from) is
655 // responding to the request, in other words it has the line
656 // in Modified or Owned state
657 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
658 pkt->print());
659
660 // if the packet needs the block to be writable, and the cache
661 // that has promised to respond (setting the cache responding
662 // flag) is not providing writable (it is in Owned rather than
663 // the Modified state), we know that there may be other Shared
664 // copies in the system; go out and invalidate them all
665 assert(pkt->needsWritable() && !pkt->responderHadWritable());
666
667 // an upstream cache that had the line in Owned state
668 // (dirty, but not writable), is responding and thus
669 // transferring the dirty line from one branch of the
670 // cache hierarchy to another
671
672 // send out an express snoop and invalidate all other
673 // copies (snooping a packet that needs writable is the
674 // same as an invalidation), thus turning the Owned line
675 // into a Modified line, note that we don't invalidate the
676 // block in the current cache or any other cache on the
677 // path to memory
678
679 // create a downstream express snoop with cleared packet
680 // flags, there is no need to allocate any data as the
681 // packet is merely used to co-ordinate state transitions
682 Packet *snoop_pkt = new Packet(pkt, true, false);
683
684 // also reset the bus time that the original packet has
685 // not yet paid for
686 snoop_pkt->headerDelay = snoop_pkt->payloadDelay = 0;
687
688 // make this an instantaneous express snoop, and let the
689 // other caches in the system know that the another cache
690 // is responding, because we have found the authorative
691 // copy (Modified or Owned) that will supply the right
692 // data
693 snoop_pkt->setExpressSnoop();
694 snoop_pkt->setCacheResponding();
695
696 // this express snoop travels towards the memory, and at
697 // every crossbar it is snooped upwards thus reaching
698 // every cache in the system
699 bool M5_VAR_USED success = memSidePort->sendTimingReq(snoop_pkt);
700 // express snoops always succeed
701 assert(success);
702
703 // main memory will delete the snoop packet
704
705 // queue for deletion, as opposed to immediate deletion, as
706 // the sending cache is still relying on the packet
707 pendingDelete.reset(pkt);
708
709 // no need to take any further action in this particular cache
710 // as an upstram cache has already committed to responding,
711 // and we have already sent out any express snoops in the
712 // section above to ensure all other copies in the system are
713 // invalidated
714 return true;
715 }
716
717 // anything that is merely forwarded pays for the forward latency and
718 // the delay provided by the crossbar
719 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
720
721 // We use lookupLatency here because it is used to specify the latency
722 // to access.
723 Cycles lat = lookupLatency;
724 CacheBlk *blk = nullptr;
725 bool satisfied = false;
726 {
727 PacketList writebacks;
728 // Note that lat is passed by reference here. The function
729 // access() calls accessBlock() which can modify lat value.
730 satisfied = access(pkt, blk, lat, writebacks);
731
732 // copy writebacks to write buffer here to ensure they logically
733 // proceed anything happening below
734 doWritebacks(writebacks, forward_time);
735 }
736
737 // Here we charge the headerDelay that takes into account the latencies
738 // of the bus, if the packet comes from it.
739 // The latency charged it is just lat that is the value of lookupLatency
740 // modified by access() function, or if not just lookupLatency.
741 // In case of a hit we are neglecting response latency.
742 // In case of a miss we are neglecting forward latency.
743 Tick request_time = clockEdge(lat) + pkt->headerDelay;
744 // Here we reset the timing of the packet.
745 pkt->headerDelay = pkt->payloadDelay = 0;
746
747 // track time of availability of next prefetch, if any
748 Tick next_pf_time = MaxTick;
749
750 bool needsResponse = pkt->needsResponse();
751
752 if (satisfied) {
753 // should never be satisfying an uncacheable access as we
754 // flush and invalidate any existing block as part of the
755 // lookup
756 assert(!pkt->req->isUncacheable());
757
758 // hit (for all other request types)
759
760 if (prefetcher && (prefetchOnAccess ||
761 (blk && blk->wasPrefetched()))) {
762 if (blk)
763 blk->status &= ~BlkHWPrefetched;
764
765 // Don't notify on SWPrefetch
766 if (!pkt->cmd.isSWPrefetch())
767 next_pf_time = prefetcher->notify(pkt);
768 }
769
770 if (needsResponse) {
771 pkt->makeTimingResponse();
772 // @todo: Make someone pay for this
773 pkt->headerDelay = pkt->payloadDelay = 0;
774
775 // In this case we are considering request_time that takes
776 // into account the delay of the xbar, if any, and just
777 // lat, neglecting responseLatency, modelling hit latency
778 // just as lookupLatency or or the value of lat overriden
779 // by access(), that calls accessBlock() function.
780 cpuSidePort->schedTimingResp(pkt, request_time, true);
781 } else {
782 DPRINTF(Cache, "%s satisfied %s, no response needed\n", __func__,
783 pkt->print());
784
785 // queue the packet for deletion, as the sending cache is
786 // still relying on it; if the block is found in access(),
787 // CleanEvict and Writeback messages will be deleted
788 // here as well
789 pendingDelete.reset(pkt);
790 }
791 } else {
792 // miss
793
794 Addr blk_addr = pkt->getBlockAddr(blkSize);
795
796 // ignore any existing MSHR if we are dealing with an
797 // uncacheable request
798 MSHR *mshr = pkt->req->isUncacheable() ? nullptr :
799 mshrQueue.findMatch(blk_addr, pkt->isSecure());
800
801 // Software prefetch handling:
802 // To keep the core from waiting on data it won't look at
803 // anyway, send back a response with dummy data. Miss handling
804 // will continue asynchronously. Unfortunately, the core will
805 // insist upon freeing original Packet/Request, so we have to
806 // create a new pair with a different lifecycle. Note that this
807 // processing happens before any MSHR munging on the behalf of
808 // this request because this new Request will be the one stored
809 // into the MSHRs, not the original.
810 if (pkt->cmd.isSWPrefetch()) {
811 assert(needsResponse);
812 assert(pkt->req->hasPaddr());
813 assert(!pkt->req->isUncacheable());
814
815 // There's no reason to add a prefetch as an additional target
816 // to an existing MSHR. If an outstanding request is already
817 // in progress, there is nothing for the prefetch to do.
818 // If this is the case, we don't even create a request at all.
819 PacketPtr pf = nullptr;
820
821 if (!mshr) {
822 // copy the request and create a new SoftPFReq packet
823 RequestPtr req = new Request(pkt->req->getPaddr(),
824 pkt->req->getSize(),
825 pkt->req->getFlags(),
826 pkt->req->masterId());
827 pf = new Packet(req, pkt->cmd);
828 pf->allocate();
829 assert(pf->getAddr() == pkt->getAddr());
830 assert(pf->getSize() == pkt->getSize());
831 }
832
833 pkt->makeTimingResponse();
834
835 // request_time is used here, taking into account lat and the delay
836 // charged if the packet comes from the xbar.
837 cpuSidePort->schedTimingResp(pkt, request_time, true);
838
839 // If an outstanding request is in progress (we found an
840 // MSHR) this is set to null
841 pkt = pf;
842 }
843
844 if (mshr) {
845 /// MSHR hit
846 /// @note writebacks will be checked in getNextMSHR()
847 /// for any conflicting requests to the same block
848
849 //@todo remove hw_pf here
850
851 // Coalesce unless it was a software prefetch (see above).
852 if (pkt) {
853 assert(!pkt->isWriteback());
854 // CleanEvicts corresponding to blocks which have
855 // outstanding requests in MSHRs are simply sunk here
856 if (pkt->cmd == MemCmd::CleanEvict) {
857 pendingDelete.reset(pkt);
858 } else {
859 DPRINTF(Cache, "%s coalescing MSHR for %s\n", __func__,
860 pkt->print());
861
862 assert(pkt->req->masterId() < system->maxMasters());
863 mshr_hits[pkt->cmdToIndex()][pkt->req->masterId()]++;
864 // We use forward_time here because it is the same
865 // considering new targets. We have multiple
866 // requests for the same address here. It
867 // specifies the latency to allocate an internal
868 // buffer and to schedule an event to the queued
869 // port and also takes into account the additional
870 // delay of the xbar.
871 mshr->allocateTarget(pkt, forward_time, order++,
872 allocOnFill(pkt->cmd));
873 if (mshr->getNumTargets() == numTarget) {
874 noTargetMSHR = mshr;
875 setBlocked(Blocked_NoTargets);
876 // need to be careful with this... if this mshr isn't
877 // ready yet (i.e. time > curTick()), we don't want to
878 // move it ahead of mshrs that are ready
879 // mshrQueue.moveToFront(mshr);
880 }
881 }
882 // We should call the prefetcher reguardless if the request is
883 // satisfied or not, reguardless if the request is in the MSHR
884 // or not. The request could be a ReadReq hit, but still not
885 // satisfied (potentially because of a prior write to the same
886 // cache line. So, even when not satisfied, tehre is an MSHR
887 // already allocated for this, we need to let the prefetcher
888 // know about the request
889 if (prefetcher) {
890 // Don't notify on SWPrefetch
891 if (!pkt->cmd.isSWPrefetch())
892 next_pf_time = prefetcher->notify(pkt);
893 }
894 }
895 } else {
896 // no MSHR
897 assert(pkt->req->masterId() < system->maxMasters());
898 if (pkt->req->isUncacheable()) {
899 mshr_uncacheable[pkt->cmdToIndex()][pkt->req->masterId()]++;
900 } else {
901 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
902 }
903
904 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
905 (pkt->req->isUncacheable() && pkt->isWrite())) {
906 // We use forward_time here because there is an
907 // uncached memory write, forwarded to WriteBuffer.
908 allocateWriteBuffer(pkt, forward_time);
909 } else {
910 if (blk && blk->isValid()) {
911 // should have flushed and have no valid block
912 assert(!pkt->req->isUncacheable());
913
914 // If we have a write miss to a valid block, we
915 // need to mark the block non-readable. Otherwise
916 // if we allow reads while there's an outstanding
917 // write miss, the read could return stale data
918 // out of the cache block... a more aggressive
919 // system could detect the overlap (if any) and
920 // forward data out of the MSHRs, but we don't do
921 // that yet. Note that we do need to leave the
922 // block valid so that it stays in the cache, in
923 // case we get an upgrade response (and hence no
924 // new data) when the write miss completes.
925 // As long as CPUs do proper store/load forwarding
926 // internally, and have a sufficiently weak memory
927 // model, this is probably unnecessary, but at some
928 // point it must have seemed like we needed it...
929 assert(pkt->needsWritable());
930 assert(!blk->isWritable());
931 blk->status &= ~BlkReadable;
932 }
933 // Here we are using forward_time, modelling the latency of
934 // a miss (outbound) just as forwardLatency, neglecting the
935 // lookupLatency component.
936 allocateMissBuffer(pkt, forward_time);
937 }
938
939 if (prefetcher) {
940 // Don't notify on SWPrefetch
941 if (!pkt->cmd.isSWPrefetch())
942 next_pf_time = prefetcher->notify(pkt);
943 }
944 }
945 }
946
947 if (next_pf_time != MaxTick)
948 schedMemSideSendEvent(next_pf_time);
949
950 return true;
951}
952
953PacketPtr
954Cache::createMissPacket(PacketPtr cpu_pkt, CacheBlk *blk,
955 bool needsWritable) const
956{
957 // should never see evictions here
958 assert(!cpu_pkt->isEviction());
959
960 bool blkValid = blk && blk->isValid();
961
962 if (cpu_pkt->req->isUncacheable() ||
963 (!blkValid && cpu_pkt->isUpgrade()) ||
964 cpu_pkt->cmd == MemCmd::InvalidateReq) {
965 // uncacheable requests and upgrades from upper-level caches
966 // that missed completely just go through as is
967 return nullptr;
968 }
969
970 assert(cpu_pkt->needsResponse());
971
972 MemCmd cmd;
973 // @TODO make useUpgrades a parameter.
974 // Note that ownership protocols require upgrade, otherwise a
975 // write miss on a shared owned block will generate a ReadExcl,
976 // which will clobber the owned copy.
977 const bool useUpgrades = true;
978 if (cpu_pkt->cmd == MemCmd::WriteLineReq) {
979 assert(!blkValid || !blk->isWritable());
980 // forward as invalidate to all other caches, this gives us
981 // the line in Exclusive state, and invalidates all other
982 // copies
983 cmd = MemCmd::InvalidateReq;
984 } else if (blkValid && useUpgrades) {
985 // only reason to be here is that blk is read only and we need
986 // it to be writable
987 assert(needsWritable);
988 assert(!blk->isWritable());
989 cmd = cpu_pkt->isLLSC() ? MemCmd::SCUpgradeReq : MemCmd::UpgradeReq;
990 } else if (cpu_pkt->cmd == MemCmd::SCUpgradeFailReq ||
991 cpu_pkt->cmd == MemCmd::StoreCondFailReq) {
992 // Even though this SC will fail, we still need to send out the
993 // request and get the data to supply it to other snoopers in the case
994 // where the determination the StoreCond fails is delayed due to
995 // all caches not being on the same local bus.
996 cmd = MemCmd::SCUpgradeFailReq;
997 } else {
998 // block is invalid
999 cmd = needsWritable ? MemCmd::ReadExReq :
1000 (isReadOnly ? MemCmd::ReadCleanReq : MemCmd::ReadSharedReq);
1001 }
1002 PacketPtr pkt = new Packet(cpu_pkt->req, cmd, blkSize);
1003
1004 // if there are upstream caches that have already marked the
1005 // packet as having sharers (not passing writable), pass that info
1006 // downstream
1007 if (cpu_pkt->hasSharers() && !needsWritable) {
1008 // note that cpu_pkt may have spent a considerable time in the
1009 // MSHR queue and that the information could possibly be out
1010 // of date, however, there is no harm in conservatively
1011 // assuming the block has sharers
1012 pkt->setHasSharers();
1013 DPRINTF(Cache, "%s: passing hasSharers from %s to %s\n",
1014 __func__, cpu_pkt->print(), pkt->print());
1015 }
1016
1017 // the packet should be block aligned
1018 assert(pkt->getAddr() == pkt->getBlockAddr(blkSize));
1019
1020 pkt->allocate();
1021 DPRINTF(Cache, "%s: created %s from %s\n", __func__, pkt->print(),
1022 cpu_pkt->print());
1023 return pkt;
1024}
1025
1026
1027Tick
1028Cache::recvAtomic(PacketPtr pkt)
1029{
1030 // We are in atomic mode so we pay just for lookupLatency here.
1031 Cycles lat = lookupLatency;
1032
1033 // Forward the request if the system is in cache bypass mode.
1034 if (system->bypassCaches())
1035 return ticksToCycles(memSidePort->sendAtomic(pkt));
1036
1037 promoteWholeLineWrites(pkt);
1038
1039 // follow the same flow as in recvTimingReq, and check if a cache
1040 // above us is responding
1041 if (pkt->cacheResponding()) {
1042 DPRINTF(Cache, "Cache above responding to %s: not responding\n",
1043 pkt->print());
1044
1045 // if a cache is responding, and it had the line in Owned
1046 // rather than Modified state, we need to invalidate any
1047 // copies that are not on the same path to memory
1048 assert(pkt->needsWritable() && !pkt->responderHadWritable());
1049 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1050
1051 return lat * clockPeriod();
1052 }
1053
1054 // should assert here that there are no outstanding MSHRs or
1055 // writebacks... that would mean that someone used an atomic
1056 // access in timing mode
1057
1058 CacheBlk *blk = nullptr;
1059 PacketList writebacks;
1060 bool satisfied = access(pkt, blk, lat, writebacks);
1061
1062 // handle writebacks resulting from the access here to ensure they
1063 // logically proceed anything happening below
1064 doWritebacksAtomic(writebacks);
1065
1066 if (!satisfied) {
1067 // MISS
1068
1069 // deal with the packets that go through the write path of
1070 // the cache, i.e. any evictions and writes
1071 if (pkt->isEviction() || pkt->cmd == MemCmd::WriteClean ||
1072 (pkt->req->isUncacheable() && pkt->isWrite())) {
1073 lat += ticksToCycles(memSidePort->sendAtomic(pkt));
1074 return lat * clockPeriod();
1075 }
1076 // only misses left
1077
1078 PacketPtr bus_pkt = createMissPacket(pkt, blk, pkt->needsWritable());
1079
1080 bool is_forward = (bus_pkt == nullptr);
1081
1082 if (is_forward) {
1083 // just forwarding the same request to the next level
1084 // no local cache operation involved
1085 bus_pkt = pkt;
1086 }
1087
1088 DPRINTF(Cache, "%s: Sending an atomic %s\n", __func__,
1089 bus_pkt->print());
1090
1091#if TRACING_ON
1092 CacheBlk::State old_state = blk ? blk->status : 0;
1093#endif
1094
1095 lat += ticksToCycles(memSidePort->sendAtomic(bus_pkt));
1096
1097 bool is_invalidate = bus_pkt->isInvalidate();
1098
1099 // We are now dealing with the response handling
1100 DPRINTF(Cache, "%s: Receive response: %s in state %i\n", __func__,
1101 bus_pkt->print(), old_state);
1102
1103 // If packet was a forward, the response (if any) is already
1104 // in place in the bus_pkt == pkt structure, so we don't need
1105 // to do anything. Otherwise, use the separate bus_pkt to
1106 // generate response to pkt and then delete it.
1107 if (!is_forward) {
1108 if (pkt->needsResponse()) {
1109 assert(bus_pkt->isResponse());
1110 if (bus_pkt->isError()) {
1111 pkt->makeAtomicResponse();
1112 pkt->copyError(bus_pkt);
1113 } else if (pkt->cmd == MemCmd::WriteLineReq) {
1114 // note the use of pkt, not bus_pkt here.
1115
1116 // write-line request to the cache that promoted
1117 // the write to a whole line
1118 blk = handleFill(pkt, blk, writebacks,
1119 allocOnFill(pkt->cmd));
1120 assert(blk != NULL);
1121 is_invalidate = false;
1122 satisfyRequest(pkt, blk);
1123 } else if (bus_pkt->isRead() ||
1124 bus_pkt->cmd == MemCmd::UpgradeResp) {
1125 // we're updating cache state to allow us to
1126 // satisfy the upstream request from the cache
1127 blk = handleFill(bus_pkt, blk, writebacks,
1128 allocOnFill(pkt->cmd));
1129 satisfyRequest(pkt, blk);
1130 maintainClusivity(pkt->fromCache(), blk);
1131 } else {
1132 // we're satisfying the upstream request without
1133 // modifying cache state, e.g., a write-through
1134 pkt->makeAtomicResponse();
1135 }
1136 }
1137 delete bus_pkt;
1138 }
1139
1140 if (is_invalidate && blk && blk->isValid()) {
1141 invalidateBlock(blk);
1142 }
1143 }
1144
1145 // Note that we don't invoke the prefetcher at all in atomic mode.
1146 // It's not clear how to do it properly, particularly for
1147 // prefetchers that aggressively generate prefetch candidates and
1148 // rely on bandwidth contention to throttle them; these will tend
1149 // to pollute the cache in atomic mode since there is no bandwidth
1150 // contention. If we ever do want to enable prefetching in atomic
1151 // mode, though, this is the place to do it... see timingAccess()
1152 // for an example (though we'd want to issue the prefetch(es)
1153 // immediately rather than calling requestMemSideBus() as we do
1154 // there).
1155
1156 // do any writebacks resulting from the response handling
1157 doWritebacksAtomic(writebacks);
1158
1159 // if we used temp block, check to see if its valid and if so
1160 // clear it out, but only do so after the call to recvAtomic is
1161 // finished so that any downstream observers (such as a snoop
1162 // filter), first see the fill, and only then see the eviction
1163 if (blk == tempBlock && tempBlock->isValid()) {
1164 // the atomic CPU calls recvAtomic for fetch and load/store
1165 // sequentuially, and we may already have a tempBlock
1166 // writeback from the fetch that we have not yet sent
1167 if (tempBlockWriteback) {
1168 // if that is the case, write the prevoius one back, and
1169 // do not schedule any new event
1170 writebackTempBlockAtomic();
1171 } else {
1172 // the writeback/clean eviction happens after the call to
1173 // recvAtomic has finished (but before any successive
1174 // calls), so that the response handling from the fill is
1175 // allowed to happen first
1176 schedule(writebackTempBlockAtomicEvent, curTick());
1177 }
1178
1179 tempBlockWriteback = (blk->isDirty() || writebackClean) ?
1180 writebackBlk(blk) : cleanEvictBlk(blk);
1181 invalidateBlock(blk);
1182 }
1183
1184 if (pkt->needsResponse()) {
1185 pkt->makeAtomicResponse();
1186 }
1187
1188 return lat * clockPeriod();
1189}
1190
1191
1192void
1193Cache::functionalAccess(PacketPtr pkt, bool fromCpuSide)
1194{
1195 if (system->bypassCaches()) {
1196 // Packets from the memory side are snoop request and
1197 // shouldn't happen in bypass mode.
1198 assert(fromCpuSide);
1199
1200 // The cache should be flushed if we are in cache bypass mode,
1201 // so we don't need to check if we need to update anything.
1202 memSidePort->sendFunctional(pkt);
1203 return;
1204 }
1205
1206 Addr blk_addr = pkt->getBlockAddr(blkSize);
1207 bool is_secure = pkt->isSecure();
1208 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
1209 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
1210
1211 pkt->pushLabel(name());
1212
1213 CacheBlkPrintWrapper cbpw(blk);
1214
1215 // Note that just because an L2/L3 has valid data doesn't mean an
1216 // L1 doesn't have a more up-to-date modified copy that still
1217 // needs to be found. As a result we always update the request if
1218 // we have it, but only declare it satisfied if we are the owner.
1219
1220 // see if we have data at all (owned or otherwise)
1221 bool have_data = blk && blk->isValid()
1222 && pkt->checkFunctional(&cbpw, blk_addr, is_secure, blkSize,
1223 blk->data);
1224
1225 // data we have is dirty if marked as such or if we have an
1226 // in-service MSHR that is pending a modified line
1227 bool have_dirty =
1228 have_data && (blk->isDirty() ||
1229 (mshr && mshr->inService && mshr->isPendingModified()));
1230
1231 bool done = have_dirty
1232 || cpuSidePort->checkFunctional(pkt)
1233 || mshrQueue.checkFunctional(pkt, blk_addr)
1234 || writeBuffer.checkFunctional(pkt, blk_addr)
1235 || memSidePort->checkFunctional(pkt);
1236
1237 DPRINTF(CacheVerbose, "%s: %s %s%s%s\n", __func__, pkt->print(),
1238 (blk && blk->isValid()) ? "valid " : "",
1239 have_data ? "data " : "", done ? "done " : "");
1240
1241 // We're leaving the cache, so pop cache->name() label
1242 pkt->popLabel();
1243
1244 if (done) {
1245 pkt->makeResponse();
1246 } else {
1247 // if it came as a request from the CPU side then make sure it
1248 // continues towards the memory side
1249 if (fromCpuSide) {
1250 memSidePort->sendFunctional(pkt);
1251 } else if (cpuSidePort->isSnooping()) {
1252 // if it came from the memory side, it must be a snoop request
1253 // and we should only forward it if we are forwarding snoops
1254 cpuSidePort->sendFunctionalSnoop(pkt);
1255 }
1256 }
1257}
1258
1259
1260/////////////////////////////////////////////////////
1261//
1262// Response handling: responses from the memory side
1263//
1264/////////////////////////////////////////////////////
1265
1266
1267void
1268Cache::handleUncacheableWriteResp(PacketPtr pkt)
1269{
1270 Tick completion_time = clockEdge(responseLatency) +
1271 pkt->headerDelay + pkt->payloadDelay;
1272
1273 // Reset the bus additional time as it is now accounted for
1274 pkt->headerDelay = pkt->payloadDelay = 0;
1275
1276 cpuSidePort->schedTimingResp(pkt, completion_time, true);
1277}
1278
1279void
1280Cache::recvTimingResp(PacketPtr pkt)
1281{
1282 assert(pkt->isResponse());
1283
1284 // all header delay should be paid for by the crossbar, unless
1285 // this is a prefetch response from above
1286 panic_if(pkt->headerDelay != 0 && pkt->cmd != MemCmd::HardPFResp,
1287 "%s saw a non-zero packet delay\n", name());
1288
1289 bool is_error = pkt->isError();
1290
1291 if (is_error) {
1292 DPRINTF(Cache, "%s: Cache received %s with error\n", __func__,
1293 pkt->print());
1294 }
1295
1296 DPRINTF(Cache, "%s: Handling response %s\n", __func__,
1297 pkt->print());
1298
1299 // if this is a write, we should be looking at an uncacheable
1300 // write
1301 if (pkt->isWrite()) {
1302 assert(pkt->req->isUncacheable());
1303 handleUncacheableWriteResp(pkt);
1304 return;
1305 }
1306
1307 // we have dealt with any (uncacheable) writes above, from here on
1308 // we know we are dealing with an MSHR due to a miss or a prefetch
1309 MSHR *mshr = dynamic_cast<MSHR*>(pkt->popSenderState());
1310 assert(mshr);
1311
1312 if (mshr == noTargetMSHR) {
1313 // we always clear at least one target
1314 clearBlocked(Blocked_NoTargets);
1315 noTargetMSHR = nullptr;
1316 }
1317
1318 // Initial target is used just for stats
1319 MSHR::Target *initial_tgt = mshr->getTarget();
1320 int stats_cmd_idx = initial_tgt->pkt->cmdToIndex();
1321 Tick miss_latency = curTick() - initial_tgt->recvTime;
1322
1323 if (pkt->req->isUncacheable()) {
1324 assert(pkt->req->masterId() < system->maxMasters());
1325 mshr_uncacheable_lat[stats_cmd_idx][pkt->req->masterId()] +=
1326 miss_latency;
1327 } else {
1328 assert(pkt->req->masterId() < system->maxMasters());
1329 mshr_miss_latency[stats_cmd_idx][pkt->req->masterId()] +=
1330 miss_latency;
1331 }
1332
1333 bool wasFull = mshrQueue.isFull();
1334
1335 PacketList writebacks;
1336
1337 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1338
1339 // upgrade deferred targets if the response has no sharers, and is
1340 // thus passing writable
1341 if (!pkt->hasSharers()) {
1342 mshr->promoteWritable();
1343 }
1344
1345 bool is_fill = !mshr->isForward &&
1346 (pkt->isRead() || pkt->cmd == MemCmd::UpgradeResp);
1347
1348 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
1349
1350 if (is_fill && !is_error) {
1351 DPRINTF(Cache, "Block for addr %#llx being updated in Cache\n",
1352 pkt->getAddr());
1353
1354 blk = handleFill(pkt, blk, writebacks, mshr->allocOnFill());
1355 assert(blk != nullptr);
1356 }
1357
1358 // allow invalidation responses originating from write-line
1359 // requests to be discarded
1360 bool is_invalidate = pkt->isInvalidate();
1361
1362 // First offset for critical word first calculations
1363 int initial_offset = initial_tgt->pkt->getOffset(blkSize);
1364
1365 bool from_cache = false;
1366 MSHR::TargetList targets = mshr->extractServiceableTargets(pkt);
1367 for (auto &target: targets) {
1368 Packet *tgt_pkt = target.pkt;
1369 switch (target.source) {
1370 case MSHR::Target::FromCPU:
1371 Tick completion_time;
1372 // Here we charge on completion_time the delay of the xbar if the
1373 // packet comes from it, charged on headerDelay.
1374 completion_time = pkt->headerDelay;
1375
1376 // Software prefetch handling for cache closest to core
1377 if (tgt_pkt->cmd.isSWPrefetch()) {
1378 // a software prefetch would have already been ack'd
1379 // immediately with dummy data so the core would be able to
1380 // retire it. This request completes right here, so we
1381 // deallocate it.
1382 delete tgt_pkt->req;
1383 delete tgt_pkt;
1384 break; // skip response
1385 }
1386
1387 // keep track of whether we have responded to another
1388 // cache
1389 from_cache = from_cache || tgt_pkt->fromCache();
1390
1391 // unlike the other packet flows, where data is found in other
1392 // caches or memory and brought back, write-line requests always
1393 // have the data right away, so the above check for "is fill?"
1394 // cannot actually be determined until examining the stored MSHR
1395 // state. We "catch up" with that logic here, which is duplicated
1396 // from above.
1397 if (tgt_pkt->cmd == MemCmd::WriteLineReq) {
1398 assert(!is_error);
1399 // we got the block in a writable state, so promote
1400 // any deferred targets if possible
1401 mshr->promoteWritable();
1402 // NB: we use the original packet here and not the response!
1403 blk = handleFill(tgt_pkt, blk, writebacks,
1404 targets.allocOnFill);
1405 assert(blk != nullptr);
1406
1407 // treat as a fill, and discard the invalidation
1408 // response
1409 is_fill = true;
1410 is_invalidate = false;
1411 }
1412
1413 if (is_fill) {
1414 satisfyRequest(tgt_pkt, blk, true, mshr->hasPostDowngrade());
1415
1416 // How many bytes past the first request is this one
1417 int transfer_offset =
1418 tgt_pkt->getOffset(blkSize) - initial_offset;
1419 if (transfer_offset < 0) {
1420 transfer_offset += blkSize;
1421 }
1422
1423 // If not critical word (offset) return payloadDelay.
1424 // responseLatency is the latency of the return path
1425 // from lower level caches/memory to an upper level cache or
1426 // the core.
1427 completion_time += clockEdge(responseLatency) +
1428 (transfer_offset ? pkt->payloadDelay : 0);
1429
1430 assert(!tgt_pkt->req->isUncacheable());
1431
1432 assert(tgt_pkt->req->masterId() < system->maxMasters());
1433 missLatency[tgt_pkt->cmdToIndex()][tgt_pkt->req->masterId()] +=
1434 completion_time - target.recvTime;
1435 } else if (pkt->cmd == MemCmd::UpgradeFailResp) {
1436 // failed StoreCond upgrade
1437 assert(tgt_pkt->cmd == MemCmd::StoreCondReq ||
1438 tgt_pkt->cmd == MemCmd::StoreCondFailReq ||
1439 tgt_pkt->cmd == MemCmd::SCUpgradeFailReq);
1440 // responseLatency is the latency of the return path
1441 // from lower level caches/memory to an upper level cache or
1442 // the core.
1443 completion_time += clockEdge(responseLatency) +
1444 pkt->payloadDelay;
1445 tgt_pkt->req->setExtraData(0);
1446 } else {
1447 // We are about to send a response to a cache above
1448 // that asked for an invalidation; we need to
1449 // invalidate our copy immediately as the most
1450 // up-to-date copy of the block will now be in the
1451 // cache above. It will also prevent this cache from
1452 // responding (if the block was previously dirty) to
1453 // snoops as they should snoop the caches above where
1454 // they will get the response from.
1455 if (is_invalidate && blk && blk->isValid()) {
1456 invalidateBlock(blk);
1457 }
1458 // not a cache fill, just forwarding response
1459 // responseLatency is the latency of the return path
1460 // from lower level cahces/memory to the core.
1461 completion_time += clockEdge(responseLatency) +
1462 pkt->payloadDelay;
1463 if (pkt->isRead() && !is_error) {
1464 // sanity check
1465 assert(pkt->getAddr() == tgt_pkt->getAddr());
1466 assert(pkt->getSize() >= tgt_pkt->getSize());
1467
1468 tgt_pkt->setData(pkt->getConstPtr<uint8_t>());
1469 }
1470 }
1471 tgt_pkt->makeTimingResponse();
1472 // if this packet is an error copy that to the new packet
1473 if (is_error)
1474 tgt_pkt->copyError(pkt);
1475 if (tgt_pkt->cmd == MemCmd::ReadResp &&
1476 (is_invalidate || mshr->hasPostInvalidate())) {
1477 // If intermediate cache got ReadRespWithInvalidate,
1478 // propagate that. Response should not have
1479 // isInvalidate() set otherwise.
1480 tgt_pkt->cmd = MemCmd::ReadRespWithInvalidate;
1481 DPRINTF(Cache, "%s: updated cmd to %s\n", __func__,
1482 tgt_pkt->print());
1483 }
1484 // Reset the bus additional time as it is now accounted for
1485 tgt_pkt->headerDelay = tgt_pkt->payloadDelay = 0;
1486 cpuSidePort->schedTimingResp(tgt_pkt, completion_time, true);
1487 break;
1488
1489 case MSHR::Target::FromPrefetcher:
1490 assert(tgt_pkt->cmd == MemCmd::HardPFReq);
1491 if (blk)
1492 blk->status |= BlkHWPrefetched;
1493 delete tgt_pkt->req;
1494 delete tgt_pkt;
1495 break;
1496
1497 case MSHR::Target::FromSnoop:
1498 // I don't believe that a snoop can be in an error state
1499 assert(!is_error);
1500 // response to snoop request
1501 DPRINTF(Cache, "processing deferred snoop...\n");
1502 // If the response is invalidating, a snooping target can
1503 // be satisfied if it is also invalidating. If the reponse is, not
1504 // only invalidating, but more specifically an InvalidateResp, the
1505 // MSHR was created due to an InvalidateReq and a cache above is
1506 // waiting to satisfy a WriteLineReq. In this case even an
1507 // non-invalidating snoop is added as a target here since this is
1508 // the ordering point. When the InvalidateResp reaches this cache,
1509 // the snooping target will snoop further the cache above with the
1510 // WriteLineReq.
1511 assert(!(is_invalidate &&
1512 pkt->cmd != MemCmd::InvalidateResp &&
1513 !mshr->hasPostInvalidate()));
1514 handleSnoop(tgt_pkt, blk, true, true, mshr->hasPostInvalidate());
1515 break;
1516
1517 default:
1518 panic("Illegal target->source enum %d\n", target.source);
1519 }
1520 }
1521
1522 maintainClusivity(from_cache, blk);
1523
1524 if (blk && blk->isValid()) {
1525 // an invalidate response stemming from a write line request
1526 // should not invalidate the block, so check if the
1527 // invalidation should be discarded
1528 if (is_invalidate || mshr->hasPostInvalidate()) {
1529 invalidateBlock(blk);
1530 } else if (mshr->hasPostDowngrade()) {
1531 blk->status &= ~BlkWritable;
1532 }
1533 }
1534
1535 if (mshr->promoteDeferredTargets()) {
1536 // avoid later read getting stale data while write miss is
1537 // outstanding.. see comment in timingAccess()
1538 if (blk) {
1539 blk->status &= ~BlkReadable;
1540 }
1541 mshrQueue.markPending(mshr);
1542 schedMemSideSendEvent(clockEdge() + pkt->payloadDelay);
1543 } else {
1544 mshrQueue.deallocate(mshr);
1545 if (wasFull && !mshrQueue.isFull()) {
1546 clearBlocked(Blocked_NoMSHRs);
1547 }
1548
1549 // Request the bus for a prefetch if this deallocation freed enough
1550 // MSHRs for a prefetch to take place
1551 if (prefetcher && mshrQueue.canPrefetch()) {
1552 Tick next_pf_time = std::max(prefetcher->nextPrefetchReadyTime(),
1553 clockEdge());
1554 if (next_pf_time != MaxTick)
1555 schedMemSideSendEvent(next_pf_time);
1556 }
1557 }
1558 // reset the xbar additional timinig as it is now accounted for
1559 pkt->headerDelay = pkt->payloadDelay = 0;
1560
1561 // copy writebacks to write buffer
1562 doWritebacks(writebacks, forward_time);
1563
1564 // if we used temp block, check to see if its valid and then clear it out
1565 if (blk == tempBlock && tempBlock->isValid()) {
1566 // We use forwardLatency here because we are copying
1567 // Writebacks/CleanEvicts to write buffer. It specifies the latency to
1568 // allocate an internal buffer and to schedule an event to the
1569 // queued port.
1570 if (blk->isDirty() || writebackClean) {
1571 PacketPtr wbPkt = writebackBlk(blk);
1572 allocateWriteBuffer(wbPkt, forward_time);
1573 // Set BLOCK_CACHED flag if cached above.
1574 if (isCachedAbove(wbPkt))
1575 wbPkt->setBlockCached();
1576 } else {
1577 PacketPtr wcPkt = cleanEvictBlk(blk);
1578 // Check to see if block is cached above. If not allocate
1579 // write buffer
1580 if (isCachedAbove(wcPkt))
1581 delete wcPkt;
1582 else
1583 allocateWriteBuffer(wcPkt, forward_time);
1584 }
1585 invalidateBlock(blk);
1586 }
1587
1588 DPRINTF(CacheVerbose, "%s: Leaving with %s\n", __func__, pkt->print());
1589 delete pkt;
1590}
1591
1592PacketPtr
1593Cache::writebackBlk(CacheBlk *blk)
1594{
1595 chatty_assert(!isReadOnly || writebackClean,
1596 "Writeback from read-only cache");
1597 assert(blk && blk->isValid() && (blk->isDirty() || writebackClean));
1598
1599 writebacks[Request::wbMasterId]++;
1600
1601 Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1602 blkSize, 0, Request::wbMasterId);
1603 if (blk->isSecure())
1604 req->setFlags(Request::SECURE);
1605
1606 req->taskId(blk->task_id);
1607 blk->task_id= ContextSwitchTaskId::Unknown;
1608 blk->tickInserted = curTick();
1609
1610 PacketPtr pkt =
1611 new Packet(req, blk->isDirty() ?
1612 MemCmd::WritebackDirty : MemCmd::WritebackClean);
1613
1614 DPRINTF(Cache, "Create Writeback %s writable: %d, dirty: %d\n",
1615 pkt->print(), blk->isWritable(), blk->isDirty());
1616
1617 if (blk->isWritable()) {
1618 // not asserting shared means we pass the block in modified
1619 // state, mark our own block non-writeable
1620 blk->status &= ~BlkWritable;
1621 } else {
1622 // we are in the Owned state, tell the receiver
1623 pkt->setHasSharers();
1624 }
1625
1626 // make sure the block is not marked dirty
1627 blk->status &= ~BlkDirty;
1628
1629 pkt->allocate();
1630 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1631
1632 return pkt;
1633}
1634
1635PacketPtr
|
1626Cache::writecleanBlk(CacheBlk *blk)
| 1636Cache::writecleanBlk(CacheBlk *blk, Request::Flags dest)
|
1627{
1628 Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1629 blkSize, 0, Request::wbMasterId);
1630 if (blk->isSecure()) {
1631 req->setFlags(Request::SECURE);
1632 }
1633 req->taskId(blk->task_id);
1634 blk->task_id = ContextSwitchTaskId::Unknown;
1635 PacketPtr pkt = new Packet(req, MemCmd::WriteClean);
1636 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1637 blk->isWritable(), blk->isDirty());
1638 // make sure the block is not marked dirty
1639 blk->status &= ~BlkDirty;
1640 pkt->allocate();
1641 // We inform the cache below that the block has sharers in the
1642 // system as we retain our copy.
1643 pkt->setHasSharers();
| 1637{
1638 Request *req = new Request(tags->regenerateBlkAddr(blk->tag, blk->set),
1639 blkSize, 0, Request::wbMasterId);
1640 if (blk->isSecure()) {
1641 req->setFlags(Request::SECURE);
1642 }
1643 req->taskId(blk->task_id);
1644 blk->task_id = ContextSwitchTaskId::Unknown;
1645 PacketPtr pkt = new Packet(req, MemCmd::WriteClean);
1646 DPRINTF(Cache, "Create %s writable: %d, dirty: %d\n", pkt->print(),
1647 blk->isWritable(), blk->isDirty());
1648 // make sure the block is not marked dirty
1649 blk->status &= ~BlkDirty;
1650 pkt->allocate();
1651 // We inform the cache below that the block has sharers in the
1652 // system as we retain our copy.
1653 pkt->setHasSharers();
|
| 1654 if (dest) {
1655 req->setFlags(dest);
1656 pkt->setWriteThrough();
1657 }
|
1644 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1645 return pkt;
1646}
1647
1648
1649PacketPtr
1650Cache::cleanEvictBlk(CacheBlk *blk)
1651{
1652 assert(!writebackClean);
1653 assert(blk && blk->isValid() && !blk->isDirty());
1654 // Creating a zero sized write, a message to the snoop filter
1655 Request *req =
1656 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1657 Request::wbMasterId);
1658 if (blk->isSecure())
1659 req->setFlags(Request::SECURE);
1660
1661 req->taskId(blk->task_id);
1662 blk->task_id = ContextSwitchTaskId::Unknown;
1663 blk->tickInserted = curTick();
1664
1665 PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1666 pkt->allocate();
1667 DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1668
1669 return pkt;
1670}
1671
1672void
1673Cache::memWriteback()
1674{
1675 CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1676 tags->forEachBlk(visitor);
1677}
1678
1679void
1680Cache::memInvalidate()
1681{
1682 CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1683 tags->forEachBlk(visitor);
1684}
1685
1686bool
1687Cache::isDirty() const
1688{
1689 CacheBlkIsDirtyVisitor visitor;
1690 tags->forEachBlk(visitor);
1691
1692 return visitor.isDirty();
1693}
1694
1695bool
1696Cache::writebackVisitor(CacheBlk &blk)
1697{
1698 if (blk.isDirty()) {
1699 assert(blk.isValid());
1700
1701 Request request(tags->regenerateBlkAddr(blk.tag, blk.set),
1702 blkSize, 0, Request::funcMasterId);
1703 request.taskId(blk.task_id);
1704 if (blk.isSecure()) {
1705 request.setFlags(Request::SECURE);
1706 }
1707
1708 Packet packet(&request, MemCmd::WriteReq);
1709 packet.dataStatic(blk.data);
1710
1711 memSidePort->sendFunctional(&packet);
1712
1713 blk.status &= ~BlkDirty;
1714 }
1715
1716 return true;
1717}
1718
1719bool
1720Cache::invalidateVisitor(CacheBlk &blk)
1721{
1722
1723 if (blk.isDirty())
1724 warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1725
1726 if (blk.isValid()) {
1727 assert(!blk.isDirty());
1728 invalidateBlock(&blk);
1729 }
1730
1731 return true;
1732}
1733
1734CacheBlk*
1735Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1736{
1737 CacheBlk *blk = tags->findVictim(addr);
1738
1739 // It is valid to return nullptr if there is no victim
1740 if (!blk)
1741 return nullptr;
1742
1743 if (blk->isValid()) {
1744 Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set);
1745 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1746 if (repl_mshr) {
1747 // must be an outstanding upgrade request
1748 // on a block we're about to replace...
1749 assert(!blk->isWritable() || blk->isDirty());
1750 assert(repl_mshr->needsWritable());
1751 // too hard to replace block with transient state
1752 // allocation failed, block not inserted
1753 return nullptr;
1754 } else {
1755 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1756 "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1757 addr, is_secure ? "s" : "ns",
1758 blk->isDirty() ? "writeback" : "clean");
1759
1760 if (blk->wasPrefetched()) {
1761 unusedPrefetches++;
1762 }
1763 // Will send up Writeback/CleanEvict snoops via isCachedAbove
1764 // when pushing this writeback list into the write buffer.
1765 if (blk->isDirty() || writebackClean) {
1766 // Save writeback packet for handling by caller
1767 writebacks.push_back(writebackBlk(blk));
1768 } else {
1769 writebacks.push_back(cleanEvictBlk(blk));
1770 }
1771 }
1772 }
1773
1774 return blk;
1775}
1776
1777void
1778Cache::invalidateBlock(CacheBlk *blk)
1779{
1780 if (blk != tempBlock)
1781 tags->invalidate(blk);
1782 blk->invalidate();
1783}
1784
1785// Note that the reason we return a list of writebacks rather than
1786// inserting them directly in the write buffer is that this function
1787// is called by both atomic and timing-mode accesses, and in atomic
1788// mode we don't mess with the write buffer (we just perform the
1789// writebacks atomically once the original request is complete).
1790CacheBlk*
1791Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1792 bool allocate)
1793{
1794 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1795 Addr addr = pkt->getAddr();
1796 bool is_secure = pkt->isSecure();
1797#if TRACING_ON
1798 CacheBlk::State old_state = blk ? blk->status : 0;
1799#endif
1800
1801 // When handling a fill, we should have no writes to this line.
1802 assert(addr == pkt->getBlockAddr(blkSize));
1803 assert(!writeBuffer.findMatch(addr, is_secure));
1804
1805 if (blk == nullptr) {
1806 // better have read new data...
1807 assert(pkt->hasData());
1808
1809 // only read responses and write-line requests have data;
1810 // note that we don't write the data here for write-line - that
1811 // happens in the subsequent call to satisfyRequest
1812 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1813
1814 // need to do a replacement if allocating, otherwise we stick
1815 // with the temporary storage
1816 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1817
1818 if (blk == nullptr) {
1819 // No replaceable block or a mostly exclusive
1820 // cache... just use temporary storage to complete the
1821 // current request and then get rid of it
1822 assert(!tempBlock->isValid());
1823 blk = tempBlock;
1824 tempBlock->set = tags->extractSet(addr);
1825 tempBlock->tag = tags->extractTag(addr);
1826 // @todo: set security state as well...
1827 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1828 is_secure ? "s" : "ns");
1829 } else {
1830 tags->insertBlock(pkt, blk);
1831 }
1832
1833 // we should never be overwriting a valid block
1834 assert(!blk->isValid());
1835 } else {
1836 // existing block... probably an upgrade
1837 assert(blk->tag == tags->extractTag(addr));
1838 // either we're getting new data or the block should already be valid
1839 assert(pkt->hasData() || blk->isValid());
1840 // don't clear block status... if block is already dirty we
1841 // don't want to lose that
1842 }
1843
1844 if (is_secure)
1845 blk->status |= BlkSecure;
1846 blk->status |= BlkValid | BlkReadable;
1847
1848 // sanity check for whole-line writes, which should always be
1849 // marked as writable as part of the fill, and then later marked
1850 // dirty as part of satisfyRequest
1851 if (pkt->cmd == MemCmd::WriteLineReq) {
1852 assert(!pkt->hasSharers());
1853 }
1854
1855 // here we deal with setting the appropriate state of the line,
1856 // and we start by looking at the hasSharers flag, and ignore the
1857 // cacheResponding flag (normally signalling dirty data) if the
1858 // packet has sharers, thus the line is never allocated as Owned
1859 // (dirty but not writable), and always ends up being either
1860 // Shared, Exclusive or Modified, see Packet::setCacheResponding
1861 // for more details
1862 if (!pkt->hasSharers()) {
1863 // we could get a writable line from memory (rather than a
1864 // cache) even in a read-only cache, note that we set this bit
1865 // even for a read-only cache, possibly revisit this decision
1866 blk->status |= BlkWritable;
1867
1868 // check if we got this via cache-to-cache transfer (i.e., from a
1869 // cache that had the block in Modified or Owned state)
1870 if (pkt->cacheResponding()) {
1871 // we got the block in Modified state, and invalidated the
1872 // owners copy
1873 blk->status |= BlkDirty;
1874
1875 chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1876 "in read-only cache %s\n", name());
1877 }
1878 }
1879
1880 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1881 addr, is_secure ? "s" : "ns", old_state, blk->print());
1882
1883 // if we got new data, copy it in (checking for a read response
1884 // and a response that has data is the same in the end)
1885 if (pkt->isRead()) {
1886 // sanity checks
1887 assert(pkt->hasData());
1888 assert(pkt->getSize() == blkSize);
1889
1890 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
1891 }
1892 // We pay for fillLatency here.
1893 blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1894 pkt->payloadDelay;
1895
1896 return blk;
1897}
1898
1899
1900/////////////////////////////////////////////////////
1901//
1902// Snoop path: requests coming in from the memory side
1903//
1904/////////////////////////////////////////////////////
1905
1906void
1907Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1908 bool already_copied, bool pending_inval)
1909{
1910 // sanity check
1911 assert(req_pkt->isRequest());
1912 assert(req_pkt->needsResponse());
1913
1914 DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1915 // timing-mode snoop responses require a new packet, unless we
1916 // already made a copy...
1917 PacketPtr pkt = req_pkt;
1918 if (!already_copied)
1919 // do not clear flags, and allocate space for data if the
1920 // packet needs it (the only packets that carry data are read
1921 // responses)
1922 pkt = new Packet(req_pkt, false, req_pkt->isRead());
1923
1924 assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
1925 pkt->hasSharers());
1926 pkt->makeTimingResponse();
1927 if (pkt->isRead()) {
1928 pkt->setDataFromBlock(blk_data, blkSize);
1929 }
1930 if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
1931 // Assume we defer a response to a read from a far-away cache
1932 // A, then later defer a ReadExcl from a cache B on the same
1933 // bus as us. We'll assert cacheResponding in both cases, but
1934 // in the latter case cacheResponding will keep the
1935 // invalidation from reaching cache A. This special response
1936 // tells cache A that it gets the block to satisfy its read,
1937 // but must immediately invalidate it.
1938 pkt->cmd = MemCmd::ReadRespWithInvalidate;
1939 }
1940 // Here we consider forward_time, paying for just forward latency and
1941 // also charging the delay provided by the xbar.
1942 // forward_time is used as send_time in next allocateWriteBuffer().
1943 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1944 // Here we reset the timing of the packet.
1945 pkt->headerDelay = pkt->payloadDelay = 0;
1946 DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
1947 pkt->print(), forward_time);
1948 memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
1949}
1950
1951uint32_t
1952Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
1953 bool is_deferred, bool pending_inval)
1954{
1955 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
1956 // deferred snoops can only happen in timing mode
1957 assert(!(is_deferred && !is_timing));
1958 // pending_inval only makes sense on deferred snoops
1959 assert(!(pending_inval && !is_deferred));
1960 assert(pkt->isRequest());
1961
1962 // the packet may get modified if we or a forwarded snooper
1963 // responds in atomic mode, so remember a few things about the
1964 // original packet up front
1965 bool invalidate = pkt->isInvalidate();
1966 bool M5_VAR_USED needs_writable = pkt->needsWritable();
1967
1968 // at the moment we could get an uncacheable write which does not
1969 // have the invalidate flag, and we need a suitable way of dealing
1970 // with this case
1971 panic_if(invalidate && pkt->req->isUncacheable(),
1972 "%s got an invalidating uncacheable snoop request %s",
1973 name(), pkt->print());
1974
1975 uint32_t snoop_delay = 0;
1976
1977 if (forwardSnoops) {
1978 // first propagate snoop upward to see if anyone above us wants to
1979 // handle it. save & restore packet src since it will get
1980 // rewritten to be relative to cpu-side bus (if any)
1981 bool alreadyResponded = pkt->cacheResponding();
1982 if (is_timing) {
1983 // copy the packet so that we can clear any flags before
1984 // forwarding it upwards, we also allocate data (passing
1985 // the pointer along in case of static data), in case
1986 // there is a snoop hit in upper levels
1987 Packet snoopPkt(pkt, true, true);
1988 snoopPkt.setExpressSnoop();
1989 // the snoop packet does not need to wait any additional
1990 // time
1991 snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
1992 cpuSidePort->sendTimingSnoopReq(&snoopPkt);
1993
1994 // add the header delay (including crossbar and snoop
1995 // delays) of the upward snoop to the snoop delay for this
1996 // cache
1997 snoop_delay += snoopPkt.headerDelay;
1998
1999 if (snoopPkt.cacheResponding()) {
2000 // cache-to-cache response from some upper cache
2001 assert(!alreadyResponded);
2002 pkt->setCacheResponding();
2003 }
2004 // upstream cache has the block, or has an outstanding
2005 // MSHR, pass the flag on
2006 if (snoopPkt.hasSharers()) {
2007 pkt->setHasSharers();
2008 }
2009 // If this request is a prefetch or clean evict and an upper level
2010 // signals block present, make sure to propagate the block
2011 // presence to the requester.
2012 if (snoopPkt.isBlockCached()) {
2013 pkt->setBlockCached();
2014 }
2015 } else {
2016 cpuSidePort->sendAtomicSnoop(pkt);
2017 if (!alreadyResponded && pkt->cacheResponding()) {
2018 // cache-to-cache response from some upper cache:
2019 // forward response to original requester
2020 assert(pkt->isResponse());
2021 }
2022 }
2023 }
2024
2025 if (!blk || !blk->isValid()) {
2026 DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2027 pkt->print());
2028 if (is_deferred) {
2029 // we no longer have the block, and will not respond, but a
2030 // packet was allocated in MSHR::handleSnoop and we have
2031 // to delete it
2032 assert(pkt->needsResponse());
2033
2034 // we have passed the block to a cache upstream, that
2035 // cache should be responding
2036 assert(pkt->cacheResponding());
2037
2038 delete pkt;
2039 }
2040 return snoop_delay;
2041 } else {
2042 DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2043 pkt->print(), blk->print());
2044 }
2045
2046 chatty_assert(!(isReadOnly && blk->isDirty()),
2047 "Should never have a dirty block in a read-only cache %s\n",
2048 name());
2049
2050 // We may end up modifying both the block state and the packet (if
2051 // we respond in atomic mode), so just figure out what to do now
2052 // and then do it later. We respond to all snoops that need
2053 // responses provided we have the block in dirty state. The
2054 // invalidation itself is taken care of below.
2055 bool respond = blk->isDirty() && pkt->needsResponse();
2056 bool have_writable = blk->isWritable();
2057
2058 // Invalidate any prefetch's from below that would strip write permissions
2059 // MemCmd::HardPFReq is only observed by upstream caches. After missing
2060 // above and in it's own cache, a new MemCmd::ReadReq is created that
2061 // downstream caches observe.
2062 if (pkt->mustCheckAbove()) {
2063 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2064 "from lower cache\n", pkt->getAddr(), pkt->print());
2065 pkt->setBlockCached();
2066 return snoop_delay;
2067 }
2068
2069 if (pkt->isRead() && !invalidate) {
2070 // reading without requiring the line in a writable state
2071 assert(!needs_writable);
2072 pkt->setHasSharers();
2073
2074 // if the requesting packet is uncacheable, retain the line in
2075 // the current state, otherwhise unset the writable flag,
2076 // which means we go from Modified to Owned (and will respond
2077 // below), remain in Owned (and will respond below), from
2078 // Exclusive to Shared, or remain in Shared
2079 if (!pkt->req->isUncacheable())
2080 blk->status &= ~BlkWritable;
2081 }
2082
2083 if (respond) {
2084 // prevent anyone else from responding, cache as well as
2085 // memory, and also prevent any memory from even seeing the
2086 // request
2087 pkt->setCacheResponding();
2088 if (have_writable) {
2089 // inform the cache hierarchy that this cache had the line
2090 // in the Modified state so that we avoid unnecessary
2091 // invalidations (see Packet::setResponderHadWritable)
2092 pkt->setResponderHadWritable();
2093
2094 // in the case of an uncacheable request there is no point
2095 // in setting the responderHadWritable flag, but since the
2096 // recipient does not care there is no harm in doing so
2097 } else {
2098 // if the packet has needsWritable set we invalidate our
2099 // copy below and all other copies will be invalidates
2100 // through express snoops, and if needsWritable is not set
2101 // we already called setHasSharers above
2102 }
2103
2104 // if we are returning a writable and dirty (Modified) line,
2105 // we should be invalidating the line
2106 panic_if(!invalidate && !pkt->hasSharers(),
2107 "%s is passing a Modified line through %s, "
2108 "but keeping the block", name(), pkt->print());
2109
2110 if (is_timing) {
2111 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2112 } else {
2113 pkt->makeAtomicResponse();
2114 // packets such as upgrades do not actually have any data
2115 // payload
2116 if (pkt->hasData())
2117 pkt->setDataFromBlock(blk->data, blkSize);
2118 }
2119 }
2120
2121 if (!respond && is_deferred) {
2122 assert(pkt->needsResponse());
2123
2124 // if we copied the deferred packet with the intention to
2125 // respond, but are not responding, then a cache above us must
2126 // be, and we can use this as the indication of whether this
2127 // is a packet where we created a copy of the request or not
2128 if (!pkt->cacheResponding()) {
2129 delete pkt->req;
2130 }
2131
2132 delete pkt;
2133 }
2134
2135 // Do this last in case it deallocates block data or something
2136 // like that
2137 if (invalidate) {
2138 invalidateBlock(blk);
2139 }
2140
2141 DPRINTF(Cache, "new state is %s\n", blk->print());
2142
2143 return snoop_delay;
2144}
2145
2146
2147void
2148Cache::recvTimingSnoopReq(PacketPtr pkt)
2149{
2150 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2151
2152 // Snoops shouldn't happen when bypassing caches
2153 assert(!system->bypassCaches());
2154
2155 // no need to snoop requests that are not in range
2156 if (!inRange(pkt->getAddr())) {
2157 return;
2158 }
2159
2160 bool is_secure = pkt->isSecure();
2161 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2162
2163 Addr blk_addr = pkt->getBlockAddr(blkSize);
2164 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2165
2166 // Update the latency cost of the snoop so that the crossbar can
2167 // account for it. Do not overwrite what other neighbouring caches
2168 // have already done, rather take the maximum. The update is
2169 // tentative, for cases where we return before an upward snoop
2170 // happens below.
2171 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2172 lookupLatency * clockPeriod());
2173
2174 // Inform request(Prefetch, CleanEvict or Writeback) from below of
2175 // MSHR hit, set setBlockCached.
2176 if (mshr && pkt->mustCheckAbove()) {
2177 DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2178 "mshr hit\n", pkt->print());
2179 pkt->setBlockCached();
2180 return;
2181 }
2182
2183 // Let the MSHR itself track the snoop and decide whether we want
2184 // to go ahead and do the regular cache snoop
2185 if (mshr && mshr->handleSnoop(pkt, order++)) {
2186 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2187 "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2188 mshr->print());
2189
2190 if (mshr->getNumTargets() > numTarget)
2191 warn("allocating bonus target for snoop"); //handle later
2192 return;
2193 }
2194
2195 //We also need to check the writeback buffers and handle those
2196 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2197 if (wb_entry) {
2198 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2199 pkt->getAddr(), is_secure ? "s" : "ns");
2200 // Expect to see only Writebacks and/or CleanEvicts here, both of
2201 // which should not be generated for uncacheable data.
2202 assert(!wb_entry->isUncacheable());
2203 // There should only be a single request responsible for generating
2204 // Writebacks/CleanEvicts.
2205 assert(wb_entry->getNumTargets() == 1);
2206 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2207 assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2208
2209 if (pkt->isEviction()) {
2210 // if the block is found in the write queue, set the BLOCK_CACHED
2211 // flag for Writeback/CleanEvict snoop. On return the snoop will
2212 // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2213 // any CleanEvicts from travelling down the memory hierarchy.
2214 pkt->setBlockCached();
2215 DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2216 "hit\n", __func__, pkt->print());
2217 return;
2218 }
2219
2220 // conceptually writebacks are no different to other blocks in
2221 // this cache, so the behaviour is modelled after handleSnoop,
2222 // the difference being that instead of querying the block
2223 // state to determine if it is dirty and writable, we use the
2224 // command and fields of the writeback packet
2225 bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2226 pkt->needsResponse();
2227 bool have_writable = !wb_pkt->hasSharers();
2228 bool invalidate = pkt->isInvalidate();
2229
2230 if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2231 assert(!pkt->needsWritable());
2232 pkt->setHasSharers();
2233 wb_pkt->setHasSharers();
2234 }
2235
2236 if (respond) {
2237 pkt->setCacheResponding();
2238
2239 if (have_writable) {
2240 pkt->setResponderHadWritable();
2241 }
2242
2243 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2244 false, false);
2245 }
2246
2247 if (invalidate) {
2248 // Invalidation trumps our writeback... discard here
2249 // Note: markInService will remove entry from writeback buffer.
2250 markInService(wb_entry);
2251 delete wb_pkt;
2252 }
2253 }
2254
2255 // If this was a shared writeback, there may still be
2256 // other shared copies above that require invalidation.
2257 // We could be more selective and return here if the
2258 // request is non-exclusive or if the writeback is
2259 // exclusive.
2260 uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2261
2262 // Override what we did when we first saw the snoop, as we now
2263 // also have the cost of the upwards snoops to account for
2264 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2265 lookupLatency * clockPeriod());
2266}
2267
2268bool
2269Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2270{
2271 // Express snoop responses from master to slave, e.g., from L1 to L2
2272 cache->recvTimingSnoopResp(pkt);
2273 return true;
2274}
2275
2276Tick
2277Cache::recvAtomicSnoop(PacketPtr pkt)
2278{
2279 // Snoops shouldn't happen when bypassing caches
2280 assert(!system->bypassCaches());
2281
2282 // no need to snoop requests that are not in range.
2283 if (!inRange(pkt->getAddr())) {
2284 return 0;
2285 }
2286
2287 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2288 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2289 return snoop_delay + lookupLatency * clockPeriod();
2290}
2291
2292
2293QueueEntry*
2294Cache::getNextQueueEntry()
2295{
2296 // Check both MSHR queue and write buffer for potential requests,
2297 // note that null does not mean there is no request, it could
2298 // simply be that it is not ready
2299 MSHR *miss_mshr = mshrQueue.getNext();
2300 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2301
2302 // If we got a write buffer request ready, first priority is a
2303 // full write buffer, otherwise we favour the miss requests
2304 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2305 // need to search MSHR queue for conflicting earlier miss.
2306 MSHR *conflict_mshr =
2307 mshrQueue.findPending(wq_entry->blkAddr,
2308 wq_entry->isSecure);
2309
2310 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2311 // Service misses in order until conflict is cleared.
2312 return conflict_mshr;
2313
2314 // @todo Note that we ignore the ready time of the conflict here
2315 }
2316
2317 // No conflicts; issue write
2318 return wq_entry;
2319 } else if (miss_mshr) {
2320 // need to check for conflicting earlier writeback
2321 WriteQueueEntry *conflict_mshr =
2322 writeBuffer.findPending(miss_mshr->blkAddr,
2323 miss_mshr->isSecure);
2324 if (conflict_mshr) {
2325 // not sure why we don't check order here... it was in the
2326 // original code but commented out.
2327
2328 // The only way this happens is if we are
2329 // doing a write and we didn't have permissions
2330 // then subsequently saw a writeback (owned got evicted)
2331 // We need to make sure to perform the writeback first
2332 // To preserve the dirty data, then we can issue the write
2333
2334 // should we return wq_entry here instead? I.e. do we
2335 // have to flush writes in order? I don't think so... not
2336 // for Alpha anyway. Maybe for x86?
2337 return conflict_mshr;
2338
2339 // @todo Note that we ignore the ready time of the conflict here
2340 }
2341
2342 // No conflicts; issue read
2343 return miss_mshr;
2344 }
2345
2346 // fall through... no pending requests. Try a prefetch.
2347 assert(!miss_mshr && !wq_entry);
2348 if (prefetcher && mshrQueue.canPrefetch()) {
2349 // If we have a miss queue slot, we can try a prefetch
2350 PacketPtr pkt = prefetcher->getPacket();
2351 if (pkt) {
2352 Addr pf_addr = pkt->getBlockAddr(blkSize);
2353 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2354 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2355 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2356 // Update statistic on number of prefetches issued
2357 // (hwpf_mshr_misses)
2358 assert(pkt->req->masterId() < system->maxMasters());
2359 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2360
2361 // allocate an MSHR and return it, note
2362 // that we send the packet straight away, so do not
2363 // schedule the send
2364 return allocateMissBuffer(pkt, curTick(), false);
2365 } else {
2366 // free the request and packet
2367 delete pkt->req;
2368 delete pkt;
2369 }
2370 }
2371 }
2372
2373 return nullptr;
2374}
2375
2376bool
2377Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2378{
2379 if (!forwardSnoops)
2380 return false;
2381 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2382 // Writeback snoops into upper level caches to check for copies of the
2383 // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2384 // packet, the cache can inform the crossbar below of presence or absence
2385 // of the block.
2386 if (is_timing) {
2387 Packet snoop_pkt(pkt, true, false);
2388 snoop_pkt.setExpressSnoop();
2389 // Assert that packet is either Writeback or CleanEvict and not a
2390 // prefetch request because prefetch requests need an MSHR and may
2391 // generate a snoop response.
2392 assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2393 snoop_pkt.senderState = nullptr;
2394 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2395 // Writeback/CleanEvict snoops do not generate a snoop response.
2396 assert(!(snoop_pkt.cacheResponding()));
2397 return snoop_pkt.isBlockCached();
2398 } else {
2399 cpuSidePort->sendAtomicSnoop(pkt);
2400 return pkt->isBlockCached();
2401 }
2402}
2403
2404Tick
2405Cache::nextQueueReadyTime() const
2406{
2407 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2408 writeBuffer.nextReadyTime());
2409
2410 // Don't signal prefetch ready time if no MSHRs available
2411 // Will signal once enoguh MSHRs are deallocated
2412 if (prefetcher && mshrQueue.canPrefetch()) {
2413 nextReady = std::min(nextReady,
2414 prefetcher->nextPrefetchReadyTime());
2415 }
2416
2417 return nextReady;
2418}
2419
2420bool
2421Cache::sendMSHRQueuePacket(MSHR* mshr)
2422{
2423 assert(mshr);
2424
2425 // use request from 1st target
2426 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2427
2428 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2429
2430 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2431
2432 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2433 // we should never have hardware prefetches to allocated
2434 // blocks
2435 assert(blk == nullptr);
2436
2437 // We need to check the caches above us to verify that
2438 // they don't have a copy of this block in the dirty state
2439 // at the moment. Without this check we could get a stale
2440 // copy from memory that might get used in place of the
2441 // dirty one.
2442 Packet snoop_pkt(tgt_pkt, true, false);
2443 snoop_pkt.setExpressSnoop();
2444 // We are sending this packet upwards, but if it hits we will
2445 // get a snoop response that we end up treating just like a
2446 // normal response, hence it needs the MSHR as its sender
2447 // state
2448 snoop_pkt.senderState = mshr;
2449 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2450
2451 // Check to see if the prefetch was squashed by an upper cache (to
2452 // prevent us from grabbing the line) or if a Check to see if a
2453 // writeback arrived between the time the prefetch was placed in
2454 // the MSHRs and when it was selected to be sent or if the
2455 // prefetch was squashed by an upper cache.
2456
2457 // It is important to check cacheResponding before
2458 // prefetchSquashed. If another cache has committed to
2459 // responding, it will be sending a dirty response which will
2460 // arrive at the MSHR allocated for this request. Checking the
2461 // prefetchSquash first may result in the MSHR being
2462 // prematurely deallocated.
2463 if (snoop_pkt.cacheResponding()) {
2464 auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2465 assert(r.second);
2466
2467 // if we are getting a snoop response with no sharers it
2468 // will be allocated as Modified
2469 bool pending_modified_resp = !snoop_pkt.hasSharers();
2470 markInService(mshr, pending_modified_resp);
2471
2472 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2473 " %#x (%s) hit\n",
2474 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2475 return false;
2476 }
2477
2478 if (snoop_pkt.isBlockCached()) {
2479 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2480 "Deallocating mshr target %#x.\n",
2481 mshr->blkAddr);
2482
2483 // Deallocate the mshr target
2484 if (mshrQueue.forceDeallocateTarget(mshr)) {
2485 // Clear block if this deallocation resulted freed an
2486 // mshr when all had previously been utilized
2487 clearBlocked(Blocked_NoMSHRs);
2488 }
2489
2490 // given that no response is expected, delete Request and Packet
2491 delete tgt_pkt->req;
2492 delete tgt_pkt;
2493
2494 return false;
2495 }
2496 }
2497
2498 // either a prefetch that is not present upstream, or a normal
2499 // MSHR request, proceed to get the packet to send downstream
2500 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2501
2502 mshr->isForward = (pkt == nullptr);
2503
2504 if (mshr->isForward) {
2505 // not a cache block request, but a response is expected
2506 // make copy of current packet to forward, keep current
2507 // copy for response handling
2508 pkt = new Packet(tgt_pkt, false, true);
2509 assert(!pkt->isWrite());
2510 }
2511
2512 // play it safe and append (rather than set) the sender state,
2513 // as forwarded packets may already have existing state
2514 pkt->pushSenderState(mshr);
2515
2516 if (!memSidePort->sendTimingReq(pkt)) {
2517 // we are awaiting a retry, but we
2518 // delete the packet and will be creating a new packet
2519 // when we get the opportunity
2520 delete pkt;
2521
2522 // note that we have now masked any requestBus and
2523 // schedSendEvent (we will wait for a retry before
2524 // doing anything), and this is so even if we do not
2525 // care about this packet and might override it before
2526 // it gets retried
2527 return true;
2528 } else {
2529 // As part of the call to sendTimingReq the packet is
2530 // forwarded to all neighbouring caches (and any caches
2531 // above them) as a snoop. Thus at this point we know if
2532 // any of the neighbouring caches are responding, and if
2533 // so, we know it is dirty, and we can determine if it is
2534 // being passed as Modified, making our MSHR the ordering
2535 // point
2536 bool pending_modified_resp = !pkt->hasSharers() &&
2537 pkt->cacheResponding();
2538 markInService(mshr, pending_modified_resp);
2539 return false;
2540 }
2541}
2542
2543bool
2544Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2545{
2546 assert(wq_entry);
2547
2548 // always a single target for write queue entries
2549 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2550
2551 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2552
2553 // forward as is, both for evictions and uncacheable writes
2554 if (!memSidePort->sendTimingReq(tgt_pkt)) {
2555 // note that we have now masked any requestBus and
2556 // schedSendEvent (we will wait for a retry before
2557 // doing anything), and this is so even if we do not
2558 // care about this packet and might override it before
2559 // it gets retried
2560 return true;
2561 } else {
2562 markInService(wq_entry);
2563 return false;
2564 }
2565}
2566
2567void
2568Cache::serialize(CheckpointOut &cp) const
2569{
2570 bool dirty(isDirty());
2571
2572 if (dirty) {
2573 warn("*** The cache still contains dirty data. ***\n");
2574 warn(" Make sure to drain the system using the correct flags.\n");
2575 warn(" This checkpoint will not restore correctly and dirty data "
2576 " in the cache will be lost!\n");
2577 }
2578
2579 // Since we don't checkpoint the data in the cache, any dirty data
2580 // will be lost when restoring from a checkpoint of a system that
2581 // wasn't drained properly. Flag the checkpoint as invalid if the
2582 // cache contains dirty data.
2583 bool bad_checkpoint(dirty);
2584 SERIALIZE_SCALAR(bad_checkpoint);
2585}
2586
2587void
2588Cache::unserialize(CheckpointIn &cp)
2589{
2590 bool bad_checkpoint;
2591 UNSERIALIZE_SCALAR(bad_checkpoint);
2592 if (bad_checkpoint) {
2593 fatal("Restoring from checkpoints with dirty caches is not supported "
2594 "in the classic memory system. Please remove any caches or "
2595 " drain them properly before taking checkpoints.\n");
2596 }
2597}
2598
2599///////////////
2600//
2601// CpuSidePort
2602//
2603///////////////
2604
2605AddrRangeList
2606Cache::CpuSidePort::getAddrRanges() const
2607{
2608 return cache->getAddrRanges();
2609}
2610
2611bool
2612Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2613{
2614 assert(!cache->system->bypassCaches());
2615
2616 // always let express snoop packets through if even if blocked
2617 if (pkt->isExpressSnoop()) {
2618 return true;
2619 } else if (isBlocked() || mustSendRetry) {
2620 // either already committed to send a retry, or blocked
2621 mustSendRetry = true;
2622 return false;
2623 }
2624 mustSendRetry = false;
2625 return true;
2626}
2627
2628bool
2629Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2630{
2631 assert(!cache->system->bypassCaches());
2632
2633 // always let express snoop packets through if even if blocked
2634 if (pkt->isExpressSnoop()) {
2635 bool M5_VAR_USED bypass_success = cache->recvTimingReq(pkt);
2636 assert(bypass_success);
2637 return true;
2638 }
2639
2640 return tryTiming(pkt) && cache->recvTimingReq(pkt);
2641}
2642
2643Tick
2644Cache::CpuSidePort::recvAtomic(PacketPtr pkt)
2645{
2646 return cache->recvAtomic(pkt);
2647}
2648
2649void
2650Cache::CpuSidePort::recvFunctional(PacketPtr pkt)
2651{
2652 // functional request
2653 cache->functionalAccess(pkt, true);
2654}
2655
2656Cache::
2657CpuSidePort::CpuSidePort(const std::string &_name, Cache *_cache,
2658 const std::string &_label)
2659 : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2660{
2661}
2662
2663Cache*
2664CacheParams::create()
2665{
2666 assert(tags);
2667
2668 return new Cache(this);
2669}
2670///////////////
2671//
2672// MemSidePort
2673//
2674///////////////
2675
2676bool
2677Cache::MemSidePort::recvTimingResp(PacketPtr pkt)
2678{
2679 cache->recvTimingResp(pkt);
2680 return true;
2681}
2682
2683// Express snooping requests to memside port
2684void
2685Cache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2686{
2687 // handle snooping requests
2688 cache->recvTimingSnoopReq(pkt);
2689}
2690
2691Tick
2692Cache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2693{
2694 return cache->recvAtomicSnoop(pkt);
2695}
2696
2697void
2698Cache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2699{
2700 // functional snoop (note that in contrast to atomic we don't have
2701 // a specific functionalSnoop method, as they have the same
2702 // behaviour regardless)
2703 cache->functionalAccess(pkt, false);
2704}
2705
2706void
2707Cache::CacheReqPacketQueue::sendDeferredPacket()
2708{
2709 // sanity check
2710 assert(!waitingOnRetry);
2711
2712 // there should never be any deferred request packets in the
2713 // queue, instead we resly on the cache to provide the packets
2714 // from the MSHR queue or write queue
2715 assert(deferredPacketReadyTime() == MaxTick);
2716
2717 // check for request packets (requests & writebacks)
2718 QueueEntry* entry = cache.getNextQueueEntry();
2719
2720 if (!entry) {
2721 // can happen if e.g. we attempt a writeback and fail, but
2722 // before the retry, the writeback is eliminated because
2723 // we snoop another cache's ReadEx.
2724 } else {
2725 // let our snoop responses go first if there are responses to
2726 // the same addresses
2727 if (checkConflictingSnoop(entry->blkAddr)) {
2728 return;
2729 }
2730 waitingOnRetry = entry->sendPacket(cache);
2731 }
2732
2733 // if we succeeded and are not waiting for a retry, schedule the
2734 // next send considering when the next queue is ready, note that
2735 // snoop responses have their own packet queue and thus schedule
2736 // their own events
2737 if (!waitingOnRetry) {
2738 schedSendEvent(cache.nextQueueReadyTime());
2739 }
2740}
2741
2742Cache::
2743MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2744 const std::string &_label)
2745 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2746 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2747 _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2748{
2749}
| 1658 std::memcpy(pkt->getPtr<uint8_t>(), blk->data, blkSize);
1659 return pkt;
1660}
1661
1662
1663PacketPtr
1664Cache::cleanEvictBlk(CacheBlk *blk)
1665{
1666 assert(!writebackClean);
1667 assert(blk && blk->isValid() && !blk->isDirty());
1668 // Creating a zero sized write, a message to the snoop filter
1669 Request *req =
1670 new Request(tags->regenerateBlkAddr(blk->tag, blk->set), blkSize, 0,
1671 Request::wbMasterId);
1672 if (blk->isSecure())
1673 req->setFlags(Request::SECURE);
1674
1675 req->taskId(blk->task_id);
1676 blk->task_id = ContextSwitchTaskId::Unknown;
1677 blk->tickInserted = curTick();
1678
1679 PacketPtr pkt = new Packet(req, MemCmd::CleanEvict);
1680 pkt->allocate();
1681 DPRINTF(Cache, "Create CleanEvict %s\n", pkt->print());
1682
1683 return pkt;
1684}
1685
1686void
1687Cache::memWriteback()
1688{
1689 CacheBlkVisitorWrapper visitor(*this, &Cache::writebackVisitor);
1690 tags->forEachBlk(visitor);
1691}
1692
1693void
1694Cache::memInvalidate()
1695{
1696 CacheBlkVisitorWrapper visitor(*this, &Cache::invalidateVisitor);
1697 tags->forEachBlk(visitor);
1698}
1699
1700bool
1701Cache::isDirty() const
1702{
1703 CacheBlkIsDirtyVisitor visitor;
1704 tags->forEachBlk(visitor);
1705
1706 return visitor.isDirty();
1707}
1708
1709bool
1710Cache::writebackVisitor(CacheBlk &blk)
1711{
1712 if (blk.isDirty()) {
1713 assert(blk.isValid());
1714
1715 Request request(tags->regenerateBlkAddr(blk.tag, blk.set),
1716 blkSize, 0, Request::funcMasterId);
1717 request.taskId(blk.task_id);
1718 if (blk.isSecure()) {
1719 request.setFlags(Request::SECURE);
1720 }
1721
1722 Packet packet(&request, MemCmd::WriteReq);
1723 packet.dataStatic(blk.data);
1724
1725 memSidePort->sendFunctional(&packet);
1726
1727 blk.status &= ~BlkDirty;
1728 }
1729
1730 return true;
1731}
1732
1733bool
1734Cache::invalidateVisitor(CacheBlk &blk)
1735{
1736
1737 if (blk.isDirty())
1738 warn_once("Invalidating dirty cache lines. Expect things to break.\n");
1739
1740 if (blk.isValid()) {
1741 assert(!blk.isDirty());
1742 invalidateBlock(&blk);
1743 }
1744
1745 return true;
1746}
1747
1748CacheBlk*
1749Cache::allocateBlock(Addr addr, bool is_secure, PacketList &writebacks)
1750{
1751 CacheBlk *blk = tags->findVictim(addr);
1752
1753 // It is valid to return nullptr if there is no victim
1754 if (!blk)
1755 return nullptr;
1756
1757 if (blk->isValid()) {
1758 Addr repl_addr = tags->regenerateBlkAddr(blk->tag, blk->set);
1759 MSHR *repl_mshr = mshrQueue.findMatch(repl_addr, blk->isSecure());
1760 if (repl_mshr) {
1761 // must be an outstanding upgrade request
1762 // on a block we're about to replace...
1763 assert(!blk->isWritable() || blk->isDirty());
1764 assert(repl_mshr->needsWritable());
1765 // too hard to replace block with transient state
1766 // allocation failed, block not inserted
1767 return nullptr;
1768 } else {
1769 DPRINTF(Cache, "replacement: replacing %#llx (%s) with %#llx "
1770 "(%s): %s\n", repl_addr, blk->isSecure() ? "s" : "ns",
1771 addr, is_secure ? "s" : "ns",
1772 blk->isDirty() ? "writeback" : "clean");
1773
1774 if (blk->wasPrefetched()) {
1775 unusedPrefetches++;
1776 }
1777 // Will send up Writeback/CleanEvict snoops via isCachedAbove
1778 // when pushing this writeback list into the write buffer.
1779 if (blk->isDirty() || writebackClean) {
1780 // Save writeback packet for handling by caller
1781 writebacks.push_back(writebackBlk(blk));
1782 } else {
1783 writebacks.push_back(cleanEvictBlk(blk));
1784 }
1785 }
1786 }
1787
1788 return blk;
1789}
1790
1791void
1792Cache::invalidateBlock(CacheBlk *blk)
1793{
1794 if (blk != tempBlock)
1795 tags->invalidate(blk);
1796 blk->invalidate();
1797}
1798
1799// Note that the reason we return a list of writebacks rather than
1800// inserting them directly in the write buffer is that this function
1801// is called by both atomic and timing-mode accesses, and in atomic
1802// mode we don't mess with the write buffer (we just perform the
1803// writebacks atomically once the original request is complete).
1804CacheBlk*
1805Cache::handleFill(PacketPtr pkt, CacheBlk *blk, PacketList &writebacks,
1806 bool allocate)
1807{
1808 assert(pkt->isResponse() || pkt->cmd == MemCmd::WriteLineReq);
1809 Addr addr = pkt->getAddr();
1810 bool is_secure = pkt->isSecure();
1811#if TRACING_ON
1812 CacheBlk::State old_state = blk ? blk->status : 0;
1813#endif
1814
1815 // When handling a fill, we should have no writes to this line.
1816 assert(addr == pkt->getBlockAddr(blkSize));
1817 assert(!writeBuffer.findMatch(addr, is_secure));
1818
1819 if (blk == nullptr) {
1820 // better have read new data...
1821 assert(pkt->hasData());
1822
1823 // only read responses and write-line requests have data;
1824 // note that we don't write the data here for write-line - that
1825 // happens in the subsequent call to satisfyRequest
1826 assert(pkt->isRead() || pkt->cmd == MemCmd::WriteLineReq);
1827
1828 // need to do a replacement if allocating, otherwise we stick
1829 // with the temporary storage
1830 blk = allocate ? allocateBlock(addr, is_secure, writebacks) : nullptr;
1831
1832 if (blk == nullptr) {
1833 // No replaceable block or a mostly exclusive
1834 // cache... just use temporary storage to complete the
1835 // current request and then get rid of it
1836 assert(!tempBlock->isValid());
1837 blk = tempBlock;
1838 tempBlock->set = tags->extractSet(addr);
1839 tempBlock->tag = tags->extractTag(addr);
1840 // @todo: set security state as well...
1841 DPRINTF(Cache, "using temp block for %#llx (%s)\n", addr,
1842 is_secure ? "s" : "ns");
1843 } else {
1844 tags->insertBlock(pkt, blk);
1845 }
1846
1847 // we should never be overwriting a valid block
1848 assert(!blk->isValid());
1849 } else {
1850 // existing block... probably an upgrade
1851 assert(blk->tag == tags->extractTag(addr));
1852 // either we're getting new data or the block should already be valid
1853 assert(pkt->hasData() || blk->isValid());
1854 // don't clear block status... if block is already dirty we
1855 // don't want to lose that
1856 }
1857
1858 if (is_secure)
1859 blk->status |= BlkSecure;
1860 blk->status |= BlkValid | BlkReadable;
1861
1862 // sanity check for whole-line writes, which should always be
1863 // marked as writable as part of the fill, and then later marked
1864 // dirty as part of satisfyRequest
1865 if (pkt->cmd == MemCmd::WriteLineReq) {
1866 assert(!pkt->hasSharers());
1867 }
1868
1869 // here we deal with setting the appropriate state of the line,
1870 // and we start by looking at the hasSharers flag, and ignore the
1871 // cacheResponding flag (normally signalling dirty data) if the
1872 // packet has sharers, thus the line is never allocated as Owned
1873 // (dirty but not writable), and always ends up being either
1874 // Shared, Exclusive or Modified, see Packet::setCacheResponding
1875 // for more details
1876 if (!pkt->hasSharers()) {
1877 // we could get a writable line from memory (rather than a
1878 // cache) even in a read-only cache, note that we set this bit
1879 // even for a read-only cache, possibly revisit this decision
1880 blk->status |= BlkWritable;
1881
1882 // check if we got this via cache-to-cache transfer (i.e., from a
1883 // cache that had the block in Modified or Owned state)
1884 if (pkt->cacheResponding()) {
1885 // we got the block in Modified state, and invalidated the
1886 // owners copy
1887 blk->status |= BlkDirty;
1888
1889 chatty_assert(!isReadOnly, "Should never see dirty snoop response "
1890 "in read-only cache %s\n", name());
1891 }
1892 }
1893
1894 DPRINTF(Cache, "Block addr %#llx (%s) moving from state %x to %s\n",
1895 addr, is_secure ? "s" : "ns", old_state, blk->print());
1896
1897 // if we got new data, copy it in (checking for a read response
1898 // and a response that has data is the same in the end)
1899 if (pkt->isRead()) {
1900 // sanity checks
1901 assert(pkt->hasData());
1902 assert(pkt->getSize() == blkSize);
1903
1904 std::memcpy(blk->data, pkt->getConstPtr<uint8_t>(), blkSize);
1905 }
1906 // We pay for fillLatency here.
1907 blk->whenReady = clockEdge() + fillLatency * clockPeriod() +
1908 pkt->payloadDelay;
1909
1910 return blk;
1911}
1912
1913
1914/////////////////////////////////////////////////////
1915//
1916// Snoop path: requests coming in from the memory side
1917//
1918/////////////////////////////////////////////////////
1919
1920void
1921Cache::doTimingSupplyResponse(PacketPtr req_pkt, const uint8_t *blk_data,
1922 bool already_copied, bool pending_inval)
1923{
1924 // sanity check
1925 assert(req_pkt->isRequest());
1926 assert(req_pkt->needsResponse());
1927
1928 DPRINTF(Cache, "%s: for %s\n", __func__, req_pkt->print());
1929 // timing-mode snoop responses require a new packet, unless we
1930 // already made a copy...
1931 PacketPtr pkt = req_pkt;
1932 if (!already_copied)
1933 // do not clear flags, and allocate space for data if the
1934 // packet needs it (the only packets that carry data are read
1935 // responses)
1936 pkt = new Packet(req_pkt, false, req_pkt->isRead());
1937
1938 assert(req_pkt->req->isUncacheable() || req_pkt->isInvalidate() ||
1939 pkt->hasSharers());
1940 pkt->makeTimingResponse();
1941 if (pkt->isRead()) {
1942 pkt->setDataFromBlock(blk_data, blkSize);
1943 }
1944 if (pkt->cmd == MemCmd::ReadResp && pending_inval) {
1945 // Assume we defer a response to a read from a far-away cache
1946 // A, then later defer a ReadExcl from a cache B on the same
1947 // bus as us. We'll assert cacheResponding in both cases, but
1948 // in the latter case cacheResponding will keep the
1949 // invalidation from reaching cache A. This special response
1950 // tells cache A that it gets the block to satisfy its read,
1951 // but must immediately invalidate it.
1952 pkt->cmd = MemCmd::ReadRespWithInvalidate;
1953 }
1954 // Here we consider forward_time, paying for just forward latency and
1955 // also charging the delay provided by the xbar.
1956 // forward_time is used as send_time in next allocateWriteBuffer().
1957 Tick forward_time = clockEdge(forwardLatency) + pkt->headerDelay;
1958 // Here we reset the timing of the packet.
1959 pkt->headerDelay = pkt->payloadDelay = 0;
1960 DPRINTF(CacheVerbose, "%s: created response: %s tick: %lu\n", __func__,
1961 pkt->print(), forward_time);
1962 memSidePort->schedTimingSnoopResp(pkt, forward_time, true);
1963}
1964
1965uint32_t
1966Cache::handleSnoop(PacketPtr pkt, CacheBlk *blk, bool is_timing,
1967 bool is_deferred, bool pending_inval)
1968{
1969 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
1970 // deferred snoops can only happen in timing mode
1971 assert(!(is_deferred && !is_timing));
1972 // pending_inval only makes sense on deferred snoops
1973 assert(!(pending_inval && !is_deferred));
1974 assert(pkt->isRequest());
1975
1976 // the packet may get modified if we or a forwarded snooper
1977 // responds in atomic mode, so remember a few things about the
1978 // original packet up front
1979 bool invalidate = pkt->isInvalidate();
1980 bool M5_VAR_USED needs_writable = pkt->needsWritable();
1981
1982 // at the moment we could get an uncacheable write which does not
1983 // have the invalidate flag, and we need a suitable way of dealing
1984 // with this case
1985 panic_if(invalidate && pkt->req->isUncacheable(),
1986 "%s got an invalidating uncacheable snoop request %s",
1987 name(), pkt->print());
1988
1989 uint32_t snoop_delay = 0;
1990
1991 if (forwardSnoops) {
1992 // first propagate snoop upward to see if anyone above us wants to
1993 // handle it. save & restore packet src since it will get
1994 // rewritten to be relative to cpu-side bus (if any)
1995 bool alreadyResponded = pkt->cacheResponding();
1996 if (is_timing) {
1997 // copy the packet so that we can clear any flags before
1998 // forwarding it upwards, we also allocate data (passing
1999 // the pointer along in case of static data), in case
2000 // there is a snoop hit in upper levels
2001 Packet snoopPkt(pkt, true, true);
2002 snoopPkt.setExpressSnoop();
2003 // the snoop packet does not need to wait any additional
2004 // time
2005 snoopPkt.headerDelay = snoopPkt.payloadDelay = 0;
2006 cpuSidePort->sendTimingSnoopReq(&snoopPkt);
2007
2008 // add the header delay (including crossbar and snoop
2009 // delays) of the upward snoop to the snoop delay for this
2010 // cache
2011 snoop_delay += snoopPkt.headerDelay;
2012
2013 if (snoopPkt.cacheResponding()) {
2014 // cache-to-cache response from some upper cache
2015 assert(!alreadyResponded);
2016 pkt->setCacheResponding();
2017 }
2018 // upstream cache has the block, or has an outstanding
2019 // MSHR, pass the flag on
2020 if (snoopPkt.hasSharers()) {
2021 pkt->setHasSharers();
2022 }
2023 // If this request is a prefetch or clean evict and an upper level
2024 // signals block present, make sure to propagate the block
2025 // presence to the requester.
2026 if (snoopPkt.isBlockCached()) {
2027 pkt->setBlockCached();
2028 }
2029 } else {
2030 cpuSidePort->sendAtomicSnoop(pkt);
2031 if (!alreadyResponded && pkt->cacheResponding()) {
2032 // cache-to-cache response from some upper cache:
2033 // forward response to original requester
2034 assert(pkt->isResponse());
2035 }
2036 }
2037 }
2038
2039 if (!blk || !blk->isValid()) {
2040 DPRINTF(CacheVerbose, "%s: snoop miss for %s\n", __func__,
2041 pkt->print());
2042 if (is_deferred) {
2043 // we no longer have the block, and will not respond, but a
2044 // packet was allocated in MSHR::handleSnoop and we have
2045 // to delete it
2046 assert(pkt->needsResponse());
2047
2048 // we have passed the block to a cache upstream, that
2049 // cache should be responding
2050 assert(pkt->cacheResponding());
2051
2052 delete pkt;
2053 }
2054 return snoop_delay;
2055 } else {
2056 DPRINTF(Cache, "%s: snoop hit for %s, old state is %s\n", __func__,
2057 pkt->print(), blk->print());
2058 }
2059
2060 chatty_assert(!(isReadOnly && blk->isDirty()),
2061 "Should never have a dirty block in a read-only cache %s\n",
2062 name());
2063
2064 // We may end up modifying both the block state and the packet (if
2065 // we respond in atomic mode), so just figure out what to do now
2066 // and then do it later. We respond to all snoops that need
2067 // responses provided we have the block in dirty state. The
2068 // invalidation itself is taken care of below.
2069 bool respond = blk->isDirty() && pkt->needsResponse();
2070 bool have_writable = blk->isWritable();
2071
2072 // Invalidate any prefetch's from below that would strip write permissions
2073 // MemCmd::HardPFReq is only observed by upstream caches. After missing
2074 // above and in it's own cache, a new MemCmd::ReadReq is created that
2075 // downstream caches observe.
2076 if (pkt->mustCheckAbove()) {
2077 DPRINTF(Cache, "Found addr %#llx in upper level cache for snoop %s "
2078 "from lower cache\n", pkt->getAddr(), pkt->print());
2079 pkt->setBlockCached();
2080 return snoop_delay;
2081 }
2082
2083 if (pkt->isRead() && !invalidate) {
2084 // reading without requiring the line in a writable state
2085 assert(!needs_writable);
2086 pkt->setHasSharers();
2087
2088 // if the requesting packet is uncacheable, retain the line in
2089 // the current state, otherwhise unset the writable flag,
2090 // which means we go from Modified to Owned (and will respond
2091 // below), remain in Owned (and will respond below), from
2092 // Exclusive to Shared, or remain in Shared
2093 if (!pkt->req->isUncacheable())
2094 blk->status &= ~BlkWritable;
2095 }
2096
2097 if (respond) {
2098 // prevent anyone else from responding, cache as well as
2099 // memory, and also prevent any memory from even seeing the
2100 // request
2101 pkt->setCacheResponding();
2102 if (have_writable) {
2103 // inform the cache hierarchy that this cache had the line
2104 // in the Modified state so that we avoid unnecessary
2105 // invalidations (see Packet::setResponderHadWritable)
2106 pkt->setResponderHadWritable();
2107
2108 // in the case of an uncacheable request there is no point
2109 // in setting the responderHadWritable flag, but since the
2110 // recipient does not care there is no harm in doing so
2111 } else {
2112 // if the packet has needsWritable set we invalidate our
2113 // copy below and all other copies will be invalidates
2114 // through express snoops, and if needsWritable is not set
2115 // we already called setHasSharers above
2116 }
2117
2118 // if we are returning a writable and dirty (Modified) line,
2119 // we should be invalidating the line
2120 panic_if(!invalidate && !pkt->hasSharers(),
2121 "%s is passing a Modified line through %s, "
2122 "but keeping the block", name(), pkt->print());
2123
2124 if (is_timing) {
2125 doTimingSupplyResponse(pkt, blk->data, is_deferred, pending_inval);
2126 } else {
2127 pkt->makeAtomicResponse();
2128 // packets such as upgrades do not actually have any data
2129 // payload
2130 if (pkt->hasData())
2131 pkt->setDataFromBlock(blk->data, blkSize);
2132 }
2133 }
2134
2135 if (!respond && is_deferred) {
2136 assert(pkt->needsResponse());
2137
2138 // if we copied the deferred packet with the intention to
2139 // respond, but are not responding, then a cache above us must
2140 // be, and we can use this as the indication of whether this
2141 // is a packet where we created a copy of the request or not
2142 if (!pkt->cacheResponding()) {
2143 delete pkt->req;
2144 }
2145
2146 delete pkt;
2147 }
2148
2149 // Do this last in case it deallocates block data or something
2150 // like that
2151 if (invalidate) {
2152 invalidateBlock(blk);
2153 }
2154
2155 DPRINTF(Cache, "new state is %s\n", blk->print());
2156
2157 return snoop_delay;
2158}
2159
2160
2161void
2162Cache::recvTimingSnoopReq(PacketPtr pkt)
2163{
2164 DPRINTF(CacheVerbose, "%s: for %s\n", __func__, pkt->print());
2165
2166 // Snoops shouldn't happen when bypassing caches
2167 assert(!system->bypassCaches());
2168
2169 // no need to snoop requests that are not in range
2170 if (!inRange(pkt->getAddr())) {
2171 return;
2172 }
2173
2174 bool is_secure = pkt->isSecure();
2175 CacheBlk *blk = tags->findBlock(pkt->getAddr(), is_secure);
2176
2177 Addr blk_addr = pkt->getBlockAddr(blkSize);
2178 MSHR *mshr = mshrQueue.findMatch(blk_addr, is_secure);
2179
2180 // Update the latency cost of the snoop so that the crossbar can
2181 // account for it. Do not overwrite what other neighbouring caches
2182 // have already done, rather take the maximum. The update is
2183 // tentative, for cases where we return before an upward snoop
2184 // happens below.
2185 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay,
2186 lookupLatency * clockPeriod());
2187
2188 // Inform request(Prefetch, CleanEvict or Writeback) from below of
2189 // MSHR hit, set setBlockCached.
2190 if (mshr && pkt->mustCheckAbove()) {
2191 DPRINTF(Cache, "Setting block cached for %s from lower cache on "
2192 "mshr hit\n", pkt->print());
2193 pkt->setBlockCached();
2194 return;
2195 }
2196
2197 // Let the MSHR itself track the snoop and decide whether we want
2198 // to go ahead and do the regular cache snoop
2199 if (mshr && mshr->handleSnoop(pkt, order++)) {
2200 DPRINTF(Cache, "Deferring snoop on in-service MSHR to blk %#llx (%s)."
2201 "mshrs: %s\n", blk_addr, is_secure ? "s" : "ns",
2202 mshr->print());
2203
2204 if (mshr->getNumTargets() > numTarget)
2205 warn("allocating bonus target for snoop"); //handle later
2206 return;
2207 }
2208
2209 //We also need to check the writeback buffers and handle those
2210 WriteQueueEntry *wb_entry = writeBuffer.findMatch(blk_addr, is_secure);
2211 if (wb_entry) {
2212 DPRINTF(Cache, "Snoop hit in writeback to addr %#llx (%s)\n",
2213 pkt->getAddr(), is_secure ? "s" : "ns");
2214 // Expect to see only Writebacks and/or CleanEvicts here, both of
2215 // which should not be generated for uncacheable data.
2216 assert(!wb_entry->isUncacheable());
2217 // There should only be a single request responsible for generating
2218 // Writebacks/CleanEvicts.
2219 assert(wb_entry->getNumTargets() == 1);
2220 PacketPtr wb_pkt = wb_entry->getTarget()->pkt;
2221 assert(wb_pkt->isEviction() || wb_pkt->cmd == MemCmd::WriteClean);
2222
2223 if (pkt->isEviction()) {
2224 // if the block is found in the write queue, set the BLOCK_CACHED
2225 // flag for Writeback/CleanEvict snoop. On return the snoop will
2226 // propagate the BLOCK_CACHED flag in Writeback packets and prevent
2227 // any CleanEvicts from travelling down the memory hierarchy.
2228 pkt->setBlockCached();
2229 DPRINTF(Cache, "%s: Squashing %s from lower cache on writequeue "
2230 "hit\n", __func__, pkt->print());
2231 return;
2232 }
2233
2234 // conceptually writebacks are no different to other blocks in
2235 // this cache, so the behaviour is modelled after handleSnoop,
2236 // the difference being that instead of querying the block
2237 // state to determine if it is dirty and writable, we use the
2238 // command and fields of the writeback packet
2239 bool respond = wb_pkt->cmd == MemCmd::WritebackDirty &&
2240 pkt->needsResponse();
2241 bool have_writable = !wb_pkt->hasSharers();
2242 bool invalidate = pkt->isInvalidate();
2243
2244 if (!pkt->req->isUncacheable() && pkt->isRead() && !invalidate) {
2245 assert(!pkt->needsWritable());
2246 pkt->setHasSharers();
2247 wb_pkt->setHasSharers();
2248 }
2249
2250 if (respond) {
2251 pkt->setCacheResponding();
2252
2253 if (have_writable) {
2254 pkt->setResponderHadWritable();
2255 }
2256
2257 doTimingSupplyResponse(pkt, wb_pkt->getConstPtr<uint8_t>(),
2258 false, false);
2259 }
2260
2261 if (invalidate) {
2262 // Invalidation trumps our writeback... discard here
2263 // Note: markInService will remove entry from writeback buffer.
2264 markInService(wb_entry);
2265 delete wb_pkt;
2266 }
2267 }
2268
2269 // If this was a shared writeback, there may still be
2270 // other shared copies above that require invalidation.
2271 // We could be more selective and return here if the
2272 // request is non-exclusive or if the writeback is
2273 // exclusive.
2274 uint32_t snoop_delay = handleSnoop(pkt, blk, true, false, false);
2275
2276 // Override what we did when we first saw the snoop, as we now
2277 // also have the cost of the upwards snoops to account for
2278 pkt->snoopDelay = std::max<uint32_t>(pkt->snoopDelay, snoop_delay +
2279 lookupLatency * clockPeriod());
2280}
2281
2282bool
2283Cache::CpuSidePort::recvTimingSnoopResp(PacketPtr pkt)
2284{
2285 // Express snoop responses from master to slave, e.g., from L1 to L2
2286 cache->recvTimingSnoopResp(pkt);
2287 return true;
2288}
2289
2290Tick
2291Cache::recvAtomicSnoop(PacketPtr pkt)
2292{
2293 // Snoops shouldn't happen when bypassing caches
2294 assert(!system->bypassCaches());
2295
2296 // no need to snoop requests that are not in range.
2297 if (!inRange(pkt->getAddr())) {
2298 return 0;
2299 }
2300
2301 CacheBlk *blk = tags->findBlock(pkt->getAddr(), pkt->isSecure());
2302 uint32_t snoop_delay = handleSnoop(pkt, blk, false, false, false);
2303 return snoop_delay + lookupLatency * clockPeriod();
2304}
2305
2306
2307QueueEntry*
2308Cache::getNextQueueEntry()
2309{
2310 // Check both MSHR queue and write buffer for potential requests,
2311 // note that null does not mean there is no request, it could
2312 // simply be that it is not ready
2313 MSHR *miss_mshr = mshrQueue.getNext();
2314 WriteQueueEntry *wq_entry = writeBuffer.getNext();
2315
2316 // If we got a write buffer request ready, first priority is a
2317 // full write buffer, otherwise we favour the miss requests
2318 if (wq_entry && (writeBuffer.isFull() || !miss_mshr)) {
2319 // need to search MSHR queue for conflicting earlier miss.
2320 MSHR *conflict_mshr =
2321 mshrQueue.findPending(wq_entry->blkAddr,
2322 wq_entry->isSecure);
2323
2324 if (conflict_mshr && conflict_mshr->order < wq_entry->order) {
2325 // Service misses in order until conflict is cleared.
2326 return conflict_mshr;
2327
2328 // @todo Note that we ignore the ready time of the conflict here
2329 }
2330
2331 // No conflicts; issue write
2332 return wq_entry;
2333 } else if (miss_mshr) {
2334 // need to check for conflicting earlier writeback
2335 WriteQueueEntry *conflict_mshr =
2336 writeBuffer.findPending(miss_mshr->blkAddr,
2337 miss_mshr->isSecure);
2338 if (conflict_mshr) {
2339 // not sure why we don't check order here... it was in the
2340 // original code but commented out.
2341
2342 // The only way this happens is if we are
2343 // doing a write and we didn't have permissions
2344 // then subsequently saw a writeback (owned got evicted)
2345 // We need to make sure to perform the writeback first
2346 // To preserve the dirty data, then we can issue the write
2347
2348 // should we return wq_entry here instead? I.e. do we
2349 // have to flush writes in order? I don't think so... not
2350 // for Alpha anyway. Maybe for x86?
2351 return conflict_mshr;
2352
2353 // @todo Note that we ignore the ready time of the conflict here
2354 }
2355
2356 // No conflicts; issue read
2357 return miss_mshr;
2358 }
2359
2360 // fall through... no pending requests. Try a prefetch.
2361 assert(!miss_mshr && !wq_entry);
2362 if (prefetcher && mshrQueue.canPrefetch()) {
2363 // If we have a miss queue slot, we can try a prefetch
2364 PacketPtr pkt = prefetcher->getPacket();
2365 if (pkt) {
2366 Addr pf_addr = pkt->getBlockAddr(blkSize);
2367 if (!tags->findBlock(pf_addr, pkt->isSecure()) &&
2368 !mshrQueue.findMatch(pf_addr, pkt->isSecure()) &&
2369 !writeBuffer.findMatch(pf_addr, pkt->isSecure())) {
2370 // Update statistic on number of prefetches issued
2371 // (hwpf_mshr_misses)
2372 assert(pkt->req->masterId() < system->maxMasters());
2373 mshr_misses[pkt->cmdToIndex()][pkt->req->masterId()]++;
2374
2375 // allocate an MSHR and return it, note
2376 // that we send the packet straight away, so do not
2377 // schedule the send
2378 return allocateMissBuffer(pkt, curTick(), false);
2379 } else {
2380 // free the request and packet
2381 delete pkt->req;
2382 delete pkt;
2383 }
2384 }
2385 }
2386
2387 return nullptr;
2388}
2389
2390bool
2391Cache::isCachedAbove(PacketPtr pkt, bool is_timing) const
2392{
2393 if (!forwardSnoops)
2394 return false;
2395 // Mirroring the flow of HardPFReqs, the cache sends CleanEvict and
2396 // Writeback snoops into upper level caches to check for copies of the
2397 // same block. Using the BLOCK_CACHED flag with the Writeback/CleanEvict
2398 // packet, the cache can inform the crossbar below of presence or absence
2399 // of the block.
2400 if (is_timing) {
2401 Packet snoop_pkt(pkt, true, false);
2402 snoop_pkt.setExpressSnoop();
2403 // Assert that packet is either Writeback or CleanEvict and not a
2404 // prefetch request because prefetch requests need an MSHR and may
2405 // generate a snoop response.
2406 assert(pkt->isEviction() || pkt->cmd == MemCmd::WriteClean);
2407 snoop_pkt.senderState = nullptr;
2408 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2409 // Writeback/CleanEvict snoops do not generate a snoop response.
2410 assert(!(snoop_pkt.cacheResponding()));
2411 return snoop_pkt.isBlockCached();
2412 } else {
2413 cpuSidePort->sendAtomicSnoop(pkt);
2414 return pkt->isBlockCached();
2415 }
2416}
2417
2418Tick
2419Cache::nextQueueReadyTime() const
2420{
2421 Tick nextReady = std::min(mshrQueue.nextReadyTime(),
2422 writeBuffer.nextReadyTime());
2423
2424 // Don't signal prefetch ready time if no MSHRs available
2425 // Will signal once enoguh MSHRs are deallocated
2426 if (prefetcher && mshrQueue.canPrefetch()) {
2427 nextReady = std::min(nextReady,
2428 prefetcher->nextPrefetchReadyTime());
2429 }
2430
2431 return nextReady;
2432}
2433
2434bool
2435Cache::sendMSHRQueuePacket(MSHR* mshr)
2436{
2437 assert(mshr);
2438
2439 // use request from 1st target
2440 PacketPtr tgt_pkt = mshr->getTarget()->pkt;
2441
2442 DPRINTF(Cache, "%s: MSHR %s\n", __func__, tgt_pkt->print());
2443
2444 CacheBlk *blk = tags->findBlock(mshr->blkAddr, mshr->isSecure);
2445
2446 if (tgt_pkt->cmd == MemCmd::HardPFReq && forwardSnoops) {
2447 // we should never have hardware prefetches to allocated
2448 // blocks
2449 assert(blk == nullptr);
2450
2451 // We need to check the caches above us to verify that
2452 // they don't have a copy of this block in the dirty state
2453 // at the moment. Without this check we could get a stale
2454 // copy from memory that might get used in place of the
2455 // dirty one.
2456 Packet snoop_pkt(tgt_pkt, true, false);
2457 snoop_pkt.setExpressSnoop();
2458 // We are sending this packet upwards, but if it hits we will
2459 // get a snoop response that we end up treating just like a
2460 // normal response, hence it needs the MSHR as its sender
2461 // state
2462 snoop_pkt.senderState = mshr;
2463 cpuSidePort->sendTimingSnoopReq(&snoop_pkt);
2464
2465 // Check to see if the prefetch was squashed by an upper cache (to
2466 // prevent us from grabbing the line) or if a Check to see if a
2467 // writeback arrived between the time the prefetch was placed in
2468 // the MSHRs and when it was selected to be sent or if the
2469 // prefetch was squashed by an upper cache.
2470
2471 // It is important to check cacheResponding before
2472 // prefetchSquashed. If another cache has committed to
2473 // responding, it will be sending a dirty response which will
2474 // arrive at the MSHR allocated for this request. Checking the
2475 // prefetchSquash first may result in the MSHR being
2476 // prematurely deallocated.
2477 if (snoop_pkt.cacheResponding()) {
2478 auto M5_VAR_USED r = outstandingSnoop.insert(snoop_pkt.req);
2479 assert(r.second);
2480
2481 // if we are getting a snoop response with no sharers it
2482 // will be allocated as Modified
2483 bool pending_modified_resp = !snoop_pkt.hasSharers();
2484 markInService(mshr, pending_modified_resp);
2485
2486 DPRINTF(Cache, "Upward snoop of prefetch for addr"
2487 " %#x (%s) hit\n",
2488 tgt_pkt->getAddr(), tgt_pkt->isSecure()? "s": "ns");
2489 return false;
2490 }
2491
2492 if (snoop_pkt.isBlockCached()) {
2493 DPRINTF(Cache, "Block present, prefetch squashed by cache. "
2494 "Deallocating mshr target %#x.\n",
2495 mshr->blkAddr);
2496
2497 // Deallocate the mshr target
2498 if (mshrQueue.forceDeallocateTarget(mshr)) {
2499 // Clear block if this deallocation resulted freed an
2500 // mshr when all had previously been utilized
2501 clearBlocked(Blocked_NoMSHRs);
2502 }
2503
2504 // given that no response is expected, delete Request and Packet
2505 delete tgt_pkt->req;
2506 delete tgt_pkt;
2507
2508 return false;
2509 }
2510 }
2511
2512 // either a prefetch that is not present upstream, or a normal
2513 // MSHR request, proceed to get the packet to send downstream
2514 PacketPtr pkt = createMissPacket(tgt_pkt, blk, mshr->needsWritable());
2515
2516 mshr->isForward = (pkt == nullptr);
2517
2518 if (mshr->isForward) {
2519 // not a cache block request, but a response is expected
2520 // make copy of current packet to forward, keep current
2521 // copy for response handling
2522 pkt = new Packet(tgt_pkt, false, true);
2523 assert(!pkt->isWrite());
2524 }
2525
2526 // play it safe and append (rather than set) the sender state,
2527 // as forwarded packets may already have existing state
2528 pkt->pushSenderState(mshr);
2529
2530 if (!memSidePort->sendTimingReq(pkt)) {
2531 // we are awaiting a retry, but we
2532 // delete the packet and will be creating a new packet
2533 // when we get the opportunity
2534 delete pkt;
2535
2536 // note that we have now masked any requestBus and
2537 // schedSendEvent (we will wait for a retry before
2538 // doing anything), and this is so even if we do not
2539 // care about this packet and might override it before
2540 // it gets retried
2541 return true;
2542 } else {
2543 // As part of the call to sendTimingReq the packet is
2544 // forwarded to all neighbouring caches (and any caches
2545 // above them) as a snoop. Thus at this point we know if
2546 // any of the neighbouring caches are responding, and if
2547 // so, we know it is dirty, and we can determine if it is
2548 // being passed as Modified, making our MSHR the ordering
2549 // point
2550 bool pending_modified_resp = !pkt->hasSharers() &&
2551 pkt->cacheResponding();
2552 markInService(mshr, pending_modified_resp);
2553 return false;
2554 }
2555}
2556
2557bool
2558Cache::sendWriteQueuePacket(WriteQueueEntry* wq_entry)
2559{
2560 assert(wq_entry);
2561
2562 // always a single target for write queue entries
2563 PacketPtr tgt_pkt = wq_entry->getTarget()->pkt;
2564
2565 DPRINTF(Cache, "%s: write %s\n", __func__, tgt_pkt->print());
2566
2567 // forward as is, both for evictions and uncacheable writes
2568 if (!memSidePort->sendTimingReq(tgt_pkt)) {
2569 // note that we have now masked any requestBus and
2570 // schedSendEvent (we will wait for a retry before
2571 // doing anything), and this is so even if we do not
2572 // care about this packet and might override it before
2573 // it gets retried
2574 return true;
2575 } else {
2576 markInService(wq_entry);
2577 return false;
2578 }
2579}
2580
2581void
2582Cache::serialize(CheckpointOut &cp) const
2583{
2584 bool dirty(isDirty());
2585
2586 if (dirty) {
2587 warn("*** The cache still contains dirty data. ***\n");
2588 warn(" Make sure to drain the system using the correct flags.\n");
2589 warn(" This checkpoint will not restore correctly and dirty data "
2590 " in the cache will be lost!\n");
2591 }
2592
2593 // Since we don't checkpoint the data in the cache, any dirty data
2594 // will be lost when restoring from a checkpoint of a system that
2595 // wasn't drained properly. Flag the checkpoint as invalid if the
2596 // cache contains dirty data.
2597 bool bad_checkpoint(dirty);
2598 SERIALIZE_SCALAR(bad_checkpoint);
2599}
2600
2601void
2602Cache::unserialize(CheckpointIn &cp)
2603{
2604 bool bad_checkpoint;
2605 UNSERIALIZE_SCALAR(bad_checkpoint);
2606 if (bad_checkpoint) {
2607 fatal("Restoring from checkpoints with dirty caches is not supported "
2608 "in the classic memory system. Please remove any caches or "
2609 " drain them properly before taking checkpoints.\n");
2610 }
2611}
2612
2613///////////////
2614//
2615// CpuSidePort
2616//
2617///////////////
2618
2619AddrRangeList
2620Cache::CpuSidePort::getAddrRanges() const
2621{
2622 return cache->getAddrRanges();
2623}
2624
2625bool
2626Cache::CpuSidePort::tryTiming(PacketPtr pkt)
2627{
2628 assert(!cache->system->bypassCaches());
2629
2630 // always let express snoop packets through if even if blocked
2631 if (pkt->isExpressSnoop()) {
2632 return true;
2633 } else if (isBlocked() || mustSendRetry) {
2634 // either already committed to send a retry, or blocked
2635 mustSendRetry = true;
2636 return false;
2637 }
2638 mustSendRetry = false;
2639 return true;
2640}
2641
2642bool
2643Cache::CpuSidePort::recvTimingReq(PacketPtr pkt)
2644{
2645 assert(!cache->system->bypassCaches());
2646
2647 // always let express snoop packets through if even if blocked
2648 if (pkt->isExpressSnoop()) {
2649 bool M5_VAR_USED bypass_success = cache->recvTimingReq(pkt);
2650 assert(bypass_success);
2651 return true;
2652 }
2653
2654 return tryTiming(pkt) && cache->recvTimingReq(pkt);
2655}
2656
2657Tick
2658Cache::CpuSidePort::recvAtomic(PacketPtr pkt)
2659{
2660 return cache->recvAtomic(pkt);
2661}
2662
2663void
2664Cache::CpuSidePort::recvFunctional(PacketPtr pkt)
2665{
2666 // functional request
2667 cache->functionalAccess(pkt, true);
2668}
2669
2670Cache::
2671CpuSidePort::CpuSidePort(const std::string &_name, Cache *_cache,
2672 const std::string &_label)
2673 : BaseCache::CacheSlavePort(_name, _cache, _label), cache(_cache)
2674{
2675}
2676
2677Cache*
2678CacheParams::create()
2679{
2680 assert(tags);
2681
2682 return new Cache(this);
2683}
2684///////////////
2685//
2686// MemSidePort
2687//
2688///////////////
2689
2690bool
2691Cache::MemSidePort::recvTimingResp(PacketPtr pkt)
2692{
2693 cache->recvTimingResp(pkt);
2694 return true;
2695}
2696
2697// Express snooping requests to memside port
2698void
2699Cache::MemSidePort::recvTimingSnoopReq(PacketPtr pkt)
2700{
2701 // handle snooping requests
2702 cache->recvTimingSnoopReq(pkt);
2703}
2704
2705Tick
2706Cache::MemSidePort::recvAtomicSnoop(PacketPtr pkt)
2707{
2708 return cache->recvAtomicSnoop(pkt);
2709}
2710
2711void
2712Cache::MemSidePort::recvFunctionalSnoop(PacketPtr pkt)
2713{
2714 // functional snoop (note that in contrast to atomic we don't have
2715 // a specific functionalSnoop method, as they have the same
2716 // behaviour regardless)
2717 cache->functionalAccess(pkt, false);
2718}
2719
2720void
2721Cache::CacheReqPacketQueue::sendDeferredPacket()
2722{
2723 // sanity check
2724 assert(!waitingOnRetry);
2725
2726 // there should never be any deferred request packets in the
2727 // queue, instead we resly on the cache to provide the packets
2728 // from the MSHR queue or write queue
2729 assert(deferredPacketReadyTime() == MaxTick);
2730
2731 // check for request packets (requests & writebacks)
2732 QueueEntry* entry = cache.getNextQueueEntry();
2733
2734 if (!entry) {
2735 // can happen if e.g. we attempt a writeback and fail, but
2736 // before the retry, the writeback is eliminated because
2737 // we snoop another cache's ReadEx.
2738 } else {
2739 // let our snoop responses go first if there are responses to
2740 // the same addresses
2741 if (checkConflictingSnoop(entry->blkAddr)) {
2742 return;
2743 }
2744 waitingOnRetry = entry->sendPacket(cache);
2745 }
2746
2747 // if we succeeded and are not waiting for a retry, schedule the
2748 // next send considering when the next queue is ready, note that
2749 // snoop responses have their own packet queue and thus schedule
2750 // their own events
2751 if (!waitingOnRetry) {
2752 schedSendEvent(cache.nextQueueReadyTime());
2753 }
2754}
2755
2756Cache::
2757MemSidePort::MemSidePort(const std::string &_name, Cache *_cache,
2758 const std::string &_label)
2759 : BaseCache::CacheMasterPort(_name, _cache, _reqQueue, _snoopRespQueue),
2760 _reqQueue(*_cache, *this, _snoopRespQueue, _label),
2761 _snoopRespQueue(*_cache, *this, _label), cache(_cache)
2762{
2763}
|