snoop_filter.cc (12429:beefb9f5f551) snoop_filter.cc (14005:6cad91d6136c)
1/*
1/*
2 * Copyright (c) 2013-2017 ARM Limited
2 * Copyright (c) 2013-2017,2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated

--- 33 unchanged lines hidden (view full) ---

44
45#include "mem/snoop_filter.hh"
46
47#include "base/logging.hh"
48#include "base/trace.hh"
49#include "debug/SnoopFilter.hh"
50#include "sim/system.hh"
51
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated

--- 33 unchanged lines hidden (view full) ---

44
45#include "mem/snoop_filter.hh"
46
47#include "base/logging.hh"
48#include "base/trace.hh"
49#include "debug/SnoopFilter.hh"
50#include "sim/system.hh"
51
52const int SnoopFilter::SNOOP_MASK_SIZE;
53
52void
53SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it)
54{
55 SnoopItem& sf_item = sf_it->second;
54void
55SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it)
56{
57 SnoopItem& sf_item = sf_it->second;
56 if (!(sf_item.requested | sf_item.holder)) {
58 if ((sf_item.requested | sf_item.holder).none()) {
57 cachedLocations.erase(sf_it);
58 DPRINTF(SnoopFilter, "%s: Removed SF entry.\n",
59 __func__);
60 }
61}
62
63std::pair<SnoopFilter::SnoopList, Cycles>
64SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)

--- 26 unchanged lines hidden (view full) ---

91
92 // Store unmodified value of snoop filter item in temp storage in
93 // case we need to revert because of a send retry in
94 // updateRequest.
95 retryItem = sf_item;
96
97 totRequests++;
98 if (is_hit) {
59 cachedLocations.erase(sf_it);
60 DPRINTF(SnoopFilter, "%s: Removed SF entry.\n",
61 __func__);
62 }
63}
64
65std::pair<SnoopFilter::SnoopList, Cycles>
66SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)

--- 26 unchanged lines hidden (view full) ---

93
94 // Store unmodified value of snoop filter item in temp storage in
95 // case we need to revert because of a send retry in
96 // updateRequest.
97 retryItem = sf_item;
98
99 totRequests++;
100 if (is_hit) {
99 // Single bit set -> value is a power of two
100 if (isPow2(interested))
101 if (interested.count() == 1)
101 hitSingleRequests++;
102 else
103 hitMultiRequests++;
104 }
105
106 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n",
107 __func__, sf_item.requested, sf_item.holder);
108
109 // If we are not allocating, we are done
110 if (!allocate)
111 return snoopSelected(maskToPortList(interested & ~req_port),
112 lookupLatency);
113
114 if (cpkt->needsResponse()) {
115 if (!cpkt->cacheResponding()) {
116 // Max one request per address per port
102 hitSingleRequests++;
103 else
104 hitMultiRequests++;
105 }
106
107 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n",
108 __func__, sf_item.requested, sf_item.holder);
109
110 // If we are not allocating, we are done
111 if (!allocate)
112 return snoopSelected(maskToPortList(interested & ~req_port),
113 lookupLatency);
114
115 if (cpkt->needsResponse()) {
116 if (!cpkt->cacheResponding()) {
117 // Max one request per address per port
117 panic_if(sf_item.requested & req_port, "double request :( " \
118 "SF value %x.%x\n", sf_item.requested, sf_item.holder);
118 panic_if((sf_item.requested & req_port).any(),
119 "double request :( SF value %x.%x\n",
120 sf_item.requested, sf_item.holder);
119
120 // Mark in-flight requests to distinguish later on
121 sf_item.requested |= req_port;
122 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
123 __func__, sf_item.requested, sf_item.holder);
124 } else {
125 // NOTE: The memInhibit might have been asserted by a cache closer
126 // to the CPU, already -> the response will not be seen by this
127 // filter -> we do not need to keep the in-flight request, but make
128 // sure that we know that that cluster has a copy
121
122 // Mark in-flight requests to distinguish later on
123 sf_item.requested |= req_port;
124 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
125 __func__, sf_item.requested, sf_item.holder);
126 } else {
127 // NOTE: The memInhibit might have been asserted by a cache closer
128 // to the CPU, already -> the response will not be seen by this
129 // filter -> we do not need to keep the in-flight request, but make
130 // sure that we know that that cluster has a copy
129 panic_if(!(sf_item.holder & req_port), "Need to hold the value!");
131 panic_if((sf_item.holder & req_port).none(),
132 "Need to hold the value!");
130 DPRINTF(SnoopFilter,
131 "%s: not marking request. SF value %x.%x\n",
132 __func__, sf_item.requested, sf_item.holder);
133 }
134 } else { // if (!cpkt->needsResponse())
135 assert(cpkt->isEviction());
136 // make sure that the sender actually had the line
133 DPRINTF(SnoopFilter,
134 "%s: not marking request. SF value %x.%x\n",
135 __func__, sf_item.requested, sf_item.holder);
136 }
137 } else { // if (!cpkt->needsResponse())
138 assert(cpkt->isEviction());
139 // make sure that the sender actually had the line
137 panic_if(!(sf_item.holder & req_port), "requester %x is not a " \
140 panic_if((sf_item.holder & req_port).none(), "requester %x is not a " \
138 "holder :( SF value %x.%x\n", req_port,
139 sf_item.requested, sf_item.holder);
140 // CleanEvicts and Writebacks -> the sender and all caches above
141 // it may not have the line anymore.
142 if (!cpkt->isBlockCached()) {
143 sf_item.holder &= ~req_port;
144 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
145 __func__, sf_item.requested, sf_item.holder);

--- 55 unchanged lines hidden (view full) ---

201 SnoopItem& sf_item = sf_it->second;
202
203 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
204 __func__, sf_item.requested, sf_item.holder);
205
206 SnoopMask interested = (sf_item.holder | sf_item.requested);
207
208 totSnoops++;
141 "holder :( SF value %x.%x\n", req_port,
142 sf_item.requested, sf_item.holder);
143 // CleanEvicts and Writebacks -> the sender and all caches above
144 // it may not have the line anymore.
145 if (!cpkt->isBlockCached()) {
146 sf_item.holder &= ~req_port;
147 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
148 __func__, sf_item.requested, sf_item.holder);

--- 55 unchanged lines hidden (view full) ---

204 SnoopItem& sf_item = sf_it->second;
205
206 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
207 __func__, sf_item.requested, sf_item.holder);
208
209 SnoopMask interested = (sf_item.holder | sf_item.requested);
210
211 totSnoops++;
209 // Single bit set -> value is a power of two
210 if (isPow2(interested))
212
213 if (interested.count() == 1)
211 hitSingleSnoops++;
212 else
213 hitMultiSnoops++;
214
215 // ReadEx and Writes require both invalidation and exlusivity, while reads
216 // require neither. Writebacks on the other hand require exclusivity but
217 // not the invalidation. Previously Writebacks did not generate upward
218 // snoops so this was never an issue. Now that Writebacks generate snoops
219 // we need a special case for Writebacks. Additionally cache maintenance
220 // operations can generate snoops as they clean and/or invalidate all
221 // caches down to the specified point of reference.
222 assert(cpkt->isWriteback() || cpkt->req->isUncacheable() ||
223 (cpkt->isInvalidate() == cpkt->needsWritable()) ||
224 cpkt->req->isCacheMaintenance());
214 hitSingleSnoops++;
215 else
216 hitMultiSnoops++;
217
218 // ReadEx and Writes require both invalidation and exlusivity, while reads
219 // require neither. Writebacks on the other hand require exclusivity but
220 // not the invalidation. Previously Writebacks did not generate upward
221 // snoops so this was never an issue. Now that Writebacks generate snoops
222 // we need a special case for Writebacks. Additionally cache maintenance
223 // operations can generate snoops as they clean and/or invalidate all
224 // caches down to the specified point of reference.
225 assert(cpkt->isWriteback() || cpkt->req->isUncacheable() ||
226 (cpkt->isInvalidate() == cpkt->needsWritable()) ||
227 cpkt->req->isCacheMaintenance());
225 if (cpkt->isInvalidate() && !sf_item.requested) {
228 if (cpkt->isInvalidate() && sf_item.requested.none()) {
226 // Early clear of the holder, if no other request is currently going on
227 // @todo: This should possibly be updated even though we do not filter
228 // upward snoops
229 sf_item.holder = 0;
230 }
231
232 eraseIfNullEntry(sf_it);
233 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n",

--- 27 unchanged lines hidden (view full) ---

261 SnoopMask rsp_mask = portToMask(rsp_port);
262 SnoopMask req_mask = portToMask(req_port);
263 SnoopItem& sf_item = cachedLocations[line_addr];
264
265 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
266 __func__, sf_item.requested, sf_item.holder);
267
268 // The source should have the line
229 // Early clear of the holder, if no other request is currently going on
230 // @todo: This should possibly be updated even though we do not filter
231 // upward snoops
232 sf_item.holder = 0;
233 }
234
235 eraseIfNullEntry(sf_it);
236 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n",

--- 27 unchanged lines hidden (view full) ---

264 SnoopMask rsp_mask = portToMask(rsp_port);
265 SnoopMask req_mask = portToMask(req_port);
266 SnoopItem& sf_item = cachedLocations[line_addr];
267
268 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
269 __func__, sf_item.requested, sf_item.holder);
270
271 // The source should have the line
269 panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\
270 "the line\n", sf_item.requested, sf_item.holder);
272 panic_if((sf_item.holder & rsp_mask).none(),
273 "SF value %x.%x does not have the line\n",
274 sf_item.requested, sf_item.holder);
271
272 // The destination should have had a request in
275
276 // The destination should have had a request in
273 panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\
277 panic_if((sf_item.requested & req_mask).none(), "SF value %x.%x missing "\
274 "the original request\n", sf_item.requested, sf_item.holder);
275
276 // If the snoop response has no sharers the line is passed in
277 // Modified state, and we know that there are no other copies, or
278 // they will all be invalidated imminently
279 if (!cpkt->hasSharers()) {
280 DPRINTF(SnoopFilter,
281 "%s: dropping %x because non-shared snoop "
282 "response SF val: %x.%x\n", __func__, rsp_mask,
283 sf_item.requested, sf_item.holder);
284 sf_item.holder = 0;
285 }
286 assert(!cpkt->isWriteback());
287 // @todo Deal with invalidating responses
288 sf_item.holder |= req_mask;
289 sf_item.requested &= ~req_mask;
278 "the original request\n", sf_item.requested, sf_item.holder);
279
280 // If the snoop response has no sharers the line is passed in
281 // Modified state, and we know that there are no other copies, or
282 // they will all be invalidated imminently
283 if (!cpkt->hasSharers()) {
284 DPRINTF(SnoopFilter,
285 "%s: dropping %x because non-shared snoop "
286 "response SF val: %x.%x\n", __func__, rsp_mask,
287 sf_item.requested, sf_item.holder);
288 sf_item.holder = 0;
289 }
290 assert(!cpkt->isWriteback());
291 // @todo Deal with invalidating responses
292 sf_item.holder |= req_mask;
293 sf_item.requested &= ~req_mask;
290 assert(sf_item.requested | sf_item.holder);
294 assert((sf_item.requested | sf_item.holder).any());
291 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
292 __func__, sf_item.requested, sf_item.holder);
293}
294
295void
296SnoopFilter::updateSnoopForward(const Packet* cpkt,
297 const SlavePort& rsp_port, const MasterPort& req_port)
298{

--- 55 unchanged lines hidden (view full) ---

354
355 SnoopMask slave_mask = portToMask(slave_port);
356 SnoopItem& sf_item = sf_it->second;
357
358 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
359 __func__, sf_item.requested, sf_item.holder);
360
361 // Make sure we have seen the actual request, too
295 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
296 __func__, sf_item.requested, sf_item.holder);
297}
298
299void
300SnoopFilter::updateSnoopForward(const Packet* cpkt,
301 const SlavePort& rsp_port, const MasterPort& req_port)
302{

--- 55 unchanged lines hidden (view full) ---

358
359 SnoopMask slave_mask = portToMask(slave_port);
360 SnoopItem& sf_item = sf_it->second;
361
362 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
363 __func__, sf_item.requested, sf_item.holder);
364
365 // Make sure we have seen the actual request, too
362 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\
363 "request bit\n", sf_item.requested, sf_item.holder);
366 panic_if((sf_item.requested & slave_mask).none(),
367 "SF value %x.%x missing request bit\n",
368 sf_item.requested, sf_item.holder);
364
365 sf_item.requested &= ~slave_mask;
366 // Update the residency of the cache line.
367
368 if (cpkt->req->isCacheMaintenance()) {
369 // A cache clean response does not carry any data so it
370 // shouldn't change the holders, unless it is invalidating.
371 if (cpkt->isInvalidate()) {
372 sf_item.holder &= ~slave_mask;
373 }
374 eraseIfNullEntry(sf_it);
375 } else {
376 // Any other response implies that a cache above will have the
377 // block.
378 sf_item.holder |= slave_mask;
369
370 sf_item.requested &= ~slave_mask;
371 // Update the residency of the cache line.
372
373 if (cpkt->req->isCacheMaintenance()) {
374 // A cache clean response does not carry any data so it
375 // shouldn't change the holders, unless it is invalidating.
376 if (cpkt->isInvalidate()) {
377 sf_item.holder &= ~slave_mask;
378 }
379 eraseIfNullEntry(sf_it);
380 } else {
381 // Any other response implies that a cache above will have the
382 // block.
383 sf_item.holder |= slave_mask;
379 assert(sf_item.holder | sf_item.requested);
384 assert((sf_item.holder | sf_item.requested).any());
380 }
381 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
382 __func__, sf_item.requested, sf_item.holder);
383}
384
385void
386SnoopFilter::regStats()
387{

--- 36 unchanged lines hidden --- 		385 }
386 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
387 __func__, sf_item.requested, sf_item.holder);
388}
389
390void
391SnoopFilter::regStats()
392{

--- 36 unchanged lines hidden ---