snoop_filter.cc (11128:b6532152a64a) snoop_filter.cc (11129:48c02e8b0bbb)
1/*
2 * Copyright (c) 2013-2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 34 unchanged lines hidden (view full) ---

43 */
44
45#include "base/misc.hh"
46#include "base/trace.hh"
47#include "debug/SnoopFilter.hh"
48#include "mem/snoop_filter.hh"
49#include "sim/system.hh"
50
1/*
2 * Copyright (c) 2013-2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 34 unchanged lines hidden (view full) ---

43 */
44
45#include "base/misc.hh"
46#include "base/trace.hh"
47#include "debug/SnoopFilter.hh"
48#include "mem/snoop_filter.hh"
49#include "sim/system.hh"
50
51void
52SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it)
53{
54 SnoopItem& sf_item = sf_it->second;
55 if (!(sf_item.requested | sf_item.holder)) {
56 cachedLocations.erase(sf_it);
57 DPRINTF(SnoopFilter, "%s: Removed SF entry.\n",
58 __func__);
59 }
60}
61
51std::pair<SnoopFilter::SnoopList, Cycles>
52SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)
53{
54 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
55 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
56
57 // Ultimately we should check if the packet came from an
58 // allocating source, not just if the port is snooping

--- 8 unchanged lines hidden (view full) ---

67 // portlist.
68 if (!is_hit && !allocate)
69 return snoopDown(lookupLatency);
70
71 // Create a new element through operator[] and modify in-place
72 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr];
73 SnoopMask interested = sf_item.holder | sf_item.requested;
74
62std::pair<SnoopFilter::SnoopList, Cycles>
63SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)
64{
65 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
66 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
67
68 // Ultimately we should check if the packet came from an
69 // allocating source, not just if the port is snooping

--- 8 unchanged lines hidden (view full) ---

78 // portlist.
79 if (!is_hit && !allocate)
80 return snoopDown(lookupLatency);
81
82 // Create a new element through operator[] and modify in-place
83 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr];
84 SnoopMask interested = sf_item.holder | sf_item.requested;
85
86 // Store unmodified value of snoop filter item in temp storage in
87 // case we need to revert because of a send retry in
88 // updateRequest.
89 retryItem = sf_item;
90
75 totRequests++;
76 if (is_hit) {
77 // Single bit set -> value is a power of two
78 if (isPow2(interested))
79 hitSingleRequests++;
80 else
81 hitMultiRequests++;
82 }
83
84 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n",
85 __func__, sf_item.requested, sf_item.holder);
86
91 totRequests++;
92 if (is_hit) {
93 // Single bit set -> value is a power of two
94 if (isPow2(interested))
95 hitSingleRequests++;
96 else
97 hitMultiRequests++;
98 }
99
100 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n",
101 __func__, sf_item.requested, sf_item.holder);
102
87 if (allocate && cpkt->needsResponse()) {
103 // If we are not allocating, we are done
104 if (!allocate)
105 return snoopSelected(maskToPortList(interested & ~req_port),
106 lookupLatency);
107
108 if (cpkt->needsResponse()) {
88 if (!cpkt->memInhibitAsserted()) {
89 // Max one request per address per port
109 if (!cpkt->memInhibitAsserted()) {
110 // Max one request per address per port
90 panic_if(sf_item.requested & req_port, "double request :( "\
111 panic_if(sf_item.requested & req_port, "double request :( " \
91 "SF value %x.%x\n", sf_item.requested, sf_item.holder);
92
93 // Mark in-flight requests to distinguish later on
94 sf_item.requested |= req_port;
112 "SF value %x.%x\n", sf_item.requested, sf_item.holder);
113
114 // Mark in-flight requests to distinguish later on
115 sf_item.requested |= req_port;
116 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
117 __func__, sf_item.requested, sf_item.holder);
95 } else {
96 // NOTE: The memInhibit might have been asserted by a cache closer
97 // to the CPU, already -> the response will not be seen by this
98 // filter -> we do not need to keep the in-flight request, but make
99 // sure that we know that that cluster has a copy
100 panic_if(!(sf_item.holder & req_port), "Need to hold the value!");
118 } else {
119 // NOTE: The memInhibit might have been asserted by a cache closer
120 // to the CPU, already -> the response will not be seen by this
121 // filter -> we do not need to keep the in-flight request, but make
122 // sure that we know that that cluster has a copy
123 panic_if(!(sf_item.holder & req_port), "Need to hold the value!");
101 DPRINTF(SnoopFilter, "%s: not marking request. SF value %x.%x\n",
124 DPRINTF(SnoopFilter,
125 "%s: not marking request. SF value %x.%x\n",
102 __func__, sf_item.requested, sf_item.holder);
103 }
126 __func__, sf_item.requested, sf_item.holder);
127 }
104 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
105 __func__, sf_item.requested, sf_item.holder);
128 } else { // if (!cpkt->needsResponse())
129 assert(cpkt->evictingBlock());
130 // make sure that the sender actually had the line
131 panic_if(!(sf_item.holder & req_port), "requester %x is not a " \
132 "holder :( SF value %x.%x\n", req_port,
133 sf_item.requested, sf_item.holder);
134 // CleanEvicts and Writebacks -> the sender and all caches above
135 // it may not have the line anymore.
136 if (!cpkt->isBlockCached()) {
137 sf_item.holder &= ~req_port;
138 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
139 __func__, sf_item.requested, sf_item.holder);
140 }
106 }
141 }
142
107 return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency);
108}
109
110void
111SnoopFilter::updateRequest(const Packet* cpkt, const SlavePort& slave_port,
112 bool will_retry)
113{
114 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
115 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
116
117 // Ultimately we should check if the packet came from an
118 // allocating source, not just if the port is snooping
119 bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping();
120 if (!allocate)
121 return;
122
123 Addr line_addr = cpkt->getBlockAddr(linesize);
143 return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency);
144}
145
146void
147SnoopFilter::updateRequest(const Packet* cpkt, const SlavePort& slave_port,
148 bool will_retry)
149{
150 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
151 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
152
153 // Ultimately we should check if the packet came from an
154 // allocating source, not just if the port is snooping
155 bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping();
156 if (!allocate)
157 return;
158
159 Addr line_addr = cpkt->getBlockAddr(linesize);
124 SnoopMask req_port = portToMask(slave_port);
125 SnoopItem& sf_item = cachedLocations[line_addr];
126
127 DPRINTF(SnoopFilter, "%s: old SF value %x.%x retry: %i\n",
128 __func__, sf_item.requested, sf_item.holder, will_retry);
129
160 auto sf_it = cachedLocations.find(line_addr);
161 assert(sf_it != cachedLocations.end());
130 if (will_retry) {
162 if (will_retry) {
131 // Unmark a request that will come again.
132 sf_item.requested &= ~req_port;
133 return;
134 }
163 // Undo any changes made in lookupRequest to the snoop filter
164 // entry if the request will come again. retryItem holds
165 // the previous value of the snoopfilter entry.
166 sf_it->second = retryItem;
135
167
136 // will_retry == false
137 if (!cpkt->needsResponse()) {
138 // Packets that will not evoke a response but still need updates of the
139 // snoop filter; WRITEBACKs for now only
140 if (cpkt->cmd == MemCmd::Writeback) {
141 // make sure that the sender actually had the line
142 panic_if(sf_item.requested & req_port, "double request :( "\
143 "SF value %x.%x\n", sf_item.requested, sf_item.holder);
144 panic_if(!(sf_item.holder & req_port), "requester %x is not a "\
145 "holder :( SF value %x.%x\n", req_port,
146 sf_item.requested, sf_item.holder);
147 // Writebacks -> the sender does not have the line anymore
148 sf_item.holder &= ~req_port;
149 } else {
150 // @todo Add CleanEvicts
151 assert(cpkt->cmd == MemCmd::CleanEvict);
152 }
153 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
154 __func__, sf_item.requested, sf_item.holder);
168 DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n",
169 __func__, retryItem.requested, retryItem.holder);
155 }
170 }
171
172 eraseIfNullEntry(sf_it);
156}
157
158std::pair<SnoopFilter::SnoopList, Cycles>
159SnoopFilter::lookupSnoop(const Packet* cpkt)
160{
161 DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n",
162 __func__, cpkt->getAddr(), cpkt->cmdString());
163
164 assert(cpkt->isRequest());
165
166 // Broadcast / filter upward snoops
167 const bool filter_upward = true; // @todo: Make configurable
168
169 if (!filter_upward)
170 return snoopAll(lookupLatency);
171
172 Addr line_addr = cpkt->getBlockAddr(linesize);
173 auto sf_it = cachedLocations.find(line_addr);
174 bool is_hit = (sf_it != cachedLocations.end());
173}
174
175std::pair<SnoopFilter::SnoopList, Cycles>
176SnoopFilter::lookupSnoop(const Packet* cpkt)
177{
178 DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n",
179 __func__, cpkt->getAddr(), cpkt->cmdString());
180
181 assert(cpkt->isRequest());
182
183 // Broadcast / filter upward snoops
184 const bool filter_upward = true; // @todo: Make configurable
185
186 if (!filter_upward)
187 return snoopAll(lookupLatency);
188
189 Addr line_addr = cpkt->getBlockAddr(linesize);
190 auto sf_it = cachedLocations.find(line_addr);
191 bool is_hit = (sf_it != cachedLocations.end());
175 // Create a new element through operator[] and modify in-place
176 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr];
177
192
193 // If the snoop filter has no entry and its an uncacheable
194 // request, do not create a new snoop filter entry, simply return
195 // a NULL portlist.
196 if (!is_hit && cpkt->req->isUncacheable())
197 return snoopDown(lookupLatency);
198
199 // If no hit in snoop filter create a new element and update iterator
200 if (!is_hit)
201 sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first;
202 SnoopItem& sf_item = sf_it->second;
203
178 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
179 __func__, sf_item.requested, sf_item.holder);
180
181 SnoopMask interested = (sf_item.holder | sf_item.requested);
182
183 totSnoops++;
184 if (is_hit) {
185 // Single bit set -> value is a power of two
186 if (isPow2(interested))
187 hitSingleSnoops++;
188 else
189 hitMultiSnoops++;
190 }
191 // ReadEx and Writes require both invalidation and exlusivity, while reads
192 // require neither. Writebacks on the other hand require exclusivity but
193 // not the invalidation. Previously Writebacks did not generate upward
194 // snoops so this was never an aissue. Now that Writebacks generate snoops
195 // we need to special case for Writebacks.
204 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
205 __func__, sf_item.requested, sf_item.holder);
206
207 SnoopMask interested = (sf_item.holder | sf_item.requested);
208
209 totSnoops++;
210 if (is_hit) {
211 // Single bit set -> value is a power of two
212 if (isPow2(interested))
213 hitSingleSnoops++;
214 else
215 hitMultiSnoops++;
216 }
217 // ReadEx and Writes require both invalidation and exlusivity, while reads
218 // require neither. Writebacks on the other hand require exclusivity but
219 // not the invalidation. Previously Writebacks did not generate upward
220 // snoops so this was never an aissue. Now that Writebacks generate snoops
221 // we need to special case for Writebacks.
196 assert(cpkt->cmd == MemCmd::Writeback ||
222 assert(cpkt->cmd == MemCmd::Writeback || cpkt->req->isUncacheable() ||
197 (cpkt->isInvalidate() == cpkt->needsExclusive()));
198 if (cpkt->isInvalidate() && !sf_item.requested) {
199 // Early clear of the holder, if no other request is currently going on
200 // @todo: This should possibly be updated even though we do not filter
201 // upward snoops
202 sf_item.holder = 0;
203 }
204
223 (cpkt->isInvalidate() == cpkt->needsExclusive()));
224 if (cpkt->isInvalidate() && !sf_item.requested) {
225 // Early clear of the holder, if no other request is currently going on
226 // @todo: This should possibly be updated even though we do not filter
227 // upward snoops
228 sf_item.holder = 0;
229 }
230
231 eraseIfNullEntry(sf_it);
205 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n",
206 __func__, sf_item.requested, sf_item.holder, interested);
207
208 return snoopSelected(maskToPortList(interested), lookupLatency);
209}
210
211void
212SnoopFilter::updateSnoopResponse(const Packet* cpkt,

--- 40 unchanged lines hidden (view full) ---

253 // The snoop filter does not see any ACKs from non-responding sharers
254 // that have been invalidated :( So below assert would be nice, but..
255 //assert(sf_item.holder == 0);
256 sf_item.holder = 0;
257 }
258 assert(cpkt->cmd != MemCmd::Writeback);
259 sf_item.holder |= req_mask;
260 sf_item.requested &= ~req_mask;
232 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n",
233 __func__, sf_item.requested, sf_item.holder, interested);
234
235 return snoopSelected(maskToPortList(interested), lookupLatency);
236}
237
238void
239SnoopFilter::updateSnoopResponse(const Packet* cpkt,

--- 40 unchanged lines hidden (view full) ---

280 // The snoop filter does not see any ACKs from non-responding sharers
281 // that have been invalidated :( So below assert would be nice, but..
282 //assert(sf_item.holder == 0);
283 sf_item.holder = 0;
284 }
285 assert(cpkt->cmd != MemCmd::Writeback);
286 sf_item.holder |= req_mask;
287 sf_item.requested &= ~req_mask;
288 assert(sf_item.requested | sf_item.holder);
261 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
262 __func__, sf_item.requested, sf_item.holder);
263}
264
265void
266SnoopFilter::updateSnoopForward(const Packet* cpkt,
267 const SlavePort& rsp_port, const MasterPort& req_port)
268{
269 DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
270 __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
271 cpkt->cmdString());
272
273 Addr line_addr = cpkt->getBlockAddr(linesize);
289 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
290 __func__, sf_item.requested, sf_item.holder);
291}
292
293void
294SnoopFilter::updateSnoopForward(const Packet* cpkt,
295 const SlavePort& rsp_port, const MasterPort& req_port)
296{
297 DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
298 __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
299 cpkt->cmdString());
300
301 Addr line_addr = cpkt->getBlockAddr(linesize);
274 SnoopItem& sf_item = cachedLocations[line_addr];
302 auto sf_it = cachedLocations.find(line_addr);
303 if (sf_it == cachedLocations.end())
304 sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first;
305 SnoopItem& sf_item = sf_it->second;
275 SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port);
276
277 assert(cpkt->isResponse());
278 assert(cpkt->memInhibitAsserted());
279
280 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
281 __func__, sf_item.requested, sf_item.holder);
282
283 // Remote (to this snoop filter) snoops update the filter already when they
284 // arrive from below, because we may not see any response.
285 if (cpkt->needsExclusive()) {
286 // If the request to this snoop response hit an in-flight transaction,
287 // the holder was not reset -> no assertion & do that here, now!
288 //assert(sf_item.holder == 0);
289 sf_item.holder = 0;
290 }
291 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
292 __func__, sf_item.requested, sf_item.holder);
306 SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port);
307
308 assert(cpkt->isResponse());
309 assert(cpkt->memInhibitAsserted());
310
311 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
312 __func__, sf_item.requested, sf_item.holder);
313
314 // Remote (to this snoop filter) snoops update the filter already when they
315 // arrive from below, because we may not see any response.
316 if (cpkt->needsExclusive()) {
317 // If the request to this snoop response hit an in-flight transaction,
318 // the holder was not reset -> no assertion & do that here, now!
319 //assert(sf_item.holder == 0);
320 sf_item.holder = 0;
321 }
322 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
323 __func__, sf_item.requested, sf_item.holder);
324 eraseIfNullEntry(sf_it);
293}
294
295void
296SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port)
297{
298 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
299 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
300

--- 11 unchanged lines hidden (view full) ---

312
313 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
314 __func__, sf_item.requested, sf_item.holder);
315
316 // Make sure we have seen the actual request, too
317 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\
318 "request bit\n", sf_item.requested, sf_item.holder);
319
325}
326
327void
328SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port)
329{
330 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
331 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
332

--- 11 unchanged lines hidden (view full) ---

344
345 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n",
346 __func__, sf_item.requested, sf_item.holder);
347
348 // Make sure we have seen the actual request, too
349 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\
350 "request bit\n", sf_item.requested, sf_item.holder);
351
320 // Update the residency of the cache line.
321 if (cpkt->needsExclusive() || !cpkt->sharedAsserted())
352 // Update the residency of the cache line. Here we assume that the
353 // line has been zapped in all caches that are not the responder.
354 if (cpkt->needsExclusive() || !cpkt->sharedAsserted())
322 sf_item.holder = 0;
323 sf_item.holder |= slave_mask;
324 sf_item.requested &= ~slave_mask;
355 sf_item.holder = 0;
356 sf_item.holder |= slave_mask;
357 sf_item.requested &= ~slave_mask;
358 assert(sf_item.holder | sf_item.requested);
325 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
326 __func__, sf_item.requested, sf_item.holder);
327}
328
329void
330SnoopFilter::regStats()
331{
332 totRequests

--- 33 unchanged lines hidden --- 		359 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n",
360 __func__, sf_item.requested, sf_item.holder);
361}
362
363void
364SnoopFilter::regStats()
365{
366 totRequests

--- 33 unchanged lines hidden ---