snoop_filter.cc (11128:b6532152a64a) | snoop_filter.cc (11129:48c02e8b0bbb) |
---|---|
1/* 2 * Copyright (c) 2013-2015 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 34 unchanged lines hidden (view full) --- 43 */ 44 45#include "base/misc.hh" 46#include "base/trace.hh" 47#include "debug/SnoopFilter.hh" 48#include "mem/snoop_filter.hh" 49#include "sim/system.hh" 50 | 1/* 2 * Copyright (c) 2013-2015 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 34 unchanged lines hidden (view full) --- 43 */ 44 45#include "base/misc.hh" 46#include "base/trace.hh" 47#include "debug/SnoopFilter.hh" 48#include "mem/snoop_filter.hh" 49#include "sim/system.hh" 50 |
51void 52SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it) 53{ 54 SnoopItem& sf_item = sf_it->second; 55 if (!(sf_item.requested | sf_item.holder)) { 56 cachedLocations.erase(sf_it); 57 DPRINTF(SnoopFilter, "%s: Removed SF entry.\n", 58 __func__); 59 } 60} 61 |
|
51std::pair<SnoopFilter::SnoopList, Cycles> 52SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port) 53{ 54 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 55 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 56 57 // Ultimately we should check if the packet came from an 58 // allocating source, not just if the port is snooping --- 8 unchanged lines hidden (view full) --- 67 // portlist. 68 if (!is_hit && !allocate) 69 return snoopDown(lookupLatency); 70 71 // Create a new element through operator[] and modify in-place 72 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr]; 73 SnoopMask interested = sf_item.holder | sf_item.requested; 74 | 62std::pair<SnoopFilter::SnoopList, Cycles> 63SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port) 64{ 65 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 66 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 67 68 // Ultimately we should check if the packet came from an 69 // allocating source, not just if the port is snooping --- 8 unchanged lines hidden (view full) --- 78 // portlist. 79 if (!is_hit && !allocate) 80 return snoopDown(lookupLatency); 81 82 // Create a new element through operator[] and modify in-place 83 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr]; 84 SnoopMask interested = sf_item.holder | sf_item.requested; 85 |
86 // Store unmodified value of snoop filter item in temp storage in 87 // case we need to revert because of a send retry in 88 // updateRequest. 89 retryItem = sf_item; 90 |
|
75 totRequests++; 76 if (is_hit) { 77 // Single bit set -> value is a power of two 78 if (isPow2(interested)) 79 hitSingleRequests++; 80 else 81 hitMultiRequests++; 82 } 83 84 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n", 85 __func__, sf_item.requested, sf_item.holder); 86 | 91 totRequests++; 92 if (is_hit) { 93 // Single bit set -> value is a power of two 94 if (isPow2(interested)) 95 hitSingleRequests++; 96 else 97 hitMultiRequests++; 98 } 99 100 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n", 101 __func__, sf_item.requested, sf_item.holder); 102 |
87 if (allocate && cpkt->needsResponse()) { | 103 // If we are not allocating, we are done 104 if (!allocate) 105 return snoopSelected(maskToPortList(interested & ~req_port), 106 lookupLatency); 107 108 if (cpkt->needsResponse()) { |
88 if (!cpkt->memInhibitAsserted()) { 89 // Max one request per address per port | 109 if (!cpkt->memInhibitAsserted()) { 110 // Max one request per address per port |
90 panic_if(sf_item.requested & req_port, "double request :( "\ | 111 panic_if(sf_item.requested & req_port, "double request :( " \ |
91 "SF value %x.%x\n", sf_item.requested, sf_item.holder); 92 93 // Mark in-flight requests to distinguish later on 94 sf_item.requested |= req_port; | 112 "SF value %x.%x\n", sf_item.requested, sf_item.holder); 113 114 // Mark in-flight requests to distinguish later on 115 sf_item.requested |= req_port; |
116 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 117 __func__, sf_item.requested, sf_item.holder); |
|
95 } else { 96 // NOTE: The memInhibit might have been asserted by a cache closer 97 // to the CPU, already -> the response will not be seen by this 98 // filter -> we do not need to keep the in-flight request, but make 99 // sure that we know that that cluster has a copy 100 panic_if(!(sf_item.holder & req_port), "Need to hold the value!"); | 118 } else { 119 // NOTE: The memInhibit might have been asserted by a cache closer 120 // to the CPU, already -> the response will not be seen by this 121 // filter -> we do not need to keep the in-flight request, but make 122 // sure that we know that that cluster has a copy 123 panic_if(!(sf_item.holder & req_port), "Need to hold the value!"); |
101 DPRINTF(SnoopFilter, "%s: not marking request. SF value %x.%x\n", | 124 DPRINTF(SnoopFilter, 125 "%s: not marking request. SF value %x.%x\n", |
102 __func__, sf_item.requested, sf_item.holder); 103 } | 126 __func__, sf_item.requested, sf_item.holder); 127 } |
104 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 105 __func__, sf_item.requested, sf_item.holder); | 128 } else { // if (!cpkt->needsResponse()) 129 assert(cpkt->evictingBlock()); 130 // make sure that the sender actually had the line 131 panic_if(!(sf_item.holder & req_port), "requester %x is not a " \ 132 "holder :( SF value %x.%x\n", req_port, 133 sf_item.requested, sf_item.holder); 134 // CleanEvicts and Writebacks -> the sender and all caches above 135 // it may not have the line anymore. 136 if (!cpkt->isBlockCached()) { 137 sf_item.holder &= ~req_port; 138 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 139 __func__, sf_item.requested, sf_item.holder); 140 } |
106 } | 141 } |
142 |
|
107 return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency); 108} 109 110void 111SnoopFilter::updateRequest(const Packet* cpkt, const SlavePort& slave_port, 112 bool will_retry) 113{ 114 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 115 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 116 117 // Ultimately we should check if the packet came from an 118 // allocating source, not just if the port is snooping 119 bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping(); 120 if (!allocate) 121 return; 122 123 Addr line_addr = cpkt->getBlockAddr(linesize); | 143 return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency); 144} 145 146void 147SnoopFilter::updateRequest(const Packet* cpkt, const SlavePort& slave_port, 148 bool will_retry) 149{ 150 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 151 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 152 153 // Ultimately we should check if the packet came from an 154 // allocating source, not just if the port is snooping 155 bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping(); 156 if (!allocate) 157 return; 158 159 Addr line_addr = cpkt->getBlockAddr(linesize); |
124 SnoopMask req_port = portToMask(slave_port); 125 SnoopItem& sf_item = cachedLocations[line_addr]; 126 127 DPRINTF(SnoopFilter, "%s: old SF value %x.%x retry: %i\n", 128 __func__, sf_item.requested, sf_item.holder, will_retry); 129 | 160 auto sf_it = cachedLocations.find(line_addr); 161 assert(sf_it != cachedLocations.end()); |
130 if (will_retry) { | 162 if (will_retry) { |
131 // Unmark a request that will come again. 132 sf_item.requested &= ~req_port; 133 return; 134 } | 163 // Undo any changes made in lookupRequest to the snoop filter 164 // entry if the request will come again. retryItem holds 165 // the previous value of the snoopfilter entry. 166 sf_it->second = retryItem; |
135 | 167 |
136 // will_retry == false 137 if (!cpkt->needsResponse()) { 138 // Packets that will not evoke a response but still need updates of the 139 // snoop filter; WRITEBACKs for now only 140 if (cpkt->cmd == MemCmd::Writeback) { 141 // make sure that the sender actually had the line 142 panic_if(sf_item.requested & req_port, "double request :( "\ 143 "SF value %x.%x\n", sf_item.requested, sf_item.holder); 144 panic_if(!(sf_item.holder & req_port), "requester %x is not a "\ 145 "holder :( SF value %x.%x\n", req_port, 146 sf_item.requested, sf_item.holder); 147 // Writebacks -> the sender does not have the line anymore 148 sf_item.holder &= ~req_port; 149 } else { 150 // @todo Add CleanEvicts 151 assert(cpkt->cmd == MemCmd::CleanEvict); 152 } 153 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 154 __func__, sf_item.requested, sf_item.holder); | 168 DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n", 169 __func__, retryItem.requested, retryItem.holder); |
155 } | 170 } |
171 172 eraseIfNullEntry(sf_it); |
|
156} 157 158std::pair<SnoopFilter::SnoopList, Cycles> 159SnoopFilter::lookupSnoop(const Packet* cpkt) 160{ 161 DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n", 162 __func__, cpkt->getAddr(), cpkt->cmdString()); 163 164 assert(cpkt->isRequest()); 165 166 // Broadcast / filter upward snoops 167 const bool filter_upward = true; // @todo: Make configurable 168 169 if (!filter_upward) 170 return snoopAll(lookupLatency); 171 172 Addr line_addr = cpkt->getBlockAddr(linesize); 173 auto sf_it = cachedLocations.find(line_addr); 174 bool is_hit = (sf_it != cachedLocations.end()); | 173} 174 175std::pair<SnoopFilter::SnoopList, Cycles> 176SnoopFilter::lookupSnoop(const Packet* cpkt) 177{ 178 DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n", 179 __func__, cpkt->getAddr(), cpkt->cmdString()); 180 181 assert(cpkt->isRequest()); 182 183 // Broadcast / filter upward snoops 184 const bool filter_upward = true; // @todo: Make configurable 185 186 if (!filter_upward) 187 return snoopAll(lookupLatency); 188 189 Addr line_addr = cpkt->getBlockAddr(linesize); 190 auto sf_it = cachedLocations.find(line_addr); 191 bool is_hit = (sf_it != cachedLocations.end()); |
175 // Create a new element through operator[] and modify in-place 176 SnoopItem& sf_item = is_hit ? sf_it->second : cachedLocations[line_addr]; | |
177 | 192 |
193 // If the snoop filter has no entry and its an uncacheable 194 // request, do not create a new snoop filter entry, simply return 195 // a NULL portlist. 196 if (!is_hit && cpkt->req->isUncacheable()) 197 return snoopDown(lookupLatency); 198 199 // If no hit in snoop filter create a new element and update iterator 200 if (!is_hit) 201 sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first; 202 SnoopItem& sf_item = sf_it->second; 203 |
|
178 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 179 __func__, sf_item.requested, sf_item.holder); 180 181 SnoopMask interested = (sf_item.holder | sf_item.requested); 182 183 totSnoops++; 184 if (is_hit) { 185 // Single bit set -> value is a power of two 186 if (isPow2(interested)) 187 hitSingleSnoops++; 188 else 189 hitMultiSnoops++; 190 } 191 // ReadEx and Writes require both invalidation and exlusivity, while reads 192 // require neither. Writebacks on the other hand require exclusivity but 193 // not the invalidation. Previously Writebacks did not generate upward 194 // snoops so this was never an aissue. Now that Writebacks generate snoops 195 // we need to special case for Writebacks. | 204 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 205 __func__, sf_item.requested, sf_item.holder); 206 207 SnoopMask interested = (sf_item.holder | sf_item.requested); 208 209 totSnoops++; 210 if (is_hit) { 211 // Single bit set -> value is a power of two 212 if (isPow2(interested)) 213 hitSingleSnoops++; 214 else 215 hitMultiSnoops++; 216 } 217 // ReadEx and Writes require both invalidation and exlusivity, while reads 218 // require neither. Writebacks on the other hand require exclusivity but 219 // not the invalidation. Previously Writebacks did not generate upward 220 // snoops so this was never an aissue. Now that Writebacks generate snoops 221 // we need to special case for Writebacks. |
196 assert(cpkt->cmd == MemCmd::Writeback || | 222 assert(cpkt->cmd == MemCmd::Writeback || cpkt->req->isUncacheable() || |
197 (cpkt->isInvalidate() == cpkt->needsExclusive())); 198 if (cpkt->isInvalidate() && !sf_item.requested) { 199 // Early clear of the holder, if no other request is currently going on 200 // @todo: This should possibly be updated even though we do not filter 201 // upward snoops 202 sf_item.holder = 0; 203 } 204 | 223 (cpkt->isInvalidate() == cpkt->needsExclusive())); 224 if (cpkt->isInvalidate() && !sf_item.requested) { 225 // Early clear of the holder, if no other request is currently going on 226 // @todo: This should possibly be updated even though we do not filter 227 // upward snoops 228 sf_item.holder = 0; 229 } 230 |
231 eraseIfNullEntry(sf_it); |
|
205 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n", 206 __func__, sf_item.requested, sf_item.holder, interested); 207 208 return snoopSelected(maskToPortList(interested), lookupLatency); 209} 210 211void 212SnoopFilter::updateSnoopResponse(const Packet* cpkt, --- 40 unchanged lines hidden (view full) --- 253 // The snoop filter does not see any ACKs from non-responding sharers 254 // that have been invalidated :( So below assert would be nice, but.. 255 //assert(sf_item.holder == 0); 256 sf_item.holder = 0; 257 } 258 assert(cpkt->cmd != MemCmd::Writeback); 259 sf_item.holder |= req_mask; 260 sf_item.requested &= ~req_mask; | 232 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n", 233 __func__, sf_item.requested, sf_item.holder, interested); 234 235 return snoopSelected(maskToPortList(interested), lookupLatency); 236} 237 238void 239SnoopFilter::updateSnoopResponse(const Packet* cpkt, --- 40 unchanged lines hidden (view full) --- 280 // The snoop filter does not see any ACKs from non-responding sharers 281 // that have been invalidated :( So below assert would be nice, but.. 282 //assert(sf_item.holder == 0); 283 sf_item.holder = 0; 284 } 285 assert(cpkt->cmd != MemCmd::Writeback); 286 sf_item.holder |= req_mask; 287 sf_item.requested &= ~req_mask; |
288 assert(sf_item.requested | sf_item.holder); |
|
261 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 262 __func__, sf_item.requested, sf_item.holder); 263} 264 265void 266SnoopFilter::updateSnoopForward(const Packet* cpkt, 267 const SlavePort& rsp_port, const MasterPort& req_port) 268{ 269 DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n", 270 __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(), 271 cpkt->cmdString()); 272 273 Addr line_addr = cpkt->getBlockAddr(linesize); | 289 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 290 __func__, sf_item.requested, sf_item.holder); 291} 292 293void 294SnoopFilter::updateSnoopForward(const Packet* cpkt, 295 const SlavePort& rsp_port, const MasterPort& req_port) 296{ 297 DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n", 298 __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(), 299 cpkt->cmdString()); 300 301 Addr line_addr = cpkt->getBlockAddr(linesize); |
274 SnoopItem& sf_item = cachedLocations[line_addr]; | 302 auto sf_it = cachedLocations.find(line_addr); 303 if (sf_it == cachedLocations.end()) 304 sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first; 305 SnoopItem& sf_item = sf_it->second; |
275 SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port); 276 277 assert(cpkt->isResponse()); 278 assert(cpkt->memInhibitAsserted()); 279 280 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 281 __func__, sf_item.requested, sf_item.holder); 282 283 // Remote (to this snoop filter) snoops update the filter already when they 284 // arrive from below, because we may not see any response. 285 if (cpkt->needsExclusive()) { 286 // If the request to this snoop response hit an in-flight transaction, 287 // the holder was not reset -> no assertion & do that here, now! 288 //assert(sf_item.holder == 0); 289 sf_item.holder = 0; 290 } 291 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 292 __func__, sf_item.requested, sf_item.holder); | 306 SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port); 307 308 assert(cpkt->isResponse()); 309 assert(cpkt->memInhibitAsserted()); 310 311 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 312 __func__, sf_item.requested, sf_item.holder); 313 314 // Remote (to this snoop filter) snoops update the filter already when they 315 // arrive from below, because we may not see any response. 316 if (cpkt->needsExclusive()) { 317 // If the request to this snoop response hit an in-flight transaction, 318 // the holder was not reset -> no assertion & do that here, now! 319 //assert(sf_item.holder == 0); 320 sf_item.holder = 0; 321 } 322 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 323 __func__, sf_item.requested, sf_item.holder); |
324 eraseIfNullEntry(sf_it); |
|
293} 294 295void 296SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port) 297{ 298 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 299 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 300 --- 11 unchanged lines hidden (view full) --- 312 313 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 314 __func__, sf_item.requested, sf_item.holder); 315 316 // Make sure we have seen the actual request, too 317 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\ 318 "request bit\n", sf_item.requested, sf_item.holder); 319 | 325} 326 327void 328SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port) 329{ 330 DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 331 __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 332 --- 11 unchanged lines hidden (view full) --- 344 345 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 346 __func__, sf_item.requested, sf_item.holder); 347 348 // Make sure we have seen the actual request, too 349 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\ 350 "request bit\n", sf_item.requested, sf_item.holder); 351 |
320 // Update the residency of the cache line. 321 if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) | 352 // Update the residency of the cache line. Here we assume that the 353 // line has been zapped in all caches that are not the responder. 354 if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) |
322 sf_item.holder = 0; 323 sf_item.holder |= slave_mask; 324 sf_item.requested &= ~slave_mask; | 355 sf_item.holder = 0; 356 sf_item.holder |= slave_mask; 357 sf_item.requested &= ~slave_mask; |
358 assert(sf_item.holder | sf_item.requested); |
|
325 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 326 __func__, sf_item.requested, sf_item.holder); 327} 328 329void 330SnoopFilter::regStats() 331{ 332 totRequests --- 33 unchanged lines hidden --- | 359 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 360 __func__, sf_item.requested, sf_item.holder); 361} 362 363void 364SnoopFilter::regStats() 365{ 366 totRequests --- 33 unchanged lines hidden --- |