snoop_filter.cc revision 12429:beefb9f5f551
1/* 2 * Copyright (c) 2013-2017 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Stephan Diestelhorst 38 */ 39 40/** 41 * @file 42 * Implementation of a snoop filter. 43 */ 44 45#include "mem/snoop_filter.hh" 46 47#include "base/logging.hh" 48#include "base/trace.hh" 49#include "debug/SnoopFilter.hh" 50#include "sim/system.hh" 51 52void 53SnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it) 54{ 55 SnoopItem& sf_item = sf_it->second; 56 if (!(sf_item.requested | sf_item.holder)) { 57 cachedLocations.erase(sf_it); 58 DPRINTF(SnoopFilter, "%s: Removed SF entry.\n", 59 __func__); 60 } 61} 62 63std::pair<SnoopFilter::SnoopList, Cycles> 64SnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port) 65{ 66 DPRINTF(SnoopFilter, "%s: src %s packet %s\n", __func__, 67 slave_port.name(), cpkt->print()); 68 69 // check if the packet came from a cache 70 bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping() && 71 cpkt->fromCache(); 72 Addr line_addr = cpkt->getBlockAddr(linesize); 73 if (cpkt->isSecure()) { 74 line_addr |= LineSecure; 75 } 76 SnoopMask req_port = portToMask(slave_port); 77 reqLookupResult = cachedLocations.find(line_addr); 78 bool is_hit = (reqLookupResult != cachedLocations.end()); 79 80 // If the snoop filter has no entry, and we should not allocate, 81 // do not create a new snoop filter entry, simply return a NULL 82 // portlist. 83 if (!is_hit && !allocate) 84 return snoopDown(lookupLatency); 85 86 // If no hit in snoop filter create a new element and update iterator 87 if (!is_hit) 88 reqLookupResult = cachedLocations.emplace(line_addr, SnoopItem()).first; 89 SnoopItem& sf_item = reqLookupResult->second; 90 SnoopMask interested = sf_item.holder | sf_item.requested; 91 92 // Store unmodified value of snoop filter item in temp storage in 93 // case we need to revert because of a send retry in 94 // updateRequest. 95 retryItem = sf_item; 96 97 totRequests++; 98 if (is_hit) { 99 // Single bit set -> value is a power of two 100 if (isPow2(interested)) 101 hitSingleRequests++; 102 else 103 hitMultiRequests++; 104 } 105 106 DPRINTF(SnoopFilter, "%s: SF value %x.%x\n", 107 __func__, sf_item.requested, sf_item.holder); 108 109 // If we are not allocating, we are done 110 if (!allocate) 111 return snoopSelected(maskToPortList(interested & ~req_port), 112 lookupLatency); 113 114 if (cpkt->needsResponse()) { 115 if (!cpkt->cacheResponding()) { 116 // Max one request per address per port 117 panic_if(sf_item.requested & req_port, "double request :( " \ 118 "SF value %x.%x\n", sf_item.requested, sf_item.holder); 119 120 // Mark in-flight requests to distinguish later on 121 sf_item.requested |= req_port; 122 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 123 __func__, sf_item.requested, sf_item.holder); 124 } else { 125 // NOTE: The memInhibit might have been asserted by a cache closer 126 // to the CPU, already -> the response will not be seen by this 127 // filter -> we do not need to keep the in-flight request, but make 128 // sure that we know that that cluster has a copy 129 panic_if(!(sf_item.holder & req_port), "Need to hold the value!"); 130 DPRINTF(SnoopFilter, 131 "%s: not marking request. SF value %x.%x\n", 132 __func__, sf_item.requested, sf_item.holder); 133 } 134 } else { // if (!cpkt->needsResponse()) 135 assert(cpkt->isEviction()); 136 // make sure that the sender actually had the line 137 panic_if(!(sf_item.holder & req_port), "requester %x is not a " \ 138 "holder :( SF value %x.%x\n", req_port, 139 sf_item.requested, sf_item.holder); 140 // CleanEvicts and Writebacks -> the sender and all caches above 141 // it may not have the line anymore. 142 if (!cpkt->isBlockCached()) { 143 sf_item.holder &= ~req_port; 144 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 145 __func__, sf_item.requested, sf_item.holder); 146 } 147 } 148 149 return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency); 150} 151 152void 153SnoopFilter::finishRequest(bool will_retry, Addr addr, bool is_secure) 154{ 155 if (reqLookupResult != cachedLocations.end()) { 156 // since we rely on the caller, do a basic check to ensure 157 // that finishRequest is being called following lookupRequest 158 Addr line_addr = (addr & ~(Addr(linesize - 1))); 159 if (is_secure) { 160 line_addr |= LineSecure; 161 } 162 assert(reqLookupResult->first == line_addr); 163 if (will_retry) { 164 // Undo any changes made in lookupRequest to the snoop filter 165 // entry if the request will come again. retryItem holds 166 // the previous value of the snoopfilter entry. 167 reqLookupResult->second = retryItem; 168 169 DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n", 170 __func__, retryItem.requested, retryItem.holder); 171 } 172 173 eraseIfNullEntry(reqLookupResult); 174 } 175} 176 177std::pair<SnoopFilter::SnoopList, Cycles> 178SnoopFilter::lookupSnoop(const Packet* cpkt) 179{ 180 DPRINTF(SnoopFilter, "%s: packet %s\n", __func__, cpkt->print()); 181 182 assert(cpkt->isRequest()); 183 184 Addr line_addr = cpkt->getBlockAddr(linesize); 185 if (cpkt->isSecure()) { 186 line_addr |= LineSecure; 187 } 188 auto sf_it = cachedLocations.find(line_addr); 189 bool is_hit = (sf_it != cachedLocations.end()); 190 191 panic_if(!is_hit && (cachedLocations.size() >= maxEntryCount), 192 "snoop filter exceeded capacity of %d cache blocks\n", 193 maxEntryCount); 194 195 // If the snoop filter has no entry, simply return a NULL 196 // portlist, there is no point creating an entry only to remove it 197 // later 198 if (!is_hit) 199 return snoopDown(lookupLatency); 200 201 SnoopItem& sf_item = sf_it->second; 202 203 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 204 __func__, sf_item.requested, sf_item.holder); 205 206 SnoopMask interested = (sf_item.holder | sf_item.requested); 207 208 totSnoops++; 209 // Single bit set -> value is a power of two 210 if (isPow2(interested)) 211 hitSingleSnoops++; 212 else 213 hitMultiSnoops++; 214 215 // ReadEx and Writes require both invalidation and exlusivity, while reads 216 // require neither. Writebacks on the other hand require exclusivity but 217 // not the invalidation. Previously Writebacks did not generate upward 218 // snoops so this was never an issue. Now that Writebacks generate snoops 219 // we need a special case for Writebacks. Additionally cache maintenance 220 // operations can generate snoops as they clean and/or invalidate all 221 // caches down to the specified point of reference. 222 assert(cpkt->isWriteback() || cpkt->req->isUncacheable() || 223 (cpkt->isInvalidate() == cpkt->needsWritable()) || 224 cpkt->req->isCacheMaintenance()); 225 if (cpkt->isInvalidate() && !sf_item.requested) { 226 // Early clear of the holder, if no other request is currently going on 227 // @todo: This should possibly be updated even though we do not filter 228 // upward snoops 229 sf_item.holder = 0; 230 } 231 232 eraseIfNullEntry(sf_it); 233 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n", 234 __func__, sf_item.requested, sf_item.holder, interested); 235 236 return snoopSelected(maskToPortList(interested), lookupLatency); 237} 238 239void 240SnoopFilter::updateSnoopResponse(const Packet* cpkt, 241 const SlavePort& rsp_port, 242 const SlavePort& req_port) 243{ 244 DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n", 245 __func__, rsp_port.name(), req_port.name(), cpkt->print()); 246 247 assert(cpkt->isResponse()); 248 assert(cpkt->cacheResponding()); 249 250 // if this snoop response is due to an uncacheable request, or is 251 // being turned into a normal response, there is nothing more to 252 // do 253 if (cpkt->req->isUncacheable() || !req_port.isSnooping()) { 254 return; 255 } 256 257 Addr line_addr = cpkt->getBlockAddr(linesize); 258 if (cpkt->isSecure()) { 259 line_addr |= LineSecure; 260 } 261 SnoopMask rsp_mask = portToMask(rsp_port); 262 SnoopMask req_mask = portToMask(req_port); 263 SnoopItem& sf_item = cachedLocations[line_addr]; 264 265 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 266 __func__, sf_item.requested, sf_item.holder); 267 268 // The source should have the line 269 panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\ 270 "the line\n", sf_item.requested, sf_item.holder); 271 272 // The destination should have had a request in 273 panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\ 274 "the original request\n", sf_item.requested, sf_item.holder); 275 276 // If the snoop response has no sharers the line is passed in 277 // Modified state, and we know that there are no other copies, or 278 // they will all be invalidated imminently 279 if (!cpkt->hasSharers()) { 280 DPRINTF(SnoopFilter, 281 "%s: dropping %x because non-shared snoop " 282 "response SF val: %x.%x\n", __func__, rsp_mask, 283 sf_item.requested, sf_item.holder); 284 sf_item.holder = 0; 285 } 286 assert(!cpkt->isWriteback()); 287 // @todo Deal with invalidating responses 288 sf_item.holder |= req_mask; 289 sf_item.requested &= ~req_mask; 290 assert(sf_item.requested | sf_item.holder); 291 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 292 __func__, sf_item.requested, sf_item.holder); 293} 294 295void 296SnoopFilter::updateSnoopForward(const Packet* cpkt, 297 const SlavePort& rsp_port, const MasterPort& req_port) 298{ 299 DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n", 300 __func__, rsp_port.name(), req_port.name(), cpkt->print()); 301 302 assert(cpkt->isResponse()); 303 assert(cpkt->cacheResponding()); 304 305 Addr line_addr = cpkt->getBlockAddr(linesize); 306 if (cpkt->isSecure()) { 307 line_addr |= LineSecure; 308 } 309 auto sf_it = cachedLocations.find(line_addr); 310 bool is_hit = sf_it != cachedLocations.end(); 311 312 // Nothing to do if it is not a hit 313 if (!is_hit) 314 return; 315 316 SnoopItem& sf_item = sf_it->second; 317 318 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 319 __func__, sf_item.requested, sf_item.holder); 320 321 // If the snoop response has no sharers the line is passed in 322 // Modified state, and we know that there are no other copies, or 323 // they will all be invalidated imminently 324 if (!cpkt->hasSharers()) { 325 sf_item.holder = 0; 326 } 327 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 328 __func__, sf_item.requested, sf_item.holder); 329 eraseIfNullEntry(sf_it); 330 331} 332 333void 334SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port) 335{ 336 DPRINTF(SnoopFilter, "%s: src %s packet %s\n", 337 __func__, slave_port.name(), cpkt->print()); 338 339 assert(cpkt->isResponse()); 340 341 // we only allocate if the packet actually came from a cache, but 342 // start by checking if the port is snooping 343 if (cpkt->req->isUncacheable() || !slave_port.isSnooping()) 344 return; 345 346 // next check if we actually allocated an entry 347 Addr line_addr = cpkt->getBlockAddr(linesize); 348 if (cpkt->isSecure()) { 349 line_addr |= LineSecure; 350 } 351 auto sf_it = cachedLocations.find(line_addr); 352 if (sf_it == cachedLocations.end()) 353 return; 354 355 SnoopMask slave_mask = portToMask(slave_port); 356 SnoopItem& sf_item = sf_it->second; 357 358 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 359 __func__, sf_item.requested, sf_item.holder); 360 361 // Make sure we have seen the actual request, too 362 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\ 363 "request bit\n", sf_item.requested, sf_item.holder); 364 365 sf_item.requested &= ~slave_mask; 366 // Update the residency of the cache line. 367 368 if (cpkt->req->isCacheMaintenance()) { 369 // A cache clean response does not carry any data so it 370 // shouldn't change the holders, unless it is invalidating. 371 if (cpkt->isInvalidate()) { 372 sf_item.holder &= ~slave_mask; 373 } 374 eraseIfNullEntry(sf_it); 375 } else { 376 // Any other response implies that a cache above will have the 377 // block. 378 sf_item.holder |= slave_mask; 379 assert(sf_item.holder | sf_item.requested); 380 } 381 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 382 __func__, sf_item.requested, sf_item.holder); 383} 384 385void 386SnoopFilter::regStats() 387{ 388 SimObject::regStats(); 389 390 totRequests 391 .name(name() + ".tot_requests") 392 .desc("Total number of requests made to the snoop filter."); 393 394 hitSingleRequests 395 .name(name() + ".hit_single_requests") 396 .desc("Number of requests hitting in the snoop filter with a single "\ 397 "holder of the requested data."); 398 399 hitMultiRequests 400 .name(name() + ".hit_multi_requests") 401 .desc("Number of requests hitting in the snoop filter with multiple "\ 402 "(>1) holders of the requested data."); 403 404 totSnoops 405 .name(name() + ".tot_snoops") 406 .desc("Total number of snoops made to the snoop filter."); 407 408 hitSingleSnoops 409 .name(name() + ".hit_single_snoops") 410 .desc("Number of snoops hitting in the snoop filter with a single "\ 411 "holder of the requested data."); 412 413 hitMultiSnoops 414 .name(name() + ".hit_multi_snoops") 415 .desc("Number of snoops hitting in the snoop filter with multiple "\ 416 "(>1) holders of the requested data."); 417} 418 419SnoopFilter * 420SnoopFilterParams::create() 421{ 422 return new SnoopFilter(this); 423} 424