snoop_filter.cc revision 11748
12SN/A/* 21762SN/A * Copyright (c) 2013-2016 ARM Limited 32SN/A * All rights reserved 42SN/A * 52SN/A * The license below extends only to copyright in the software and shall 62SN/A * not be construed as granting a license to any other intellectual 72SN/A * property including but not limited to intellectual property relating 82SN/A * to a hardware implementation of the functionality of the software 92SN/A * licensed hereunder. You may use the software subject to the license 102SN/A * terms below provided that you ensure that this notice is replicated 112SN/A * unmodified and in its entirety in all distributions of the software, 122SN/A * modified or unmodified, in source code or in binary form. 132SN/A * 142SN/A * Redistribution and use in source and binary forms, with or without 152SN/A * modification, are permitted provided that the following conditions are 162SN/A * met: redistributions of source code must retain the above copyright 172SN/A * notice, this list of conditions and the following disclaimer; 182SN/A * redistributions in binary form must reproduce the above copyright 192SN/A * notice, this list of conditions and the following disclaimer in the 202SN/A * documentation and/or other materials provided with the distribution; 212SN/A * neither the name of the copyright holders nor the names of its 222SN/A * contributors may be used to endorse or promote products derived from 232SN/A * this software without specific prior written permission. 242SN/A * 252SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 262SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 272665SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 282665SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 292SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 302SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 312SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 322SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 336214Snate@binkert.org * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 342SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 352SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 362SN/A * 376214Snate@binkert.org * Authors: Stephan Diestelhorst 386214Snate@binkert.org */ 392SN/A 402SN/A/** 412SN/A * @file 429180Sandreas.hansson@arm.com * Implementation of a snoop filter. 4310474Sandreas.hansson@arm.com */ 449500Snilay@cs.wisc.edu 4511004SAndreas.Sandberg@ARM.com#include "base/misc.hh" 469180Sandreas.hansson@arm.com#include "base/trace.hh" 4710276SAndreas.Sandberg@ARM.com#include "debug/SnoopFilter.hh" 4810276SAndreas.Sandberg@ARM.com#include "mem/snoop_filter.hh" 492SN/A#include "sim/system.hh" 505543SN/A 512SN/Avoid 525543SN/ASnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it) 532SN/A{ 542SN/A SnoopItem& sf_item = sf_it->second; 552SN/A if (!(sf_item.requested | sf_item.holder)) { 562SN/A cachedLocations.erase(sf_it); 572SN/A DPRINTF(SnoopFilter, "%s: Removed SF entry.\n", 582SN/A __func__); 592SN/A } 602SN/A} 619158Sandreas.hansson@arm.com 622SN/Astd::pair<SnoopFilter::SnoopList, Cycles> 639158Sandreas.hansson@arm.comSnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port) 642SN/A{ 659158Sandreas.hansson@arm.com DPRINTF(SnoopFilter, "%s: src %s packet %s\n", __func__, 662667SN/A slave_port.name(), cpkt->print()); 672130SN/A 689180Sandreas.hansson@arm.com // check if the packet came from a cache 699180Sandreas.hansson@arm.com bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping() && 709180Sandreas.hansson@arm.com cpkt->fromCache(); 719180Sandreas.hansson@arm.com Addr line_addr = cpkt->getBlockAddr(linesize); 729180Sandreas.hansson@arm.com if (cpkt->isSecure()) { 739180Sandreas.hansson@arm.com line_addr |= LineSecure; 749180Sandreas.hansson@arm.com } 759180Sandreas.hansson@arm.com SnoopMask req_port = portToMask(slave_port); 7611990Sandreas.sandberg@arm.com reqLookupResult = cachedLocations.find(line_addr); 779180Sandreas.hansson@arm.com bool is_hit = (reqLookupResult != cachedLocations.end()); 789180Sandreas.hansson@arm.com 799180Sandreas.hansson@arm.com // If the snoop filter has no entry, and we should not allocate, 809180Sandreas.hansson@arm.com // do not create a new snoop filter entry, simply return a NULL 819180Sandreas.hansson@arm.com // portlist. 829180Sandreas.hansson@arm.com if (!is_hit && !allocate) 839180Sandreas.hansson@arm.com return snoopDown(lookupLatency); 849180Sandreas.hansson@arm.com 859180Sandreas.hansson@arm.com // If no hit in snoop filter create a new element and update iterator 869180Sandreas.hansson@arm.com if (!is_hit) 879180Sandreas.hansson@arm.com reqLookupResult = cachedLocations.emplace(line_addr, SnoopItem()).first; 889180Sandreas.hansson@arm.com SnoopItem& sf_item = reqLookupResult->second; 899180Sandreas.hansson@arm.com SnoopMask interested = sf_item.holder | sf_item.requested; 909180Sandreas.hansson@arm.com 919180Sandreas.hansson@arm.com // Store unmodified value of snoop filter item in temp storage in 929180Sandreas.hansson@arm.com // case we need to revert because of a send retry in 939180Sandreas.hansson@arm.com // updateRequest. 9411004SAndreas.Sandberg@ARM.com retryItem = sf_item; 959180Sandreas.hansson@arm.com 969184Sandreas.hansson@arm.com totRequests++; 979184Sandreas.hansson@arm.com if (is_hit) { 989184Sandreas.hansson@arm.com // Single bit set -> value is a power of two 999180Sandreas.hansson@arm.com if (isPow2(interested)) 10011004SAndreas.Sandberg@ARM.com hitSingleRequests++; 1019180Sandreas.hansson@arm.com else 1029180Sandreas.hansson@arm.com hitMultiRequests++; 1039180Sandreas.hansson@arm.com } 1049180Sandreas.hansson@arm.com 1059180Sandreas.hansson@arm.com DPRINTF(SnoopFilter, "%s: SF value %x.%x\n", 1069180Sandreas.hansson@arm.com __func__, sf_item.requested, sf_item.holder); 1079180Sandreas.hansson@arm.com 1089180Sandreas.hansson@arm.com // If we are not allocating, we are done 1099180Sandreas.hansson@arm.com if (!allocate) 1109180Sandreas.hansson@arm.com return snoopSelected(maskToPortList(interested & ~req_port), 11111004SAndreas.Sandberg@ARM.com lookupLatency); 1129180Sandreas.hansson@arm.com 1139180Sandreas.hansson@arm.com if (cpkt->needsResponse()) { 1149180Sandreas.hansson@arm.com if (!cpkt->cacheResponding()) { 11511004SAndreas.Sandberg@ARM.com // Max one request per address per port 1169180Sandreas.hansson@arm.com panic_if(sf_item.requested & req_port, "double request :( " \ 1179180Sandreas.hansson@arm.com "SF value %x.%x\n", sf_item.requested, sf_item.holder); 11811004SAndreas.Sandberg@ARM.com 1199498Snilay@cs.wisc.edu // Mark in-flight requests to distinguish later on 1209498Snilay@cs.wisc.edu sf_item.requested |= req_port; 12111004SAndreas.Sandberg@ARM.com DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 12211004SAndreas.Sandberg@ARM.com __func__, sf_item.requested, sf_item.holder); 12311004SAndreas.Sandberg@ARM.com } else { 12411004SAndreas.Sandberg@ARM.com // NOTE: The memInhibit might have been asserted by a cache closer 12511004SAndreas.Sandberg@ARM.com // to the CPU, already -> the response will not be seen by this 1269498Snilay@cs.wisc.edu // filter -> we do not need to keep the in-flight request, but make 12711004SAndreas.Sandberg@ARM.com // sure that we know that that cluster has a copy 1289498Snilay@cs.wisc.edu panic_if(!(sf_item.holder & req_port), "Need to hold the value!"); 1299498Snilay@cs.wisc.edu DPRINTF(SnoopFilter, 13011004SAndreas.Sandberg@ARM.com "%s: not marking request. SF value %x.%x\n", 1319498Snilay@cs.wisc.edu __func__, sf_item.requested, sf_item.holder); 1329498Snilay@cs.wisc.edu } 1339500Snilay@cs.wisc.edu } else { // if (!cpkt->needsResponse()) 1349180Sandreas.hansson@arm.com assert(cpkt->isEviction()); 1359180Sandreas.hansson@arm.com // make sure that the sender actually had the line 1369180Sandreas.hansson@arm.com panic_if(!(sf_item.holder & req_port), "requester %x is not a " \ 1372130SN/A "holder :( SF value %x.%x\n", req_port, 1382130SN/A sf_item.requested, sf_item.holder); 1392130SN/A // CleanEvicts and Writebacks -> the sender and all caches above 1402130SN/A // it may not have the line anymore. 1412130SN/A if (!cpkt->isBlockCached()) { 1422130SN/A sf_item.holder &= ~req_port; 1432130SN/A DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 1447720Sgblack@eecs.umich.edu __func__, sf_item.requested, sf_item.holder); 1457720Sgblack@eecs.umich.edu } 1467720Sgblack@eecs.umich.edu } 1477720Sgblack@eecs.umich.edu 1487720Sgblack@eecs.umich.edu return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency); 1497720Sgblack@eecs.umich.edu} 1507720Sgblack@eecs.umich.edu 1517720Sgblack@eecs.umich.eduvoid 1527720Sgblack@eecs.umich.eduSnoopFilter::finishRequest(bool will_retry, Addr addr, bool is_secure) 1537720Sgblack@eecs.umich.edu{ 1547720Sgblack@eecs.umich.edu if (reqLookupResult != cachedLocations.end()) { 1557720Sgblack@eecs.umich.edu // since we rely on the caller, do a basic check to ensure 1567720Sgblack@eecs.umich.edu // that finishRequest is being called following lookupRequest 1577720Sgblack@eecs.umich.edu Addr line_addr = (addr & ~(Addr(linesize - 1))); 1587720Sgblack@eecs.umich.edu if (is_secure) { 1597720Sgblack@eecs.umich.edu line_addr |= LineSecure; 1607720Sgblack@eecs.umich.edu } 1617720Sgblack@eecs.umich.edu assert(reqLookupResult->first == line_addr); 1627720Sgblack@eecs.umich.edu if (will_retry) { 1637720Sgblack@eecs.umich.edu // Undo any changes made in lookupRequest to the snoop filter 1647720Sgblack@eecs.umich.edu // entry if the request will come again. retryItem holds 1657720Sgblack@eecs.umich.edu // the previous value of the snoopfilter entry. 1662438SN/A reqLookupResult->second = retryItem; 1672438SN/A 1686221Snate@binkert.org DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n", 1696221Snate@binkert.org __func__, retryItem.requested, retryItem.holder); 1706221Snate@binkert.org } 1716221Snate@binkert.org 1726221Snate@binkert.org eraseIfNullEntry(reqLookupResult); 1736221Snate@binkert.org } 17411005Sandreas.sandberg@arm.com} 17511005Sandreas.sandberg@arm.com 17611005Sandreas.sandberg@arm.comstd::pair<SnoopFilter::SnoopList, Cycles> 17711005Sandreas.sandberg@arm.comSnoopFilter::lookupSnoop(const Packet* cpkt) 1789031Sandreas.hansson@arm.com{ 1799031Sandreas.hansson@arm.com DPRINTF(SnoopFilter, "%s: packet %s\n", __func__, cpkt->print()); 1809031Sandreas.hansson@arm.com 1819031Sandreas.hansson@arm.com assert(cpkt->isRequest()); 1829031Sandreas.hansson@arm.com 1839031Sandreas.hansson@arm.com Addr line_addr = cpkt->getBlockAddr(linesize); 1847678Sgblack@eecs.umich.edu if (cpkt->isSecure()) { 18510474Sandreas.hansson@arm.com line_addr |= LineSecure; 18610474Sandreas.hansson@arm.com } 18710474Sandreas.hansson@arm.com auto sf_it = cachedLocations.find(line_addr); 18810474Sandreas.hansson@arm.com bool is_hit = (sf_it != cachedLocations.end()); 18910474Sandreas.hansson@arm.com 1907678Sgblack@eecs.umich.edu panic_if(!is_hit && (cachedLocations.size() >= maxEntryCount), 19111306Santhony.gutierrez@amd.com "snoop filter exceeded capacity of %d cache blocks\n", 19211306Santhony.gutierrez@amd.com maxEntryCount); 19311306Santhony.gutierrez@amd.com 19411306Santhony.gutierrez@amd.com // If the snoop filter has no entry, simply return a NULL 19511306Santhony.gutierrez@amd.com // portlist, there is no point creating an entry only to remove it 19611306Santhony.gutierrez@amd.com // later 19711306Santhony.gutierrez@amd.com if (!is_hit) 19811306Santhony.gutierrez@amd.com return snoopDown(lookupLatency); 19911306Santhony.gutierrez@amd.com 20011306Santhony.gutierrez@amd.com SnoopItem& sf_item = sf_it->second; 20111306Santhony.gutierrez@amd.com 20211306Santhony.gutierrez@amd.com DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 20311306Santhony.gutierrez@amd.com __func__, sf_item.requested, sf_item.holder); 20410839Sandreas.sandberg@arm.com 20510839Sandreas.sandberg@arm.com SnoopMask interested = (sf_item.holder | sf_item.requested); 20610839Sandreas.sandberg@arm.com 20710839Sandreas.sandberg@arm.com totSnoops++; 20810839Sandreas.sandberg@arm.com // Single bit set -> value is a power of two 2096214Snate@binkert.org if (isPow2(interested)) 210 hitSingleSnoops++; 211 else 212 hitMultiSnoops++; 213 214 // ReadEx and Writes require both invalidation and exlusivity, while reads 215 // require neither. Writebacks on the other hand require exclusivity but 216 // not the invalidation. Previously Writebacks did not generate upward 217 // snoops so this was never an aissue. Now that Writebacks generate snoops 218 // we need to special case for Writebacks. 219 assert(cpkt->isWriteback() || cpkt->req->isUncacheable() || 220 (cpkt->isInvalidate() == cpkt->needsWritable())); 221 if (cpkt->isInvalidate() && !sf_item.requested) { 222 // Early clear of the holder, if no other request is currently going on 223 // @todo: This should possibly be updated even though we do not filter 224 // upward snoops 225 sf_item.holder = 0; 226 } 227 228 eraseIfNullEntry(sf_it); 229 DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n", 230 __func__, sf_item.requested, sf_item.holder, interested); 231 232 return snoopSelected(maskToPortList(interested), lookupLatency); 233} 234 235void 236SnoopFilter::updateSnoopResponse(const Packet* cpkt, 237 const SlavePort& rsp_port, 238 const SlavePort& req_port) 239{ 240 DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n", 241 __func__, rsp_port.name(), req_port.name(), cpkt->print()); 242 243 assert(cpkt->isResponse()); 244 assert(cpkt->cacheResponding()); 245 246 // if this snoop response is due to an uncacheable request, or is 247 // being turned into a normal response, there is nothing more to 248 // do 249 if (cpkt->req->isUncacheable() || !req_port.isSnooping()) { 250 return; 251 } 252 253 Addr line_addr = cpkt->getBlockAddr(linesize); 254 if (cpkt->isSecure()) { 255 line_addr |= LineSecure; 256 } 257 SnoopMask rsp_mask = portToMask(rsp_port); 258 SnoopMask req_mask = portToMask(req_port); 259 SnoopItem& sf_item = cachedLocations[line_addr]; 260 261 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 262 __func__, sf_item.requested, sf_item.holder); 263 264 // The source should have the line 265 panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\ 266 "the line\n", sf_item.requested, sf_item.holder); 267 268 // The destination should have had a request in 269 panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\ 270 "the original request\n", sf_item.requested, sf_item.holder); 271 272 // If the snoop response has no sharers the line is passed in 273 // Modified state, and we know that there are no other copies, or 274 // they will all be invalidated imminently 275 if (!cpkt->hasSharers()) { 276 DPRINTF(SnoopFilter, 277 "%s: dropping %x because non-shared snoop " 278 "response SF val: %x.%x\n", __func__, rsp_mask, 279 sf_item.requested, sf_item.holder); 280 sf_item.holder = 0; 281 } 282 assert(!cpkt->isWriteback()); 283 // @todo Deal with invalidating responses 284 sf_item.holder |= req_mask; 285 sf_item.requested &= ~req_mask; 286 assert(sf_item.requested | sf_item.holder); 287 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 288 __func__, sf_item.requested, sf_item.holder); 289} 290 291void 292SnoopFilter::updateSnoopForward(const Packet* cpkt, 293 const SlavePort& rsp_port, const MasterPort& req_port) 294{ 295 DPRINTF(SnoopFilter, "%s: rsp %s req %s packet %s\n", 296 __func__, rsp_port.name(), req_port.name(), cpkt->print()); 297 298 assert(cpkt->isResponse()); 299 assert(cpkt->cacheResponding()); 300 301 Addr line_addr = cpkt->getBlockAddr(linesize); 302 if (cpkt->isSecure()) { 303 line_addr |= LineSecure; 304 } 305 auto sf_it = cachedLocations.find(line_addr); 306 bool is_hit = sf_it != cachedLocations.end(); 307 308 // Nothing to do if it is not a hit 309 if (!is_hit) 310 return; 311 312 SnoopItem& sf_item = sf_it->second; 313 314 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 315 __func__, sf_item.requested, sf_item.holder); 316 317 // If the snoop response has no sharers the line is passed in 318 // Modified state, and we know that there are no other copies, or 319 // they will all be invalidated imminently 320 if (!cpkt->hasSharers()) { 321 sf_item.holder = 0; 322 } 323 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 324 __func__, sf_item.requested, sf_item.holder); 325 eraseIfNullEntry(sf_it); 326 327} 328 329void 330SnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port) 331{ 332 DPRINTF(SnoopFilter, "%s: src %s packet %s\n", 333 __func__, slave_port.name(), cpkt->print()); 334 335 assert(cpkt->isResponse()); 336 337 // we only allocate if the packet actually came from a cache, but 338 // start by checking if the port is snooping 339 if (cpkt->req->isUncacheable() || !slave_port.isSnooping()) 340 return; 341 342 // next check if we actually allocated an entry 343 Addr line_addr = cpkt->getBlockAddr(linesize); 344 if (cpkt->isSecure()) { 345 line_addr |= LineSecure; 346 } 347 auto sf_it = cachedLocations.find(line_addr); 348 if (sf_it == cachedLocations.end()) 349 return; 350 351 SnoopMask slave_mask = portToMask(slave_port); 352 SnoopItem& sf_item = sf_it->second; 353 354 DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 355 __func__, sf_item.requested, sf_item.holder); 356 357 // Make sure we have seen the actual request, too 358 panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\ 359 "request bit\n", sf_item.requested, sf_item.holder); 360 361 // Update the residency of the cache line. 362 sf_item.holder |= slave_mask; 363 sf_item.requested &= ~slave_mask; 364 assert(sf_item.holder | sf_item.requested); 365 DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 366 __func__, sf_item.requested, sf_item.holder); 367} 368 369void 370SnoopFilter::regStats() 371{ 372 SimObject::regStats(); 373 374 totRequests 375 .name(name() + ".tot_requests") 376 .desc("Total number of requests made to the snoop filter."); 377 378 hitSingleRequests 379 .name(name() + ".hit_single_requests") 380 .desc("Number of requests hitting in the snoop filter with a single "\ 381 "holder of the requested data."); 382 383 hitMultiRequests 384 .name(name() + ".hit_multi_requests") 385 .desc("Number of requests hitting in the snoop filter with multiple "\ 386 "(>1) holders of the requested data."); 387 388 totSnoops 389 .name(name() + ".tot_snoops") 390 .desc("Total number of snoops made to the snoop filter."); 391 392 hitSingleSnoops 393 .name(name() + ".hit_single_snoops") 394 .desc("Number of snoops hitting in the snoop filter with a single "\ 395 "holder of the requested data."); 396 397 hitMultiSnoops 398 .name(name() + ".hit_multi_snoops") 399 .desc("Number of snoops hitting in the snoop filter with multiple "\ 400 "(>1) holders of the requested data."); 401} 402 403SnoopFilter * 404SnoopFilterParams::create() 405{ 406 return new SnoopFilter(this); 407} 408