snoop_filter.cc revision 11131
12292SN/A/* 22329SN/A * Copyright (c) 2013-2015 ARM Limited 32292SN/A * All rights reserved 42292SN/A * 52292SN/A * The license below extends only to copyright in the software and shall 62292SN/A * not be construed as granting a license to any other intellectual 72292SN/A * property including but not limited to intellectual property relating 82292SN/A * to a hardware implementation of the functionality of the software 92292SN/A * licensed hereunder. You may use the software subject to the license 102292SN/A * terms below provided that you ensure that this notice is replicated 112292SN/A * unmodified and in its entirety in all distributions of the software, 122292SN/A * modified or unmodified, in source code or in binary form. 132292SN/A * 142292SN/A * Redistribution and use in source and binary forms, with or without 152292SN/A * modification, are permitted provided that the following conditions are 162292SN/A * met: redistributions of source code must retain the above copyright 172292SN/A * notice, this list of conditions and the following disclaimer; 182292SN/A * redistributions in binary form must reproduce the above copyright 192292SN/A * notice, this list of conditions and the following disclaimer in the 202292SN/A * documentation and/or other materials provided with the distribution; 212292SN/A * neither the name of the copyright holders nor the names of its 222292SN/A * contributors may be used to endorse or promote products derived from 232292SN/A * this software without specific prior written permission. 242292SN/A * 252292SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 262292SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 272689Sktlim@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 282689Sktlim@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 292292SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 302292SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 312292SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 322292SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 332292SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 342292SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 352292SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 362292SN/A * 372292SN/A * Authors: Stephan Diestelhorst <stephan.diestelhorst@arm.com> 382292SN/A */ 392292SN/A 402669Sktlim@umich.edu/** 412292SN/A * @file 422292SN/A * Definition of a snoop filter. 432292SN/A */ 442292SN/A 452292SN/A#include "base/misc.hh" 462292SN/A#include "base/trace.hh" 472733Sktlim@umich.edu#include "debug/SnoopFilter.hh" 482292SN/A#include "mem/snoop_filter.hh" 492292SN/A#include "sim/system.hh" 502292SN/A 512292SN/Avoid 522348SN/ASnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it) 532292SN/A{ 542292SN/A SnoopItem& sf_item = sf_it->second; 552292SN/A if (!(sf_item.requested | sf_item.holder)) { 562292SN/A cachedLocations.erase(sf_it); 572292SN/A DPRINTF(SnoopFilter, "%s: Removed SF entry.\n", 582292SN/A __func__); 592292SN/A } 604329Sktlim@umich.edu} 612292SN/A 622292SN/Astd::pair<SnoopFilter::SnoopList, Cycles> 632292SN/ASnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port) 642292SN/A{ 652727Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 662727Sktlim@umich.edu __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 672727Sktlim@umich.edu 682871Sktlim@umich.edu // Ultimately we should check if the packet came from an 692871Sktlim@umich.edu // allocating source, not just if the port is snooping 702871Sktlim@umich.edu bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping(); 712871Sktlim@umich.edu Addr line_addr = cpkt->getBlockAddr(linesize); 722871Sktlim@umich.edu SnoopMask req_port = portToMask(slave_port); 732907Sktlim@umich.edu reqLookupResult = cachedLocations.find(line_addr); 742871Sktlim@umich.edu bool is_hit = (reqLookupResult != cachedLocations.end()); 752292SN/A 762292SN/A // If the snoop filter has no entry, and we should not allocate, 772348SN/A // do not create a new snoop filter entry, simply return a NULL 782307SN/A // portlist. 792348SN/A if (!is_hit && !allocate) 802307SN/A return snoopDown(lookupLatency); 812307SN/A 822292SN/A // If no hit in snoop filter create a new element and update iterator 832292SN/A if (!is_hit) 842292SN/A reqLookupResult = cachedLocations.emplace(line_addr, SnoopItem()).first; 852292SN/A SnoopItem& sf_item = reqLookupResult->second; 862292SN/A SnoopMask interested = sf_item.holder | sf_item.requested; 872292SN/A 882292SN/A // Store unmodified value of snoop filter item in temp storage in 892292SN/A // case we need to revert because of a send retry in 902292SN/A // updateRequest. 912292SN/A retryItem = sf_item; 922292SN/A 932329SN/A totRequests++; 942329SN/A if (is_hit) { 952292SN/A // Single bit set -> value is a power of two 962292SN/A if (isPow2(interested)) 972292SN/A hitSingleRequests++; 982292SN/A else 992292SN/A hitMultiRequests++; 1002292SN/A } 1012292SN/A 1022292SN/A DPRINTF(SnoopFilter, "%s: SF value %x.%x\n", 1032292SN/A __func__, sf_item.requested, sf_item.holder); 1042292SN/A 1052292SN/A // If we are not allocating, we are done 1062292SN/A if (!allocate) 1072292SN/A return snoopSelected(maskToPortList(interested & ~req_port), 1082292SN/A lookupLatency); 1092292SN/A 1102329SN/A if (cpkt->needsResponse()) { 1112329SN/A if (!cpkt->memInhibitAsserted()) { 1122329SN/A // Max one request per address per port 1132292SN/A panic_if(sf_item.requested & req_port, "double request :( " \ 1142292SN/A "SF value %x.%x\n", sf_item.requested, sf_item.holder); 1152292SN/A 1162329SN/A // Mark in-flight requests to distinguish later on 1172329SN/A sf_item.requested |= req_port; 1182292SN/A DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 1192292SN/A __func__, sf_item.requested, sf_item.holder); 1202292SN/A } else { 1212292SN/A // NOTE: The memInhibit might have been asserted by a cache closer 1222292SN/A // to the CPU, already -> the response will not be seen by this 1232292SN/A // filter -> we do not need to keep the in-flight request, but make 1242292SN/A // sure that we know that that cluster has a copy 1252292SN/A panic_if(!(sf_item.holder & req_port), "Need to hold the value!"); 1262292SN/A DPRINTF(SnoopFilter, 1272292SN/A "%s: not marking request. SF value %x.%x\n", 1282292SN/A __func__, sf_item.requested, sf_item.holder); 1292292SN/A } 1302329SN/A } else { // if (!cpkt->needsResponse()) 1312329SN/A assert(cpkt->evictingBlock()); 1322292SN/A // make sure that the sender actually had the line 1332292SN/A panic_if(!(sf_item.holder & req_port), "requester %x is not a " \ 1342292SN/A "holder :( SF value %x.%x\n", req_port, 1352292SN/A sf_item.requested, sf_item.holder); 1362292SN/A // CleanEvicts and Writebacks -> the sender and all caches above 1372292SN/A // it may not have the line anymore. 1382292SN/A if (!cpkt->isBlockCached()) { 1392329SN/A sf_item.holder &= ~req_port; 1402329SN/A DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 1412292SN/A __func__, sf_item.requested, sf_item.holder); 1422292SN/A } 1432292SN/A } 1442292SN/A 1452329SN/A return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency); 1462329SN/A} 1472292SN/A 1482292SN/Avoid 1492292SN/ASnoopFilter::finishRequest(bool will_retry, const Packet* cpkt) 1502292SN/A{ 1512292SN/A if (reqLookupResult != cachedLocations.end()) { 1522292SN/A // since we rely on the caller, do a basic check to ensure 1532292SN/A // that finishRequest is being called following lookupRequest 1542292SN/A assert(reqLookupResult->first == cpkt->getBlockAddr(linesize)); 1552329SN/A if (will_retry) { 1562329SN/A // Undo any changes made in lookupRequest to the snoop filter 1572292SN/A // entry if the request will come again. retryItem holds 1582292SN/A // the previous value of the snoopfilter entry. 1592329SN/A reqLookupResult->second = retryItem; 1602329SN/A 1612329SN/A DPRINTF(SnoopFilter, "%s: restored SF value %x.%x\n", 1622292SN/A __func__, retryItem.requested, retryItem.holder); 1632292SN/A } 1642292SN/A 1652292SN/A eraseIfNullEntry(reqLookupResult); 1662292SN/A } 1672292SN/A} 1682292SN/A 1692329SN/Astd::pair<SnoopFilter::SnoopList, Cycles> 1702329SN/ASnoopFilter::lookupSnoop(const Packet* cpkt) 1712329SN/A{ 1722292SN/A DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n", 1732292SN/A __func__, cpkt->getAddr(), cpkt->cmdString()); 1742292SN/A 1752292SN/A assert(cpkt->isRequest()); 1762292SN/A 1772292SN/A // Broadcast / filter upward snoops 1782292SN/A const bool filter_upward = true; // @todo: Make configurable 1792292SN/A 1802292SN/A if (!filter_upward) 1812329SN/A return snoopAll(lookupLatency); 1822329SN/A 1832292SN/A Addr line_addr = cpkt->getBlockAddr(linesize); 1842292SN/A auto sf_it = cachedLocations.find(line_addr); 1852292SN/A bool is_hit = (sf_it != cachedLocations.end()); 1862292SN/A 1872329SN/A // If the snoop filter has no entry and its an uncacheable 1882329SN/A // request, do not create a new snoop filter entry, simply return 1892292SN/A // a NULL portlist. 1902292SN/A if (!is_hit && cpkt->req->isUncacheable()) 1912292SN/A return snoopDown(lookupLatency); 1922292SN/A 1932329SN/A // If no hit in snoop filter create a new element and update iterator 1942329SN/A if (!is_hit) 1952292SN/A sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first; 1962292SN/A SnoopItem& sf_item = sf_it->second; 1972292SN/A 1982292SN/A DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 1992329SN/A __func__, sf_item.requested, sf_item.holder); 2002329SN/A 2012292SN/A SnoopMask interested = (sf_item.holder | sf_item.requested); 2022292SN/A 2032292SN/A totSnoops++; 2042292SN/A if (is_hit) { 2052292SN/A // Single bit set -> value is a power of two 2062292SN/A if (isPow2(interested)) 2072292SN/A hitSingleSnoops++; 2082292SN/A else 2092292SN/A hitMultiSnoops++; 2102292SN/A } 2112292SN/A // ReadEx and Writes require both invalidation and exlusivity, while reads 2122292SN/A // require neither. Writebacks on the other hand require exclusivity but 2132292SN/A // not the invalidation. Previously Writebacks did not generate upward 2142292SN/A // snoops so this was never an aissue. Now that Writebacks generate snoops 2152292SN/A // we need to special case for Writebacks. 2162292SN/A assert(cpkt->cmd == MemCmd::Writeback || cpkt->req->isUncacheable() || 2172292SN/A (cpkt->isInvalidate() == cpkt->needsExclusive())); 2182292SN/A if (cpkt->isInvalidate() && !sf_item.requested) { 2192292SN/A // Early clear of the holder, if no other request is currently going on 2202292SN/A // @todo: This should possibly be updated even though we do not filter 2212292SN/A // upward snoops 2222292SN/A sf_item.holder = 0; 2232292SN/A } 2242292SN/A 2252292SN/A eraseIfNullEntry(sf_it); 2262292SN/A DPRINTF(SnoopFilter, "%s: new SF value %x.%x interest: %x \n", 2272292SN/A __func__, sf_item.requested, sf_item.holder, interested); 2282292SN/A 2292292SN/A return snoopSelected(maskToPortList(interested), lookupLatency); 2302292SN/A} 2312292SN/A 2322292SN/Avoid 2332292SN/ASnoopFilter::updateSnoopResponse(const Packet* cpkt, 2342292SN/A const SlavePort& rsp_port, 2352292SN/A const SlavePort& req_port) 2362292SN/A{ 2372292SN/A DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n", 2382329SN/A __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(), 2392292SN/A cpkt->cmdString()); 2402292SN/A 2412292SN/A assert(cpkt->isResponse()); 2422329SN/A assert(cpkt->memInhibitAsserted()); 2432329SN/A 2442329SN/A // Ultimately we should check if the packet came from an 2452292SN/A // allocating source, not just if the port is snooping 2462329SN/A bool allocate = !cpkt->req->isUncacheable() && req_port.isSnooping(); 2472329SN/A if (!allocate) 2482292SN/A return; 2492292SN/A 2502292SN/A Addr line_addr = cpkt->getBlockAddr(linesize); 2512292SN/A SnoopMask rsp_mask = portToMask(rsp_port); 2522292SN/A SnoopMask req_mask = portToMask(req_port); 2532292SN/A SnoopItem& sf_item = cachedLocations[line_addr]; 2542329SN/A 2552329SN/A DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 2562292SN/A __func__, sf_item.requested, sf_item.holder); 2572907Sktlim@umich.edu 2582907Sktlim@umich.edu // The source should have the line 2592907Sktlim@umich.edu panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\ 2602907Sktlim@umich.edu "the line\n", sf_item.requested, sf_item.holder); 2612907Sktlim@umich.edu 2622907Sktlim@umich.edu // The destination should have had a request in 2632907Sktlim@umich.edu panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\ 2642907Sktlim@umich.edu "the original request\n", sf_item.requested, sf_item.holder); 2652907Sktlim@umich.edu 2662292SN/A // Update the residency of the cache line. 2672292SN/A if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) { 2682292SN/A DPRINTF(SnoopFilter, "%s: dropping %x because needs: %i shared: %i "\ 2692329SN/A "SF val: %x.%x\n", __func__, rsp_mask, 2702329SN/A cpkt->needsExclusive(), cpkt->sharedAsserted(), 2712292SN/A sf_item.requested, sf_item.holder); 2722292SN/A 2732292SN/A sf_item.holder &= ~rsp_mask; 2742669Sktlim@umich.edu // The snoop filter does not see any ACKs from non-responding sharers 2752292SN/A // that have been invalidated :( So below assert would be nice, but.. 2762292SN/A //assert(sf_item.holder == 0); 2772292SN/A sf_item.holder = 0; 2782292SN/A } 2792292SN/A assert(cpkt->cmd != MemCmd::Writeback); 2802669Sktlim@umich.edu sf_item.holder |= req_mask; 2812292SN/A sf_item.requested &= ~req_mask; 2824329Sktlim@umich.edu assert(sf_item.requested | sf_item.holder); 2834329Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 2844329Sktlim@umich.edu __func__, sf_item.requested, sf_item.holder); 2854329Sktlim@umich.edu} 2864329Sktlim@umich.edu 2874329Sktlim@umich.eduvoid 2882907Sktlim@umich.eduSnoopFilter::updateSnoopForward(const Packet* cpkt, 2892907Sktlim@umich.edu const SlavePort& rsp_port, const MasterPort& req_port) 2902907Sktlim@umich.edu{ 2912907Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n", 2922907Sktlim@umich.edu __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(), 2932907Sktlim@umich.edu cpkt->cmdString()); 2942907Sktlim@umich.edu 2952907Sktlim@umich.edu Addr line_addr = cpkt->getBlockAddr(linesize); 2962907Sktlim@umich.edu auto sf_it = cachedLocations.find(line_addr); 2972907Sktlim@umich.edu if (sf_it == cachedLocations.end()) 2982907Sktlim@umich.edu sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first; 2992907Sktlim@umich.edu SnoopItem& sf_item = sf_it->second; 3004329Sktlim@umich.edu SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port); 3012907Sktlim@umich.edu 3022907Sktlim@umich.edu assert(cpkt->isResponse()); 3033647Srdreslin@umich.edu assert(cpkt->memInhibitAsserted()); 3043647Srdreslin@umich.edu 3054192Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 3064192Sktlim@umich.edu __func__, sf_item.requested, sf_item.holder); 3072907Sktlim@umich.edu 3082907Sktlim@umich.edu // Remote (to this snoop filter) snoops update the filter already when they 3092907Sktlim@umich.edu // arrive from below, because we may not see any response. 3102907Sktlim@umich.edu if (cpkt->needsExclusive()) { 3112907Sktlim@umich.edu // If the request to this snoop response hit an in-flight transaction, 3122907Sktlim@umich.edu // the holder was not reset -> no assertion & do that here, now! 3132907Sktlim@umich.edu //assert(sf_item.holder == 0); 3142907Sktlim@umich.edu sf_item.holder = 0; 3152907Sktlim@umich.edu } 3162907Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 3172907Sktlim@umich.edu __func__, sf_item.requested, sf_item.holder); 3182907Sktlim@umich.edu eraseIfNullEntry(sf_it); 3192907Sktlim@umich.edu} 3203846Shsul@eecs.umich.edu 3212907Sktlim@umich.eduvoid 3222907Sktlim@umich.eduSnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port) 3232907Sktlim@umich.edu{ 3242907Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n", 3252907Sktlim@umich.edu __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString()); 3262907Sktlim@umich.edu 3272907Sktlim@umich.edu assert(cpkt->isResponse()); 3282907Sktlim@umich.edu 3292907Sktlim@umich.edu // Ultimately we should check if the packet came from an 3302907Sktlim@umich.edu // allocating source, not just if the port is snooping 3312907Sktlim@umich.edu bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping(); 3322907Sktlim@umich.edu if (!allocate) 3332907Sktlim@umich.edu return; 3344192Sktlim@umich.edu 3354192Sktlim@umich.edu Addr line_addr = cpkt->getBlockAddr(linesize); 3364192Sktlim@umich.edu SnoopMask slave_mask = portToMask(slave_port); 3374192Sktlim@umich.edu SnoopItem& sf_item = cachedLocations[line_addr]; 3384192Sktlim@umich.edu 3392907Sktlim@umich.edu DPRINTF(SnoopFilter, "%s: old SF value %x.%x\n", 3402292SN/A __func__, sf_item.requested, sf_item.holder); 3412292SN/A 3422292SN/A // Make sure we have seen the actual request, too 3432292SN/A panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\ 3442292SN/A "request bit\n", sf_item.requested, sf_item.holder); 3452292SN/A 3462292SN/A // Update the residency of the cache line. Here we assume that the 3472292SN/A // line has been zapped in all caches that are not the responder. 3482292SN/A if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) 3492292SN/A sf_item.holder = 0; 3502292SN/A sf_item.holder |= slave_mask; 3512292SN/A sf_item.requested &= ~slave_mask; 3522292SN/A assert(sf_item.holder | sf_item.requested); 3532292SN/A DPRINTF(SnoopFilter, "%s: new SF value %x.%x\n", 3542292SN/A __func__, sf_item.requested, sf_item.holder); 3552292SN/A} 3562292SN/A 3572292SN/Avoid 3582292SN/ASnoopFilter::regStats() 3592292SN/A{ 3602292SN/A totRequests 3612292SN/A .name(name() + ".tot_requests") 3622907Sktlim@umich.edu .desc("Total number of requests made to the snoop filter."); 3632907Sktlim@umich.edu 3642907Sktlim@umich.edu hitSingleRequests 3652907Sktlim@umich.edu .name(name() + ".hit_single_requests") 3662292SN/A .desc("Number of requests hitting in the snoop filter with a single "\ 3672292SN/A "holder of the requested data."); 3682292SN/A 3692292SN/A hitMultiRequests 3702292SN/A .name(name() + ".hit_multi_requests") 3712669Sktlim@umich.edu .desc("Number of requests hitting in the snoop filter with multiple "\ 3722292SN/A "(>1) holders of the requested data."); 3732669Sktlim@umich.edu 3742292SN/A totSnoops 3752292SN/A .name(name() + ".tot_snoops") 3762292SN/A .desc("Total number of snoops made to the snoop filter."); 3772292SN/A 3782292SN/A hitSingleSnoops 3792292SN/A .name(name() + ".hit_single_snoops") 3802292SN/A .desc("Number of snoops hitting in the snoop filter with a single "\ 3812669Sktlim@umich.edu "holder of the requested data."); 3822292SN/A 3832669Sktlim@umich.edu hitMultiSnoops 3842292SN/A .name(name() + ".hit_multi_snoops") 3852292SN/A .desc("Number of snoops hitting in the snoop filter with multiple "\ 3862292SN/A "(>1) holders of the requested data."); 3872292SN/A} 3882292SN/A 389SnoopFilter * 390SnoopFilterParams::create() 391{ 392 return new SnoopFilter(this); 393} 394