snoop_filter.cc revision 11131
12292SN/A/*
22329SN/A * Copyright (c) 2013-2015 ARM Limited
32292SN/A * All rights reserved
42292SN/A *
52292SN/A * The license below extends only to copyright in the software and shall
62292SN/A * not be construed as granting a license to any other intellectual
72292SN/A * property including but not limited to intellectual property relating
82292SN/A * to a hardware implementation of the functionality of the software
92292SN/A * licensed hereunder.  You may use the software subject to the license
102292SN/A * terms below provided that you ensure that this notice is replicated
112292SN/A * unmodified and in its entirety in all distributions of the software,
122292SN/A * modified or unmodified, in source code or in binary form.
132292SN/A *
142292SN/A * Redistribution and use in source and binary forms, with or without
152292SN/A * modification, are permitted provided that the following conditions are
162292SN/A * met: redistributions of source code must retain the above copyright
172292SN/A * notice, this list of conditions and the following disclaimer;
182292SN/A * redistributions in binary form must reproduce the above copyright
192292SN/A * notice, this list of conditions and the following disclaimer in the
202292SN/A * documentation and/or other materials provided with the distribution;
212292SN/A * neither the name of the copyright holders nor the names of its
222292SN/A * contributors may be used to endorse or promote products derived from
232292SN/A * this software without specific prior written permission.
242292SN/A *
252292SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
262292SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
272689Sktlim@umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
282689Sktlim@umich.edu * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
292292SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
302292SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
312292SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
322292SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
332292SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
342292SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
352292SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
362292SN/A *
372292SN/A * Authors: Stephan Diestelhorst <stephan.diestelhorst@arm.com>
382292SN/A */
392292SN/A
402669Sktlim@umich.edu/**
412292SN/A * @file
422292SN/A * Definition of a snoop filter.
432292SN/A */
442292SN/A
452292SN/A#include "base/misc.hh"
462292SN/A#include "base/trace.hh"
472733Sktlim@umich.edu#include "debug/SnoopFilter.hh"
482292SN/A#include "mem/snoop_filter.hh"
492292SN/A#include "sim/system.hh"
502292SN/A
512292SN/Avoid
522348SN/ASnoopFilter::eraseIfNullEntry(SnoopFilterCache::iterator& sf_it)
532292SN/A{
542292SN/A    SnoopItem& sf_item = sf_it->second;
552292SN/A    if (!(sf_item.requested | sf_item.holder)) {
562292SN/A        cachedLocations.erase(sf_it);
572292SN/A        DPRINTF(SnoopFilter, "%s:   Removed SF entry.\n",
582292SN/A                __func__);
592292SN/A    }
604329Sktlim@umich.edu}
612292SN/A
622292SN/Astd::pair<SnoopFilter::SnoopList, Cycles>
632292SN/ASnoopFilter::lookupRequest(const Packet* cpkt, const SlavePort& slave_port)
642292SN/A{
652727Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
662727Sktlim@umich.edu            __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
672727Sktlim@umich.edu
682871Sktlim@umich.edu    // Ultimately we should check if the packet came from an
692871Sktlim@umich.edu    // allocating source, not just if the port is snooping
702871Sktlim@umich.edu    bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping();
712871Sktlim@umich.edu    Addr line_addr = cpkt->getBlockAddr(linesize);
722871Sktlim@umich.edu    SnoopMask req_port = portToMask(slave_port);
732907Sktlim@umich.edu    reqLookupResult = cachedLocations.find(line_addr);
742871Sktlim@umich.edu    bool is_hit = (reqLookupResult != cachedLocations.end());
752292SN/A
762292SN/A    // If the snoop filter has no entry, and we should not allocate,
772348SN/A    // do not create a new snoop filter entry, simply return a NULL
782307SN/A    // portlist.
792348SN/A    if (!is_hit && !allocate)
802307SN/A        return snoopDown(lookupLatency);
812307SN/A
822292SN/A    // If no hit in snoop filter create a new element and update iterator
832292SN/A    if (!is_hit)
842292SN/A        reqLookupResult = cachedLocations.emplace(line_addr, SnoopItem()).first;
852292SN/A    SnoopItem& sf_item = reqLookupResult->second;
862292SN/A    SnoopMask interested = sf_item.holder | sf_item.requested;
872292SN/A
882292SN/A    // Store unmodified value of snoop filter item in temp storage in
892292SN/A    // case we need to revert because of a send retry in
902292SN/A    // updateRequest.
912292SN/A    retryItem = sf_item;
922292SN/A
932329SN/A    totRequests++;
942329SN/A    if (is_hit) {
952292SN/A        // Single bit set -> value is a power of two
962292SN/A        if (isPow2(interested))
972292SN/A            hitSingleRequests++;
982292SN/A        else
992292SN/A            hitMultiRequests++;
1002292SN/A    }
1012292SN/A
1022292SN/A    DPRINTF(SnoopFilter, "%s:   SF value %x.%x\n",
1032292SN/A            __func__, sf_item.requested, sf_item.holder);
1042292SN/A
1052292SN/A    // If we are not allocating, we are done
1062292SN/A    if (!allocate)
1072292SN/A        return snoopSelected(maskToPortList(interested & ~req_port),
1082292SN/A                             lookupLatency);
1092292SN/A
1102329SN/A    if (cpkt->needsResponse()) {
1112329SN/A        if (!cpkt->memInhibitAsserted()) {
1122329SN/A            // Max one request per address per port
1132292SN/A            panic_if(sf_item.requested & req_port, "double request :( " \
1142292SN/A                     "SF value %x.%x\n", sf_item.requested, sf_item.holder);
1152292SN/A
1162329SN/A            // Mark in-flight requests to distinguish later on
1172329SN/A            sf_item.requested |= req_port;
1182292SN/A            DPRINTF(SnoopFilter, "%s:   new SF value %x.%x\n",
1192292SN/A                    __func__,  sf_item.requested, sf_item.holder);
1202292SN/A        } else {
1212292SN/A            // NOTE: The memInhibit might have been asserted by a cache closer
1222292SN/A            // to the CPU, already -> the response will not be seen by this
1232292SN/A            // filter -> we do not need to keep the in-flight request, but make
1242292SN/A            // sure that we know that that cluster has a copy
1252292SN/A            panic_if(!(sf_item.holder & req_port), "Need to hold the value!");
1262292SN/A            DPRINTF(SnoopFilter,
1272292SN/A                    "%s: not marking request. SF value %x.%x\n",
1282292SN/A                    __func__,  sf_item.requested, sf_item.holder);
1292292SN/A        }
1302329SN/A    } else { // if (!cpkt->needsResponse())
1312329SN/A        assert(cpkt->evictingBlock());
1322292SN/A        // make sure that the sender actually had the line
1332292SN/A        panic_if(!(sf_item.holder & req_port), "requester %x is not a " \
1342292SN/A                 "holder :( SF value %x.%x\n", req_port,
1352292SN/A                 sf_item.requested, sf_item.holder);
1362292SN/A        // CleanEvicts and Writebacks -> the sender and all caches above
1372292SN/A        // it may not have the line anymore.
1382292SN/A        if (!cpkt->isBlockCached()) {
1392329SN/A            sf_item.holder &= ~req_port;
1402329SN/A            DPRINTF(SnoopFilter, "%s:   new SF value %x.%x\n",
1412292SN/A                    __func__,  sf_item.requested, sf_item.holder);
1422292SN/A        }
1432292SN/A    }
1442292SN/A
1452329SN/A    return snoopSelected(maskToPortList(interested & ~req_port), lookupLatency);
1462329SN/A}
1472292SN/A
1482292SN/Avoid
1492292SN/ASnoopFilter::finishRequest(bool will_retry, const Packet* cpkt)
1502292SN/A{
1512292SN/A    if (reqLookupResult != cachedLocations.end()) {
1522292SN/A        // since we rely on the caller, do a basic check to ensure
1532292SN/A        // that finishRequest is being called following lookupRequest
1542292SN/A        assert(reqLookupResult->first == cpkt->getBlockAddr(linesize));
1552329SN/A        if (will_retry) {
1562329SN/A            // Undo any changes made in lookupRequest to the snoop filter
1572292SN/A            // entry if the request will come again. retryItem holds
1582292SN/A            // the previous value of the snoopfilter entry.
1592329SN/A            reqLookupResult->second = retryItem;
1602329SN/A
1612329SN/A            DPRINTF(SnoopFilter, "%s:   restored SF value %x.%x\n",
1622292SN/A                    __func__,  retryItem.requested, retryItem.holder);
1632292SN/A        }
1642292SN/A
1652292SN/A        eraseIfNullEntry(reqLookupResult);
1662292SN/A    }
1672292SN/A}
1682292SN/A
1692329SN/Astd::pair<SnoopFilter::SnoopList, Cycles>
1702329SN/ASnoopFilter::lookupSnoop(const Packet* cpkt)
1712329SN/A{
1722292SN/A    DPRINTF(SnoopFilter, "%s: packet addr 0x%x cmd %s\n",
1732292SN/A            __func__, cpkt->getAddr(), cpkt->cmdString());
1742292SN/A
1752292SN/A    assert(cpkt->isRequest());
1762292SN/A
1772292SN/A    // Broadcast / filter upward snoops
1782292SN/A    const bool filter_upward = true;  // @todo: Make configurable
1792292SN/A
1802292SN/A    if (!filter_upward)
1812329SN/A        return snoopAll(lookupLatency);
1822329SN/A
1832292SN/A    Addr line_addr = cpkt->getBlockAddr(linesize);
1842292SN/A    auto sf_it = cachedLocations.find(line_addr);
1852292SN/A    bool is_hit = (sf_it != cachedLocations.end());
1862292SN/A
1872329SN/A    // If the snoop filter has no entry and its an uncacheable
1882329SN/A    // request, do not create a new snoop filter entry, simply return
1892292SN/A    // a NULL portlist.
1902292SN/A    if (!is_hit && cpkt->req->isUncacheable())
1912292SN/A        return snoopDown(lookupLatency);
1922292SN/A
1932329SN/A    // If no hit in snoop filter create a new element and update iterator
1942329SN/A    if (!is_hit)
1952292SN/A        sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first;
1962292SN/A    SnoopItem& sf_item = sf_it->second;
1972292SN/A
1982292SN/A    DPRINTF(SnoopFilter, "%s:   old SF value %x.%x\n",
1992329SN/A            __func__, sf_item.requested, sf_item.holder);
2002329SN/A
2012292SN/A    SnoopMask interested = (sf_item.holder | sf_item.requested);
2022292SN/A
2032292SN/A    totSnoops++;
2042292SN/A    if (is_hit) {
2052292SN/A        // Single bit set -> value is a power of two
2062292SN/A        if (isPow2(interested))
2072292SN/A            hitSingleSnoops++;
2082292SN/A        else
2092292SN/A            hitMultiSnoops++;
2102292SN/A    }
2112292SN/A    // ReadEx and Writes require both invalidation and exlusivity, while reads
2122292SN/A    // require neither. Writebacks on the other hand require exclusivity but
2132292SN/A    // not the invalidation. Previously Writebacks did not generate upward
2142292SN/A    // snoops so this was never an aissue. Now that Writebacks generate snoops
2152292SN/A    // we need to special case for Writebacks.
2162292SN/A    assert(cpkt->cmd == MemCmd::Writeback || cpkt->req->isUncacheable() ||
2172292SN/A           (cpkt->isInvalidate() == cpkt->needsExclusive()));
2182292SN/A    if (cpkt->isInvalidate() && !sf_item.requested) {
2192292SN/A        // Early clear of the holder, if no other request is currently going on
2202292SN/A        // @todo: This should possibly be updated even though we do not filter
2212292SN/A        // upward snoops
2222292SN/A        sf_item.holder = 0;
2232292SN/A    }
2242292SN/A
2252292SN/A    eraseIfNullEntry(sf_it);
2262292SN/A    DPRINTF(SnoopFilter, "%s:   new SF value %x.%x interest: %x \n",
2272292SN/A            __func__, sf_item.requested, sf_item.holder, interested);
2282292SN/A
2292292SN/A    return snoopSelected(maskToPortList(interested), lookupLatency);
2302292SN/A}
2312292SN/A
2322292SN/Avoid
2332292SN/ASnoopFilter::updateSnoopResponse(const Packet* cpkt,
2342292SN/A                                 const SlavePort& rsp_port,
2352292SN/A                                 const SlavePort& req_port)
2362292SN/A{
2372292SN/A    DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
2382329SN/A            __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
2392292SN/A            cpkt->cmdString());
2402292SN/A
2412292SN/A    assert(cpkt->isResponse());
2422329SN/A    assert(cpkt->memInhibitAsserted());
2432329SN/A
2442329SN/A    // Ultimately we should check if the packet came from an
2452292SN/A    // allocating source, not just if the port is snooping
2462329SN/A    bool allocate = !cpkt->req->isUncacheable() && req_port.isSnooping();
2472329SN/A    if (!allocate)
2482292SN/A        return;
2492292SN/A
2502292SN/A    Addr line_addr = cpkt->getBlockAddr(linesize);
2512292SN/A    SnoopMask rsp_mask = portToMask(rsp_port);
2522292SN/A    SnoopMask req_mask = portToMask(req_port);
2532292SN/A    SnoopItem& sf_item = cachedLocations[line_addr];
2542329SN/A
2552329SN/A    DPRINTF(SnoopFilter, "%s:   old SF value %x.%x\n",
2562292SN/A            __func__,  sf_item.requested, sf_item.holder);
2572907Sktlim@umich.edu
2582907Sktlim@umich.edu    // The source should have the line
2592907Sktlim@umich.edu    panic_if(!(sf_item.holder & rsp_mask), "SF value %x.%x does not have "\
2602907Sktlim@umich.edu             "the line\n", sf_item.requested, sf_item.holder);
2612907Sktlim@umich.edu
2622907Sktlim@umich.edu    // The destination should have had a request in
2632907Sktlim@umich.edu    panic_if(!(sf_item.requested & req_mask), "SF value %x.%x missing "\
2642907Sktlim@umich.edu             "the original request\n",  sf_item.requested, sf_item.holder);
2652907Sktlim@umich.edu
2662292SN/A    // Update the residency of the cache line.
2672292SN/A    if (cpkt->needsExclusive() || !cpkt->sharedAsserted()) {
2682292SN/A        DPRINTF(SnoopFilter, "%s:  dropping %x because needs: %i shared: %i "\
2692329SN/A                "SF val: %x.%x\n", __func__,  rsp_mask,
2702329SN/A                cpkt->needsExclusive(), cpkt->sharedAsserted(),
2712292SN/A                sf_item.requested, sf_item.holder);
2722292SN/A
2732292SN/A        sf_item.holder &= ~rsp_mask;
2742669Sktlim@umich.edu        // The snoop filter does not see any ACKs from non-responding sharers
2752292SN/A        // that have been invalidated :(  So below assert would be nice, but..
2762292SN/A        //assert(sf_item.holder == 0);
2772292SN/A        sf_item.holder = 0;
2782292SN/A    }
2792292SN/A    assert(cpkt->cmd != MemCmd::Writeback);
2802669Sktlim@umich.edu    sf_item.holder |=  req_mask;
2812292SN/A    sf_item.requested &= ~req_mask;
2824329Sktlim@umich.edu    assert(sf_item.requested | sf_item.holder);
2834329Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s:   new SF value %x.%x\n",
2844329Sktlim@umich.edu            __func__, sf_item.requested, sf_item.holder);
2854329Sktlim@umich.edu}
2864329Sktlim@umich.edu
2874329Sktlim@umich.eduvoid
2882907Sktlim@umich.eduSnoopFilter::updateSnoopForward(const Packet* cpkt,
2892907Sktlim@umich.edu        const SlavePort& rsp_port, const MasterPort& req_port)
2902907Sktlim@umich.edu{
2912907Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s: packet rsp %s req %s addr 0x%x cmd %s\n",
2922907Sktlim@umich.edu            __func__, rsp_port.name(), req_port.name(), cpkt->getAddr(),
2932907Sktlim@umich.edu            cpkt->cmdString());
2942907Sktlim@umich.edu
2952907Sktlim@umich.edu    Addr line_addr = cpkt->getBlockAddr(linesize);
2962907Sktlim@umich.edu    auto sf_it = cachedLocations.find(line_addr);
2972907Sktlim@umich.edu    if (sf_it == cachedLocations.end())
2982907Sktlim@umich.edu        sf_it = cachedLocations.emplace(line_addr, SnoopItem()).first;
2992907Sktlim@umich.edu    SnoopItem& sf_item = sf_it->second;
3004329Sktlim@umich.edu    SnoopMask rsp_mask M5_VAR_USED = portToMask(rsp_port);
3012907Sktlim@umich.edu
3022907Sktlim@umich.edu    assert(cpkt->isResponse());
3033647Srdreslin@umich.edu    assert(cpkt->memInhibitAsserted());
3043647Srdreslin@umich.edu
3054192Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s:   old SF value %x.%x\n",
3064192Sktlim@umich.edu            __func__,  sf_item.requested, sf_item.holder);
3072907Sktlim@umich.edu
3082907Sktlim@umich.edu    // Remote (to this snoop filter) snoops update the filter already when they
3092907Sktlim@umich.edu    // arrive from below, because we may not see any response.
3102907Sktlim@umich.edu    if (cpkt->needsExclusive()) {
3112907Sktlim@umich.edu        // If the request to this snoop response hit an in-flight transaction,
3122907Sktlim@umich.edu        // the holder was not reset -> no assertion & do that here, now!
3132907Sktlim@umich.edu        //assert(sf_item.holder == 0);
3142907Sktlim@umich.edu        sf_item.holder = 0;
3152907Sktlim@umich.edu    }
3162907Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s:   new SF value %x.%x\n",
3172907Sktlim@umich.edu            __func__, sf_item.requested, sf_item.holder);
3182907Sktlim@umich.edu    eraseIfNullEntry(sf_it);
3192907Sktlim@umich.edu}
3203846Shsul@eecs.umich.edu
3212907Sktlim@umich.eduvoid
3222907Sktlim@umich.eduSnoopFilter::updateResponse(const Packet* cpkt, const SlavePort& slave_port)
3232907Sktlim@umich.edu{
3242907Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s: packet src %s addr 0x%x cmd %s\n",
3252907Sktlim@umich.edu            __func__, slave_port.name(), cpkt->getAddr(), cpkt->cmdString());
3262907Sktlim@umich.edu
3272907Sktlim@umich.edu    assert(cpkt->isResponse());
3282907Sktlim@umich.edu
3292907Sktlim@umich.edu    // Ultimately we should check if the packet came from an
3302907Sktlim@umich.edu    // allocating source, not just if the port is snooping
3312907Sktlim@umich.edu    bool allocate = !cpkt->req->isUncacheable() && slave_port.isSnooping();
3322907Sktlim@umich.edu    if (!allocate)
3332907Sktlim@umich.edu        return;
3344192Sktlim@umich.edu
3354192Sktlim@umich.edu    Addr line_addr = cpkt->getBlockAddr(linesize);
3364192Sktlim@umich.edu    SnoopMask slave_mask = portToMask(slave_port);
3374192Sktlim@umich.edu    SnoopItem& sf_item = cachedLocations[line_addr];
3384192Sktlim@umich.edu
3392907Sktlim@umich.edu    DPRINTF(SnoopFilter, "%s:   old SF value %x.%x\n",
3402292SN/A            __func__,  sf_item.requested, sf_item.holder);
3412292SN/A
3422292SN/A    // Make sure we have seen the actual request, too
3432292SN/A    panic_if(!(sf_item.requested & slave_mask), "SF value %x.%x missing "\
3442292SN/A             "request bit\n", sf_item.requested, sf_item.holder);
3452292SN/A
3462292SN/A    // Update the residency of the cache line. Here we assume that the
3472292SN/A    // line has been zapped in all caches that are not the responder.
3482292SN/A     if (cpkt->needsExclusive() || !cpkt->sharedAsserted())
3492292SN/A        sf_item.holder = 0;
3502292SN/A    sf_item.holder |=  slave_mask;
3512292SN/A    sf_item.requested &= ~slave_mask;
3522292SN/A    assert(sf_item.holder | sf_item.requested);
3532292SN/A    DPRINTF(SnoopFilter, "%s:   new SF value %x.%x\n",
3542292SN/A            __func__, sf_item.requested, sf_item.holder);
3552292SN/A}
3562292SN/A
3572292SN/Avoid
3582292SN/ASnoopFilter::regStats()
3592292SN/A{
3602292SN/A    totRequests
3612292SN/A        .name(name() + ".tot_requests")
3622907Sktlim@umich.edu        .desc("Total number of requests made to the snoop filter.");
3632907Sktlim@umich.edu
3642907Sktlim@umich.edu    hitSingleRequests
3652907Sktlim@umich.edu        .name(name() + ".hit_single_requests")
3662292SN/A        .desc("Number of requests hitting in the snoop filter with a single "\
3672292SN/A              "holder of the requested data.");
3682292SN/A
3692292SN/A    hitMultiRequests
3702292SN/A        .name(name() + ".hit_multi_requests")
3712669Sktlim@umich.edu        .desc("Number of requests hitting in the snoop filter with multiple "\
3722292SN/A              "(>1) holders of the requested data.");
3732669Sktlim@umich.edu
3742292SN/A    totSnoops
3752292SN/A        .name(name() + ".tot_snoops")
3762292SN/A        .desc("Total number of snoops made to the snoop filter.");
3772292SN/A
3782292SN/A    hitSingleSnoops
3792292SN/A        .name(name() + ".hit_single_snoops")
3802292SN/A        .desc("Number of snoops hitting in the snoop filter with a single "\
3812669Sktlim@umich.edu              "holder of the requested data.");
3822292SN/A
3832669Sktlim@umich.edu    hitMultiSnoops
3842292SN/A        .name(name() + ".hit_multi_snoops")
3852292SN/A        .desc("Number of snoops hitting in the snoop filter with multiple "\
3862292SN/A              "(>1) holders of the requested data.");
3872292SN/A}
3882292SN/A
389SnoopFilter *
390SnoopFilterParams::create()
391{
392    return new SnoopFilter(this);
393}
394