12381SN/A/* 214006Stiago.muck@arm.com * Copyright (c) 2011-2015, 2017, 2019 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142381SN/A * Copyright (c) 2002-2005 The Regents of The University of Michigan 152381SN/A * All rights reserved. 162381SN/A * 172381SN/A * Redistribution and use in source and binary forms, with or without 182381SN/A * modification, are permitted provided that the following conditions are 192381SN/A * met: redistributions of source code must retain the above copyright 202381SN/A * notice, this list of conditions and the following disclaimer; 212381SN/A * redistributions in binary form must reproduce the above copyright 222381SN/A * notice, this list of conditions and the following disclaimer in the 232381SN/A * documentation and/or other materials provided with the distribution; 242381SN/A * neither the name of the copyright holders nor the names of its 252381SN/A * contributors may be used to endorse or promote products derived from 262381SN/A * this software without specific prior written permission. 272381SN/A * 282381SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292381SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302381SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312381SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322381SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332381SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342381SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352381SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362381SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372381SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382381SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ron Dreslinski 412772SN/A * Ali Saidi 428715SN/A * Andreas Hansson 438922SN/A * William Wang 442381SN/A */ 452381SN/A 462381SN/A/** 472982SN/A * @file 4810405Sandreas.hansson@arm.com * Declaration of a coherent crossbar. 492381SN/A */ 502381SN/A 5110405Sandreas.hansson@arm.com#ifndef __MEM_COHERENT_XBAR_HH__ 5210405Sandreas.hansson@arm.com#define __MEM_COHERENT_XBAR_HH__ 532381SN/A 5412351Snikos.nikoleris@arm.com#include <unordered_map> 5511859Sandreas.hansson@arm.com#include <unordered_set> 5611859Sandreas.hansson@arm.com 5710402SN/A#include "mem/snoop_filter.hh" 5810405Sandreas.hansson@arm.com#include "mem/xbar.hh" 5910405Sandreas.hansson@arm.com#include "params/CoherentXBar.hh" 602381SN/A 619036SN/A/** 6210405Sandreas.hansson@arm.com * A coherent crossbar connects a number of (potentially) snooping 6310405Sandreas.hansson@arm.com * masters and slaves, and routes the request and response packets 6410405Sandreas.hansson@arm.com * based on the address, and also forwards all requests to the 6510405Sandreas.hansson@arm.com * snoopers and deals with the snoop responses. 669036SN/A * 6710405Sandreas.hansson@arm.com * The coherent crossbar can be used as a template for modelling QPI, 6810405Sandreas.hansson@arm.com * HyperTransport, ACE and coherent OCP buses, and is typically used 6910405Sandreas.hansson@arm.com * for the L1-to-L2 buses and as the main system interconnect. @sa 7010405Sandreas.hansson@arm.com * \ref gem5MemorySystem "gem5 Memory System" 719036SN/A */ 7210405Sandreas.hansson@arm.comclass CoherentXBar : public BaseXBar 732381SN/A{ 749031SN/A 759036SN/A protected: 769036SN/A 778922SN/A /** 7810405Sandreas.hansson@arm.com * Declare the layers of this crossbar, one vector for requests, 7910405Sandreas.hansson@arm.com * one for responses, and one for snoop responses 809092SN/A */ 819715SN/A std::vector<ReqLayer*> reqLayers; 829715SN/A std::vector<RespLayer*> respLayers; 8310713Sandreas.hansson@arm.com std::vector<SnoopRespLayer*> snoopLayers; 849092SN/A 859092SN/A /** 8610405Sandreas.hansson@arm.com * Declaration of the coherent crossbar slave port type, one will 8710405Sandreas.hansson@arm.com * be instantiated for each of the master ports connecting to the 8810405Sandreas.hansson@arm.com * crossbar. 898922SN/A */ 9010888Sandreas.hansson@arm.com class CoherentXBarSlavePort : public QueuedSlavePort 912381SN/A { 929036SN/A 938922SN/A private: 949036SN/A 9510405Sandreas.hansson@arm.com /** A reference to the crossbar to which this port belongs. */ 9610405Sandreas.hansson@arm.com CoherentXBar &xbar; 972381SN/A 9810888Sandreas.hansson@arm.com /** A normal packet queue used to store responses. */ 9910888Sandreas.hansson@arm.com RespPacketQueue queue; 10010888Sandreas.hansson@arm.com 1012381SN/A public: 1022381SN/A 10310405Sandreas.hansson@arm.com CoherentXBarSlavePort(const std::string &_name, 10410405Sandreas.hansson@arm.com CoherentXBar &_xbar, PortID _id) 10510888Sandreas.hansson@arm.com : QueuedSlavePort(_name, &_xbar, queue, _id), xbar(_xbar), 10610888Sandreas.hansson@arm.com queue(_xbar, *this) 1078922SN/A { } 1088922SN/A 1098922SN/A protected: 1108922SN/A 11113808Sgabeblack@google.com bool 11213808Sgabeblack@google.com recvTimingReq(PacketPtr pkt) override 11313808Sgabeblack@google.com { 11413808Sgabeblack@google.com return xbar.recvTimingReq(pkt, id); 11513808Sgabeblack@google.com } 1168922SN/A 11713808Sgabeblack@google.com bool 11813808Sgabeblack@google.com recvTimingSnoopResp(PacketPtr pkt) override 11913808Sgabeblack@google.com { 12013808Sgabeblack@google.com return xbar.recvTimingSnoopResp(pkt, id); 12113808Sgabeblack@google.com } 1228948SN/A 12313808Sgabeblack@google.com Tick 12413808Sgabeblack@google.com recvAtomic(PacketPtr pkt) override 12513808Sgabeblack@google.com { 12613847Sgabeblack@google.com return xbar.recvAtomicBackdoor(pkt, id); 12713847Sgabeblack@google.com } 12813847Sgabeblack@google.com 12913847Sgabeblack@google.com Tick 13013847Sgabeblack@google.com recvAtomicBackdoor(PacketPtr pkt, MemBackdoorPtr &backdoor) override 13113847Sgabeblack@google.com { 13213847Sgabeblack@google.com return xbar.recvAtomicBackdoor(pkt, id, &backdoor); 13313808Sgabeblack@google.com } 1348922SN/A 13513808Sgabeblack@google.com void 13613808Sgabeblack@google.com recvFunctional(PacketPtr pkt) override 13713808Sgabeblack@google.com { 13813808Sgabeblack@google.com xbar.recvFunctional(pkt, id); 13913808Sgabeblack@google.com } 1408922SN/A 14113808Sgabeblack@google.com AddrRangeList 14213808Sgabeblack@google.com getAddrRanges() const override 14313808Sgabeblack@google.com { 14413808Sgabeblack@google.com return xbar.getAddrRanges(); 14513808Sgabeblack@google.com } 1469036SN/A 1479036SN/A }; 1489036SN/A 1499036SN/A /** 15010405Sandreas.hansson@arm.com * Declaration of the coherent crossbar master port type, one will be 1519036SN/A * instantiated for each of the slave interfaces connecting to the 15210405Sandreas.hansson@arm.com * crossbar. 1539036SN/A */ 15410405Sandreas.hansson@arm.com class CoherentXBarMasterPort : public MasterPort 1559036SN/A { 1569036SN/A private: 15710405Sandreas.hansson@arm.com /** A reference to the crossbar to which this port belongs. */ 15810405Sandreas.hansson@arm.com CoherentXBar &xbar; 1599036SN/A 1609036SN/A public: 1619036SN/A 16210405Sandreas.hansson@arm.com CoherentXBarMasterPort(const std::string &_name, 16310405Sandreas.hansson@arm.com CoherentXBar &_xbar, PortID _id) 16410405Sandreas.hansson@arm.com : MasterPort(_name, &_xbar, _id), xbar(_xbar) 1659036SN/A { } 1669036SN/A 1679036SN/A protected: 1689036SN/A 1699036SN/A /** 1709036SN/A * Determine if this port should be considered a snooper. For 17110405Sandreas.hansson@arm.com * a coherent crossbar master port this is always true. 1729036SN/A * 1739036SN/A * @return a boolean that is true if this port is snooping 1749036SN/A */ 17513808Sgabeblack@google.com bool isSnooping() const override { return true; } 1769036SN/A 17713808Sgabeblack@google.com bool 17813808Sgabeblack@google.com recvTimingResp(PacketPtr pkt) override 17913808Sgabeblack@google.com { 18013808Sgabeblack@google.com return xbar.recvTimingResp(pkt, id); 18113808Sgabeblack@google.com } 1829036SN/A 18313808Sgabeblack@google.com void 18413808Sgabeblack@google.com recvTimingSnoopReq(PacketPtr pkt) override 18513808Sgabeblack@google.com { 18613808Sgabeblack@google.com return xbar.recvTimingSnoopReq(pkt, id); 18713808Sgabeblack@google.com } 1889036SN/A 18913808Sgabeblack@google.com Tick 19013808Sgabeblack@google.com recvAtomicSnoop(PacketPtr pkt) override 19113808Sgabeblack@google.com { 19213808Sgabeblack@google.com return xbar.recvAtomicSnoop(pkt, id); 19313808Sgabeblack@google.com } 1949036SN/A 19513808Sgabeblack@google.com void 19613808Sgabeblack@google.com recvFunctionalSnoop(PacketPtr pkt) override 19713808Sgabeblack@google.com { 19813808Sgabeblack@google.com xbar.recvFunctionalSnoop(pkt, id); 19913808Sgabeblack@google.com } 2009036SN/A 20113808Sgabeblack@google.com void recvRangeChange() override { xbar.recvRangeChange(id); } 20213808Sgabeblack@google.com void recvReqRetry() override { xbar.recvReqRetry(id); } 2038922SN/A 2048922SN/A }; 2058922SN/A 2069716SN/A /** 2079716SN/A * Internal class to bridge between an incoming snoop response 2089716SN/A * from a slave port and forwarding it through an outgoing slave 2099716SN/A * port. It is effectively a dangling master port. 2109716SN/A */ 2119716SN/A class SnoopRespPort : public MasterPort 2129716SN/A { 2139716SN/A 2149716SN/A private: 2159716SN/A 2169716SN/A /** The port which we mirror internally. */ 21710888Sandreas.hansson@arm.com QueuedSlavePort& slavePort; 2189716SN/A 2199716SN/A public: 2209716SN/A 2219716SN/A /** 2229716SN/A * Create a snoop response port that mirrors a given slave port. 2239716SN/A */ 22410888Sandreas.hansson@arm.com SnoopRespPort(QueuedSlavePort& slave_port, CoherentXBar& _xbar) : 22510405Sandreas.hansson@arm.com MasterPort(slave_port.name() + ".snoopRespPort", &_xbar), 2269778SN/A slavePort(slave_port) { } 2279716SN/A 2289716SN/A /** 2299716SN/A * Override the sending of retries and pass them on through 2309716SN/A * the mirrored slave port. 2319716SN/A */ 23213808Sgabeblack@google.com void 23313808Sgabeblack@google.com sendRetryResp() override 23413808Sgabeblack@google.com { 23510713Sandreas.hansson@arm.com // forward it as a snoop response retry 23610713Sandreas.hansson@arm.com slavePort.sendRetrySnoopResp(); 2379716SN/A } 2389716SN/A 23913808Sgabeblack@google.com void 24013808Sgabeblack@google.com recvReqRetry() override 24113808Sgabeblack@google.com { 24213808Sgabeblack@google.com panic("SnoopRespPort should never see retry"); 24313808Sgabeblack@google.com } 2449716SN/A 24513808Sgabeblack@google.com bool 24613808Sgabeblack@google.com recvTimingResp(PacketPtr pkt) override 2479716SN/A { 24813808Sgabeblack@google.com panic("SnoopRespPort should never see timing response"); 2499716SN/A } 2509716SN/A 2519716SN/A }; 2529716SN/A 2539716SN/A std::vector<SnoopRespPort*> snoopRespPorts; 2549716SN/A 25510888Sandreas.hansson@arm.com std::vector<QueuedSlavePort*> snoopPorts; 2564475SN/A 2578948SN/A /** 25810656Sandreas.hansson@arm.com * Store the outstanding requests that we are expecting snoop 25910656Sandreas.hansson@arm.com * responses from so we can determine which snoop responses we 26010656Sandreas.hansson@arm.com * generated and which ones were merely forwarded. 2618948SN/A */ 26211168Sandreas.hansson@arm.com std::unordered_set<RequestPtr> outstandingSnoop; 2638948SN/A 2649524SN/A /** 26512351Snikos.nikoleris@arm.com * Store the outstanding cache maintenance that we are expecting 26612351Snikos.nikoleris@arm.com * snoop responses from so we can determine when we received all 26712351Snikos.nikoleris@arm.com * snoop responses and if any of the agents satisfied the request. 26812351Snikos.nikoleris@arm.com */ 26912351Snikos.nikoleris@arm.com std::unordered_map<PacketId, PacketPtr> outstandingCMO; 27012351Snikos.nikoleris@arm.com 27112351Snikos.nikoleris@arm.com /** 2729524SN/A * Keep a pointer to the system to be allow to querying memory system 2739524SN/A * properties. 2749524SN/A */ 2759524SN/A System *system; 2769524SN/A 27710402SN/A /** A snoop filter that tracks cache line residency and can restrict the 27810402SN/A * broadcast needed for probes. NULL denotes an absent filter. */ 27910402SN/A SnoopFilter *snoopFilter; 28010402SN/A 28114006Stiago.muck@arm.com /** Cycles of snoop response latency.*/ 28210719SMarco.Balboni@ARM.com const Cycles snoopResponseLatency; 28314006Stiago.muck@arm.com 28414006Stiago.muck@arm.com /** Maximum number of outstading snoops sanity check*/ 28514006Stiago.muck@arm.com const unsigned int maxOutstandingSnoopCheck; 28614006Stiago.muck@arm.com 28714006Stiago.muck@arm.com /** Maximum routing table size sanity check*/ 28814006Stiago.muck@arm.com const unsigned int maxRoutingTableSizeCheck; 28914006Stiago.muck@arm.com 29014006Stiago.muck@arm.com /** Is this crossbar the point of coherency? **/ 29111334Sandreas.hansson@arm.com const bool pointOfCoherency; 29214006Stiago.muck@arm.com 29314006Stiago.muck@arm.com /** Is this crossbar the point of unification? **/ 29412341Snikos.nikoleris@arm.com const bool pointOfUnification; 29512341Snikos.nikoleris@arm.com 29610883Sali.jafri@arm.com /** 29711190Sandreas.hansson@arm.com * Upstream caches need this packet until true is returned, so 29811190Sandreas.hansson@arm.com * hold it for deletion until a subsequent call 29910883Sali.jafri@arm.com */ 30011190Sandreas.hansson@arm.com std::unique_ptr<Packet> pendingDelete; 30110883Sali.jafri@arm.com 3029945SN/A bool recvTimingReq(PacketPtr pkt, PortID slave_port_id); 3039945SN/A bool recvTimingResp(PacketPtr pkt, PortID master_port_id); 3049945SN/A void recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id); 3059945SN/A bool recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id); 30610713Sandreas.hansson@arm.com void recvReqRetry(PortID master_port_id); 3079092SN/A 3088948SN/A /** 3098948SN/A * Forward a timing packet to our snoopers, potentially excluding 3108948SN/A * one of the connected coherent masters to avoid sending a packet 3118948SN/A * back to where it came from. 3128948SN/A * 3138948SN/A * @param pkt Packet to forward 3148948SN/A * @param exclude_slave_port_id Id of slave port to exclude 3158948SN/A */ 31613808Sgabeblack@google.com void 31713808Sgabeblack@google.com forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id) 31813808Sgabeblack@google.com { 31910402SN/A forwardTiming(pkt, exclude_slave_port_id, snoopPorts); 32010402SN/A } 32110402SN/A 32210402SN/A /** 32310402SN/A * Forward a timing packet to a selected list of snoopers, potentially 32410402SN/A * excluding one of the connected coherent masters to avoid sending a packet 32510402SN/A * back to where it came from. 32610402SN/A * 32710402SN/A * @param pkt Packet to forward 32810402SN/A * @param exclude_slave_port_id Id of slave port to exclude 32910402SN/A * @param dests Vector of destination ports for the forwarded pkt 33010402SN/A */ 33110402SN/A void forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 33210888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests); 3338948SN/A 33413847Sgabeblack@google.com Tick recvAtomicBackdoor(PacketPtr pkt, PortID slave_port_id, 33513847Sgabeblack@google.com MemBackdoorPtr *backdoor=nullptr); 3369032SN/A Tick recvAtomicSnoop(PacketPtr pkt, PortID master_port_id); 3378948SN/A 3388948SN/A /** 3398948SN/A * Forward an atomic packet to our snoopers, potentially excluding 3408948SN/A * one of the connected coherent masters to avoid sending a packet 3418948SN/A * back to where it came from. 3428948SN/A * 3438948SN/A * @param pkt Packet to forward 3448948SN/A * @param exclude_slave_port_id Id of slave port to exclude 3458948SN/A * 3468948SN/A * @return a pair containing the snoop response and snoop latency 3478948SN/A */ 34813808Sgabeblack@google.com std::pair<MemCmd, Tick> 34913808Sgabeblack@google.com forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id) 35010402SN/A { 35110888Sandreas.hansson@arm.com return forwardAtomic(pkt, exclude_slave_port_id, InvalidPortID, 35210888Sandreas.hansson@arm.com snoopPorts); 35310402SN/A } 35410402SN/A 35510402SN/A /** 35610402SN/A * Forward an atomic packet to a selected list of snoopers, potentially 35710402SN/A * excluding one of the connected coherent masters to avoid sending a packet 35810402SN/A * back to where it came from. 35910402SN/A * 36010402SN/A * @param pkt Packet to forward 36110402SN/A * @param exclude_slave_port_id Id of slave port to exclude 36210402SN/A * @param source_master_port_id Id of the master port for snoops from below 36310402SN/A * @param dests Vector of destination ports for the forwarded pkt 36410402SN/A * 36510402SN/A * @return a pair containing the snoop response and snoop latency 36610402SN/A */ 36710402SN/A std::pair<MemCmd, Tick> forwardAtomic(PacketPtr pkt, 36810402SN/A PortID exclude_slave_port_id, 36910402SN/A PortID source_master_port_id, 37010888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& 37110888Sandreas.hansson@arm.com dests); 3728948SN/A 37310405Sandreas.hansson@arm.com /** Function called by the port when the crossbar is recieving a Functional 3744475SN/A transaction.*/ 3759032SN/A void recvFunctional(PacketPtr pkt, PortID slave_port_id); 3764475SN/A 37710405Sandreas.hansson@arm.com /** Function called by the port when the crossbar is recieving a functional 3788948SN/A snoop transaction.*/ 3799032SN/A void recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id); 3808948SN/A 3818948SN/A /** 3828948SN/A * Forward a functional packet to our snoopers, potentially 3838948SN/A * excluding one of the connected coherent masters to avoid 3848948SN/A * sending a packet back to where it came from. 3858948SN/A * 3868948SN/A * @param pkt Packet to forward 3878948SN/A * @param exclude_slave_port_id Id of slave port to exclude 3888948SN/A */ 3899031SN/A void forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id); 3908948SN/A 39111334Sandreas.hansson@arm.com /** 39211334Sandreas.hansson@arm.com * Determine if the crossbar should sink the packet, as opposed to 39311334Sandreas.hansson@arm.com * forwarding it, or responding. 39411334Sandreas.hansson@arm.com */ 39511334Sandreas.hansson@arm.com bool sinkPacket(const PacketPtr pkt) const; 39611334Sandreas.hansson@arm.com 39712346Snikos.nikoleris@arm.com /** 39812346Snikos.nikoleris@arm.com * Determine if the crossbar should forward the packet, as opposed to 39912346Snikos.nikoleris@arm.com * responding to it. 40012346Snikos.nikoleris@arm.com */ 40112346Snikos.nikoleris@arm.com bool forwardPacket(const PacketPtr pkt); 40212346Snikos.nikoleris@arm.com 40312346Snikos.nikoleris@arm.com /** 40412346Snikos.nikoleris@arm.com * Determine if the packet's destination is the memory below 40512346Snikos.nikoleris@arm.com * 40612346Snikos.nikoleris@arm.com * The memory below is the destination for a cache mainteance 40712346Snikos.nikoleris@arm.com * operation to the Point of Coherence/Unification if this is the 40812346Snikos.nikoleris@arm.com * Point of Coherence/Unification. 40912346Snikos.nikoleris@arm.com * 41012346Snikos.nikoleris@arm.com * @param pkt The processed packet 41112346Snikos.nikoleris@arm.com * 41212346Snikos.nikoleris@arm.com * @return Whether the memory below is the destination for the packet 41312346Snikos.nikoleris@arm.com */ 41413808Sgabeblack@google.com bool 41513808Sgabeblack@google.com isDestination(const PacketPtr pkt) const 41612346Snikos.nikoleris@arm.com { 41712346Snikos.nikoleris@arm.com return (pkt->req->isToPOC() && pointOfCoherency) || 41812346Snikos.nikoleris@arm.com (pkt->req->isToPOU() && pointOfUnification); 41912346Snikos.nikoleris@arm.com } 42012346Snikos.nikoleris@arm.com 42110405Sandreas.hansson@arm.com Stats::Scalar snoops; 42211564Sdavid.guillen@arm.com Stats::Scalar snoopTraffic; 42310401SN/A Stats::Distribution snoopFanout; 4249712SN/A 4252381SN/A public: 4262381SN/A 4279036SN/A virtual void init(); 4282568SN/A 42910405Sandreas.hansson@arm.com CoherentXBar(const CoherentXBarParams *p); 4309092SN/A 43110405Sandreas.hansson@arm.com virtual ~CoherentXBar(); 4329715SN/A 4339712SN/A virtual void regStats(); 4342381SN/A}; 4352381SN/A 43610405Sandreas.hansson@arm.com#endif //__MEM_COHERENT_XBAR_HH__ 437