coherent_xbar.cc revision 11284
17404SAli.Saidi@ARM.com/* 210717Sandreas.hansson@arm.com * Copyright (c) 2011-2015 ARM Limited 37404SAli.Saidi@ARM.com * All rights reserved 47404SAli.Saidi@ARM.com * 57404SAli.Saidi@ARM.com * The license below extends only to copyright in the software and shall 67404SAli.Saidi@ARM.com * not be construed as granting a license to any other intellectual 77404SAli.Saidi@ARM.com * property including but not limited to intellectual property relating 87404SAli.Saidi@ARM.com * to a hardware implementation of the functionality of the software 97404SAli.Saidi@ARM.com * licensed hereunder. You may use the software subject to the license 107404SAli.Saidi@ARM.com * terms below provided that you ensure that this notice is replicated 117404SAli.Saidi@ARM.com * unmodified and in its entirety in all distributions of the software, 127404SAli.Saidi@ARM.com * modified or unmodified, in source code or in binary form. 137404SAli.Saidi@ARM.com * 147404SAli.Saidi@ARM.com * Copyright (c) 2006 The Regents of The University of Michigan 157404SAli.Saidi@ARM.com * All rights reserved. 167404SAli.Saidi@ARM.com * 177404SAli.Saidi@ARM.com * Redistribution and use in source and binary forms, with or without 187404SAli.Saidi@ARM.com * modification, are permitted provided that the following conditions are 197404SAli.Saidi@ARM.com * met: redistributions of source code must retain the above copyright 207404SAli.Saidi@ARM.com * notice, this list of conditions and the following disclaimer; 217404SAli.Saidi@ARM.com * redistributions in binary form must reproduce the above copyright 227404SAli.Saidi@ARM.com * notice, this list of conditions and the following disclaimer in the 237404SAli.Saidi@ARM.com * documentation and/or other materials provided with the distribution; 247404SAli.Saidi@ARM.com * neither the name of the copyright holders nor the names of its 257404SAli.Saidi@ARM.com * contributors may be used to endorse or promote products derived from 267404SAli.Saidi@ARM.com * this software without specific prior written permission. 277404SAli.Saidi@ARM.com * 287404SAli.Saidi@ARM.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 297404SAli.Saidi@ARM.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 307404SAli.Saidi@ARM.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 317404SAli.Saidi@ARM.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 327404SAli.Saidi@ARM.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 337404SAli.Saidi@ARM.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 347404SAli.Saidi@ARM.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 357404SAli.Saidi@ARM.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 367404SAli.Saidi@ARM.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 377404SAli.Saidi@ARM.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3810037SARM gem5 Developers * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 397404SAli.Saidi@ARM.com * 4010873Sandreas.sandberg@arm.com * Authors: Ali Saidi 417404SAli.Saidi@ARM.com * Andreas Hansson 4210474Sandreas.hansson@arm.com * William Wang 4310474Sandreas.hansson@arm.com */ 447404SAli.Saidi@ARM.com 4510037SARM gem5 Developers/** 4610037SARM gem5 Developers * @file 477404SAli.Saidi@ARM.com * Definition of a crossbar object. 487728SAli.Saidi@ARM.com */ 497404SAli.Saidi@ARM.com 508245Snate@binkert.org#include "base/misc.hh" 519152Satgutier@umich.edu#include "base/trace.hh" 528245Snate@binkert.org#include "debug/AddrRanges.hh" 538245Snate@binkert.org#include "debug/CoherentXBar.hh" 5410873Sandreas.sandberg@arm.com#include "mem/coherent_xbar.hh" 557748SAli.Saidi@ARM.com#include "sim/system.hh" 567404SAli.Saidi@ARM.com 577404SAli.Saidi@ARM.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 587404SAli.Saidi@ARM.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter), 597404SAli.Saidi@ARM.com snoopResponseLatency(p->snoop_response_latency) 6010913Sandreas.sandberg@arm.com{ 6110717Sandreas.hansson@arm.com // create the ports based on the size of the master and slave 6210717Sandreas.hansson@arm.com // vector ports, and the presence of the default port, the ports 6310717Sandreas.hansson@arm.com // are enumerated starting from zero 649258SAli.Saidi@ARM.com for (int i = 0; i < p->port_master_connection_count; ++i) { 6510621SCurtis.Dunham@arm.com std::string portName = csprintf("%s.master[%d]", name(), i); 6610621SCurtis.Dunham@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 6710037SARM gem5 Developers masterPorts.push_back(bp); 6810037SARM gem5 Developers reqLayers.push_back(new ReqLayer(*bp, *this, 6910037SARM gem5 Developers csprintf(".reqLayer%d", i))); 7010037SARM gem5 Developers snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 717439Sdam.sunwoo@arm.com csprintf(".snoopLayer%d", i))); 727576SAli.Saidi@ARM.com } 7310037SARM gem5 Developers 7410037SARM gem5 Developers // see if we have a default slave device connected and if so add 7510037SARM gem5 Developers // our corresponding master port 7610717Sandreas.hansson@arm.com if (p->port_default_connection_count) { 7710037SARM gem5 Developers defaultPortID = masterPorts.size(); 7810037SARM gem5 Developers std::string portName = name() + ".default"; 7910037SARM gem5 Developers MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 8010037SARM gem5 Developers defaultPortID); 8110037SARM gem5 Developers masterPorts.push_back(bp); 8210037SARM gem5 Developers reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 8310037SARM gem5 Developers defaultPortID))); 8410037SARM gem5 Developers snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 8510037SARM gem5 Developers csprintf(".snoopLayer%d", 8610037SARM gem5 Developers defaultPortID))); 8710037SARM gem5 Developers } 8810037SARM gem5 Developers 897439Sdam.sunwoo@arm.com // create the slave ports, once again starting at zero 907404SAli.Saidi@ARM.com for (int i = 0; i < p->port_slave_connection_count; ++i) { 917404SAli.Saidi@ARM.com std::string portName = csprintf("%s.slave[%d]", name(), i); 927404SAli.Saidi@ARM.com QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 937404SAli.Saidi@ARM.com slavePorts.push_back(bp); 947404SAli.Saidi@ARM.com respLayers.push_back(new RespLayer(*bp, *this, 957404SAli.Saidi@ARM.com csprintf(".respLayer%d", i))); 9610717Sandreas.hansson@arm.com snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 9710717Sandreas.hansson@arm.com } 9810717Sandreas.hansson@arm.com 9910717Sandreas.hansson@arm.com clearPortCache(); 10010717Sandreas.hansson@arm.com} 10110717Sandreas.hansson@arm.com 10210717Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 10310717Sandreas.hansson@arm.com{ 10410717Sandreas.hansson@arm.com for (auto l: reqLayers) 10510717Sandreas.hansson@arm.com delete l; 10610717Sandreas.hansson@arm.com for (auto l: respLayers) 10710717Sandreas.hansson@arm.com delete l; 10810717Sandreas.hansson@arm.com for (auto l: snoopLayers) 10910717Sandreas.hansson@arm.com delete l; 11010717Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11110717Sandreas.hansson@arm.com delete p; 11210717Sandreas.hansson@arm.com} 11310717Sandreas.hansson@arm.com 11410717Sandreas.hansson@arm.comvoid 11510717Sandreas.hansson@arm.comCoherentXBar::init() 11610717Sandreas.hansson@arm.com{ 11710717Sandreas.hansson@arm.com // the base class is responsible for determining the block size 11810717Sandreas.hansson@arm.com BaseXBar::init(); 11910717Sandreas.hansson@arm.com 12010717Sandreas.hansson@arm.com // iterate over our slave ports and determine which of our 12110717Sandreas.hansson@arm.com // neighbouring master ports are snooping and add them as snoopers 12210717Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 12310717Sandreas.hansson@arm.com // check if the connected master port is snooping 12410717Sandreas.hansson@arm.com if (p->isSnooping()) { 12510537Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12610537Sandreas.hansson@arm.com p->getMasterPort().name()); 12710537Sandreas.hansson@arm.com snoopPorts.push_back(p); 12810537Sandreas.hansson@arm.com } 12910537Sandreas.hansson@arm.com } 13010537Sandreas.hansson@arm.com 13110537Sandreas.hansson@arm.com if (snoopPorts.empty()) 13210537Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 13310537Sandreas.hansson@arm.com 13410037SARM gem5 Developers // inform the snoop filter about the slave ports so it can create 13510037SARM gem5 Developers // its own internal representation 13610037SARM gem5 Developers if (snoopFilter) 1379152Satgutier@umich.edu snoopFilter->setSlavePorts(slavePorts); 1389152Satgutier@umich.edu} 1399152Satgutier@umich.edu 14010913Sandreas.sandberg@arm.combool 14110913Sandreas.sandberg@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1429152Satgutier@umich.edu{ 14310913Sandreas.sandberg@arm.com // determine the source port based on the id 1449152Satgutier@umich.edu SlavePort *src_port = slavePorts[slave_port_id]; 14510913Sandreas.sandberg@arm.com 1469152Satgutier@umich.edu // remember if the packet is an express snoop 1479152Satgutier@umich.edu bool is_express_snoop = pkt->isExpressSnoop(); 1489152Satgutier@umich.edu bool cache_responding = pkt->cacheResponding(); 14910913Sandreas.sandberg@arm.com // for normal requests, going downstream, the express snoop flag 15010913Sandreas.sandberg@arm.com // and the cache responding flag should always be the same 1517404SAli.Saidi@ARM.com assert(is_express_snoop == cache_responding); 15210037SARM gem5 Developers 1539152Satgutier@umich.edu // determine the destination based on the address 15410037SARM gem5 Developers PortID master_port_id = findPort(pkt->getAddr()); 15510037SARM gem5 Developers 15610037SARM gem5 Developers // test if the crossbar should be considered occupied for the current 15710037SARM gem5 Developers // port, and exclude express snoops from the check 15810037SARM gem5 Developers if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 15910037SARM gem5 Developers DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 16010037SARM gem5 Developers src_port->name(), pkt->cmdString(), pkt->getAddr()); 16110037SARM gem5 Developers return false; 1629152Satgutier@umich.edu } 16310913Sandreas.sandberg@arm.com 16410037SARM gem5 Developers DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 16510037SARM gem5 Developers src_port->name(), pkt->cmdString(), is_express_snoop, 16610913Sandreas.sandberg@arm.com pkt->getAddr()); 1677733SAli.Saidi@ARM.com 1687404SAli.Saidi@ARM.com // store size and command as they might be modified when 1697404SAli.Saidi@ARM.com // forwarding the packet 1707748SAli.Saidi@ARM.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1719342SAndreas.Sandberg@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 1727748SAli.Saidi@ARM.com 1739524SAndreas.Sandberg@ARM.com // store the old header delay so we can restore it if needed 1749152Satgutier@umich.edu Tick old_header_delay = pkt->headerDelay; 1759152Satgutier@umich.edu 17610621SCurtis.Dunham@arm.com // a request sees the frontend and forward latency 1777748SAli.Saidi@ARM.com Tick xbar_delay = (frontendLatency + forwardLatency) * clockPeriod(); 1787748SAli.Saidi@ARM.com 1797748SAli.Saidi@ARM.com // set the packet header and payload delay 1807404SAli.Saidi@ARM.com calcPacketTiming(pkt, xbar_delay); 18110037SARM gem5 Developers 18210037SARM gem5 Developers // determine how long to be crossbar layer is busy 18310037SARM gem5 Developers Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 18410037SARM gem5 Developers 1857404SAli.Saidi@ARM.com if (!system->bypassCaches()) { 1868733Sgeoffrey.blake@arm.com assert(pkt->snoopDelay == 0); 18710621SCurtis.Dunham@arm.com 18810621SCurtis.Dunham@arm.com // the packet is a memory-mapped request and should be 18910109SGeoffrey.Blake@arm.com // broadcasted to our snoopers but the source 19010037SARM gem5 Developers if (snoopFilter) { 19110109SGeoffrey.Blake@arm.com // check with the snoop filter where to forward this packet 1927439Sdam.sunwoo@arm.com auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 1937439Sdam.sunwoo@arm.com // the time required by a packet to be delivered through 1947439Sdam.sunwoo@arm.com // the xbar has to be charged also with to lookup latency 1957439Sdam.sunwoo@arm.com // of the snoop filter 1967404SAli.Saidi@ARM.com pkt->headerDelay += sf_res.second * clockPeriod(); 1977439Sdam.sunwoo@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 1987439Sdam.sunwoo@arm.com " SF size: %i lat: %i\n", src_port->name(), 19910109SGeoffrey.Blake@arm.com pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 20010109SGeoffrey.Blake@arm.com sf_res.second); 20110109SGeoffrey.Blake@arm.com 20210109SGeoffrey.Blake@arm.com if (pkt->isEviction()) { 20310109SGeoffrey.Blake@arm.com // for block-evicting packets, i.e. writebacks and 20410109SGeoffrey.Blake@arm.com // clean evictions, there is no need to snoop up, as 20510109SGeoffrey.Blake@arm.com // all we do is determine if the block is cached or 20610109SGeoffrey.Blake@arm.com // not, instead just set it here based on the snoop 2078202SAli.Saidi@ARM.com // filter result 2088202SAli.Saidi@ARM.com if (!sf_res.first.empty()) 2098202SAli.Saidi@ARM.com pkt->setBlockCached(); 2108202SAli.Saidi@ARM.com } else { 2118202SAli.Saidi@ARM.com forwardTiming(pkt, slave_port_id, sf_res.first); 2128202SAli.Saidi@ARM.com } 2138202SAli.Saidi@ARM.com } else { 21410037SARM gem5 Developers forwardTiming(pkt, slave_port_id); 21510621SCurtis.Dunham@arm.com } 21610474Sandreas.hansson@arm.com 2178202SAli.Saidi@ARM.com // add the snoop delay to our header delay, and then reset it 2187439Sdam.sunwoo@arm.com pkt->headerDelay += pkt->snoopDelay; 21910621SCurtis.Dunham@arm.com pkt->snoopDelay = 0; 2207439Sdam.sunwoo@arm.com } 22110621SCurtis.Dunham@arm.com 2227439Sdam.sunwoo@arm.com // forwardTiming snooped into peer caches of the sender, and if 22310037SARM gem5 Developers // this is a clean evict or clean writeback, but the packet is 22410037SARM gem5 Developers // found in a cache, do not forward it 2257439Sdam.sunwoo@arm.com if ((pkt->cmd == MemCmd::CleanEvict || 2267439Sdam.sunwoo@arm.com pkt->cmd == MemCmd::WritebackClean) && pkt->isBlockCached()) { 2277439Sdam.sunwoo@arm.com DPRINTF(CoherentXBar, "Clean evict/writeback %#llx still cached, " 22810037SARM gem5 Developers "not forwarding\n", pkt->getAddr()); 22910037SARM gem5 Developers 23010037SARM gem5 Developers // update the layer state and schedule an idle event 2317439Sdam.sunwoo@arm.com reqLayers[master_port_id]->succeededTiming(packetFinishTime); 2328733Sgeoffrey.blake@arm.com 2337439Sdam.sunwoo@arm.com // queue the packet for deletion 23410037SARM gem5 Developers pendingDelete.reset(pkt); 23510037SARM gem5 Developers 23610037SARM gem5 Developers return true; 2377404SAli.Saidi@ARM.com } 2387436Sdam.sunwoo@arm.com 2397436Sdam.sunwoo@arm.com // remember if the packet will generate a snoop response by 24010037SARM gem5 Developers // checking if a cache set the cacheResponding flag during the 24110037SARM gem5 Developers // snooping above 24210037SARM gem5 Developers const bool expect_snoop_resp = !cache_responding && pkt->cacheResponding(); 24310037SARM gem5 Developers const bool expect_response = pkt->needsResponse() && 24410037SARM gem5 Developers !pkt->cacheResponding(); 24510037SARM gem5 Developers 24610037SARM gem5 Developers // since it is a normal request, attempt to send the packet 24710037SARM gem5 Developers bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 24810037SARM gem5 Developers 24910037SARM gem5 Developers if (snoopFilter && !system->bypassCaches()) { 25010037SARM gem5 Developers // Let the snoop filter know about the success of the send operation 25110037SARM gem5 Developers snoopFilter->finishRequest(!success, pkt); 25210324SCurtis.Dunham@arm.com } 25310037SARM gem5 Developers 25410037SARM gem5 Developers // check if we were successful in sending the packet onwards 25510037SARM gem5 Developers if (!success) { 25610037SARM gem5 Developers // express snoops should never be forced to retry 25710037SARM gem5 Developers assert(!is_express_snoop); 25810324SCurtis.Dunham@arm.com 25910037SARM gem5 Developers // restore the header delay 26010037SARM gem5 Developers pkt->headerDelay = old_header_delay; 26110037SARM gem5 Developers 26210037SARM gem5 Developers DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 26310324SCurtis.Dunham@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 26410037SARM gem5 Developers 26510037SARM gem5 Developers // update the layer state and schedule an idle event 26610037SARM gem5 Developers reqLayers[master_port_id]->failedTiming(src_port, 26710037SARM gem5 Developers clockEdge(Cycles(1))); 26810037SARM gem5 Developers } else { 26910037SARM gem5 Developers // express snoops currently bypass the crossbar state entirely 27010037SARM gem5 Developers if (!is_express_snoop) { 27110037SARM gem5 Developers // if this particular request will generate a snoop 27210037SARM gem5 Developers // response 27310037SARM gem5 Developers if (expect_snoop_resp) { 27410037SARM gem5 Developers // we should never have an exsiting request outstanding 27510037SARM gem5 Developers assert(outstandingSnoop.find(pkt->req) == 27610037SARM gem5 Developers outstandingSnoop.end()); 27710037SARM gem5 Developers outstandingSnoop.insert(pkt->req); 2787439Sdam.sunwoo@arm.com 2797439Sdam.sunwoo@arm.com // basic sanity check on the outstanding snoops 2807439Sdam.sunwoo@arm.com panic_if(outstandingSnoop.size() > 512, 2817439Sdam.sunwoo@arm.com "Outstanding snoop requests exceeded 512\n"); 2827439Sdam.sunwoo@arm.com } 28310621SCurtis.Dunham@arm.com 28410621SCurtis.Dunham@arm.com // remember where to route the normal response to 28510037SARM gem5 Developers if (expect_response || expect_snoop_resp) { 28610037SARM gem5 Developers assert(routeTo.find(pkt->req) == routeTo.end()); 28710037SARM gem5 Developers routeTo[pkt->req] = slave_port_id; 28810037SARM gem5 Developers 28910037SARM gem5 Developers panic_if(routeTo.size() > 512, 29010037SARM gem5 Developers "Routing table exceeds 512 packets\n"); 29110037SARM gem5 Developers } 2927728SAli.Saidi@ARM.com 29310037SARM gem5 Developers // update the layer state and schedule an idle event 29410037SARM gem5 Developers reqLayers[master_port_id]->succeededTiming(packetFinishTime); 29510037SARM gem5 Developers } 29610037SARM gem5 Developers 29710037SARM gem5 Developers // stats updates only consider packets that were successfully sent 29810037SARM gem5 Developers pktCount[slave_port_id][master_port_id]++; 29910037SARM gem5 Developers pktSize[slave_port_id][master_port_id] += pkt_size; 30010037SARM gem5 Developers transDist[pkt_cmd]++; 30110037SARM gem5 Developers 30210037SARM gem5 Developers if (is_express_snoop) 30310037SARM gem5 Developers snoops++; 30410621SCurtis.Dunham@arm.com } 30510621SCurtis.Dunham@arm.com 30610621SCurtis.Dunham@arm.com return success; 30710621SCurtis.Dunham@arm.com} 30810037SARM gem5 Developers 30910037SARM gem5 Developersbool 31010037SARM gem5 DevelopersCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 31110109SGeoffrey.Blake@arm.com{ 31210037SARM gem5 Developers // determine the source port based on the id 31310109SGeoffrey.Blake@arm.com MasterPort *src_port = masterPorts[master_port_id]; 31410037SARM gem5 Developers 31510109SGeoffrey.Blake@arm.com // determine the destination 31610037SARM gem5 Developers const auto route_lookup = routeTo.find(pkt->req); 31710109SGeoffrey.Blake@arm.com assert(route_lookup != routeTo.end()); 31810109SGeoffrey.Blake@arm.com const PortID slave_port_id = route_lookup->second; 31910109SGeoffrey.Blake@arm.com assert(slave_port_id != InvalidPortID); 32010109SGeoffrey.Blake@arm.com assert(slave_port_id < respLayers.size()); 32110109SGeoffrey.Blake@arm.com 32210109SGeoffrey.Blake@arm.com // test if the crossbar should be considered occupied for the 32310109SGeoffrey.Blake@arm.com // current port 32410109SGeoffrey.Blake@arm.com if (!respLayers[slave_port_id]->tryTiming(src_port)) { 32510109SGeoffrey.Blake@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 32610037SARM gem5 Developers src_port->name(), pkt->cmdString(), pkt->getAddr()); 3277728SAli.Saidi@ARM.com return false; 3288067SAli.Saidi@ARM.com } 3297728SAli.Saidi@ARM.com 3307728SAli.Saidi@ARM.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 33110621SCurtis.Dunham@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 3327728SAli.Saidi@ARM.com 3337728SAli.Saidi@ARM.com // store size and command as they might be modified when 33410621SCurtis.Dunham@arm.com // forwarding the packet 33510037SARM gem5 Developers unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 33610037SARM gem5 Developers unsigned int pkt_cmd = pkt->cmdToIndex(); 33710037SARM gem5 Developers 33810037SARM gem5 Developers // a response sees the response latency 33910037SARM gem5 Developers Tick xbar_delay = responseLatency * clockPeriod(); 34010037SARM gem5 Developers 3417728SAli.Saidi@ARM.com // set the packet header and payload delay 3427728SAli.Saidi@ARM.com calcPacketTiming(pkt, xbar_delay); 3437728SAli.Saidi@ARM.com 3447728SAli.Saidi@ARM.com // determine how long to be crossbar layer is busy 3457728SAli.Saidi@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 3467728SAli.Saidi@ARM.com 3477728SAli.Saidi@ARM.com if (snoopFilter && !system->bypassCaches()) { 3487728SAli.Saidi@ARM.com // let the snoop filter inspect the response and update its state 3497728SAli.Saidi@ARM.com snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 3507728SAli.Saidi@ARM.com } 35110621SCurtis.Dunham@arm.com 3527728SAli.Saidi@ARM.com // send the packet through the destination slave port and pay for 3539258SAli.Saidi@ARM.com // any outstanding header delay 35410037SARM gem5 Developers Tick latency = pkt->headerDelay; 35510037SARM gem5 Developers pkt->headerDelay = 0; 35610037SARM gem5 Developers slavePorts[slave_port_id]->schedTimingResp(pkt, curTick() + latency); 35710037SARM gem5 Developers 35810037SARM gem5 Developers // remove the request from the routing table 35910037SARM gem5 Developers routeTo.erase(route_lookup); 3609535Smrinmoy.ghosh@arm.com 36110037SARM gem5 Developers respLayers[slave_port_id]->succeededTiming(packetFinishTime); 36210037SARM gem5 Developers 36310037SARM gem5 Developers // stats updates 36410037SARM gem5 Developers pktCount[slave_port_id][master_port_id]++; 3659258SAli.Saidi@ARM.com pktSize[slave_port_id][master_port_id] += pkt_size; 3669535Smrinmoy.ghosh@arm.com transDist[pkt_cmd]++; 3679535Smrinmoy.ghosh@arm.com 3689535Smrinmoy.ghosh@arm.com return true; 3699535Smrinmoy.ghosh@arm.com} 3709535Smrinmoy.ghosh@arm.com 3719535Smrinmoy.ghosh@arm.comvoid 3729258SAli.Saidi@ARM.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 3739258SAli.Saidi@ARM.com{ 3749258SAli.Saidi@ARM.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 37510579SAndrew.Bardsley@arm.com masterPorts[master_port_id]->name(), pkt->cmdString(), 37610579SAndrew.Bardsley@arm.com pkt->getAddr()); 37710579SAndrew.Bardsley@arm.com 37810037SARM gem5 Developers // update stats here as we know the forwarding will succeed 37910579SAndrew.Bardsley@arm.com transDist[pkt->cmdToIndex()]++; 38010037SARM gem5 Developers snoops++; 38110579SAndrew.Bardsley@arm.com 38210037SARM gem5 Developers // we should only see express snoops from caches 38310579SAndrew.Bardsley@arm.com assert(pkt->isExpressSnoop()); 38410579SAndrew.Bardsley@arm.com 38510579SAndrew.Bardsley@arm.com // set the packet header and payload delay, for now use forward latency 38610579SAndrew.Bardsley@arm.com // @todo Assess the choice of latency further 38710579SAndrew.Bardsley@arm.com calcPacketTiming(pkt, forwardLatency * clockPeriod()); 38810579SAndrew.Bardsley@arm.com 38910579SAndrew.Bardsley@arm.com // remember if a cache has already committed to responding so we 39010579SAndrew.Bardsley@arm.com // can see if it changes during the snooping 3919258SAli.Saidi@ARM.com const bool cache_responding = pkt->cacheResponding(); 3929258SAli.Saidi@ARM.com 3939258SAli.Saidi@ARM.com assert(pkt->snoopDelay == 0); 3949258SAli.Saidi@ARM.com 3959258SAli.Saidi@ARM.com if (snoopFilter) { 3969258SAli.Saidi@ARM.com // let the Snoop Filter work its magic and guide probing 3979258SAli.Saidi@ARM.com auto sf_res = snoopFilter->lookupSnoop(pkt); 3989258SAli.Saidi@ARM.com // the time required by a packet to be delivered through 3999258SAli.Saidi@ARM.com // the xbar has to be charged also with to lookup latency 4009535Smrinmoy.ghosh@arm.com // of the snoop filter 4019258SAli.Saidi@ARM.com pkt->headerDelay += sf_res.second * clockPeriod(); 4029258SAli.Saidi@ARM.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 40310621SCurtis.Dunham@arm.com " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 4049258SAli.Saidi@ARM.com pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 40510037SARM gem5 Developers sf_res.second); 40610037SARM gem5 Developers 4079258SAli.Saidi@ARM.com // forward to all snoopers 4089535Smrinmoy.ghosh@arm.com forwardTiming(pkt, InvalidPortID, sf_res.first); 4099535Smrinmoy.ghosh@arm.com } else { 41010474Sandreas.hansson@arm.com forwardTiming(pkt, InvalidPortID); 41110474Sandreas.hansson@arm.com } 41210474Sandreas.hansson@arm.com 4139535Smrinmoy.ghosh@arm.com // add the snoop delay to our header delay, and then reset it 4149535Smrinmoy.ghosh@arm.com pkt->headerDelay += pkt->snoopDelay; 41510621SCurtis.Dunham@arm.com pkt->snoopDelay = 0; 41610037SARM gem5 Developers 41710037SARM gem5 Developers // if we can expect a response, remember how to route it 41810037SARM gem5 Developers if (!cache_responding && pkt->cacheResponding()) { 4199535Smrinmoy.ghosh@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 4209258SAli.Saidi@ARM.com routeTo[pkt->req] = master_port_id; 4219258SAli.Saidi@ARM.com } 4229258SAli.Saidi@ARM.com 4239258SAli.Saidi@ARM.com // a snoop request came from a connected slave device (one of 4249258SAli.Saidi@ARM.com // our master ports), and if it is not coming from the slave 4259535Smrinmoy.ghosh@arm.com // device responsible for the address range something is 4269258SAli.Saidi@ARM.com // wrong, hence there is nothing further to do as the packet 42710037SARM gem5 Developers // would be going back to where it came from 42810037SARM gem5 Developers assert(master_port_id == findPort(pkt->getAddr())); 42910037SARM gem5 Developers} 4309535Smrinmoy.ghosh@arm.com 4319535Smrinmoy.ghosh@arm.combool 4329258SAli.Saidi@ARM.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 4339535Smrinmoy.ghosh@arm.com{ 4349258SAli.Saidi@ARM.com // determine the source port based on the id 43510621SCurtis.Dunham@arm.com SlavePort* src_port = slavePorts[slave_port_id]; 4369258SAli.Saidi@ARM.com 43710621SCurtis.Dunham@arm.com // get the destination 4389258SAli.Saidi@ARM.com const auto route_lookup = routeTo.find(pkt->req); 4399258SAli.Saidi@ARM.com assert(route_lookup != routeTo.end()); 4407728SAli.Saidi@ARM.com const PortID dest_port_id = route_lookup->second; 4417728SAli.Saidi@ARM.com assert(dest_port_id != InvalidPortID); 4427728SAli.Saidi@ARM.com 4437728SAli.Saidi@ARM.com // determine if the response is from a snoop request we 4447728SAli.Saidi@ARM.com // created as the result of a normal request (in which case it 4457404SAli.Saidi@ARM.com // should be in the outstandingSnoop), or if we merely forwarded 4467404SAli.Saidi@ARM.com // someone else's snoop request 4477404SAli.Saidi@ARM.com const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 44810037SARM gem5 Developers outstandingSnoop.end(); 4497404SAli.Saidi@ARM.com 45010037SARM gem5 Developers // test if the crossbar should be considered occupied for the 45110037SARM gem5 Developers // current port, note that the check is bypassed if the response 45210037SARM gem5 Developers // is being passed on as a normal response since this is occupying 4537406SAli.Saidi@ARM.com // the response layer rather than the snoop response layer 45410621SCurtis.Dunham@arm.com if (forwardAsSnoop) { 45510621SCurtis.Dunham@arm.com assert(dest_port_id < snoopLayers.size()); 45610037SARM gem5 Developers if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 45710037SARM gem5 Developers DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 4587406SAli.Saidi@ARM.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 45910037SARM gem5 Developers return false; 46010037SARM gem5 Developers } 46110037SARM gem5 Developers } else { 46210474Sandreas.hansson@arm.com // get the master port that mirrors this slave port internally 46310474Sandreas.hansson@arm.com MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 46410474Sandreas.hansson@arm.com assert(dest_port_id < respLayers.size()); 46510474Sandreas.hansson@arm.com if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 46610474Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 46710037SARM gem5 Developers snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 46810474Sandreas.hansson@arm.com return false; 46910474Sandreas.hansson@arm.com } 47010474Sandreas.hansson@arm.com } 47110474Sandreas.hansson@arm.com 47210474Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 47310037SARM gem5 Developers src_port->name(), pkt->cmdString(), pkt->getAddr()); 47410037SARM gem5 Developers 47510037SARM gem5 Developers // store size and command as they might be modified when 4767404SAli.Saidi@ARM.com // forwarding the packet 4777406SAli.Saidi@ARM.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 47810037SARM gem5 Developers unsigned int pkt_cmd = pkt->cmdToIndex(); 47910037SARM gem5 Developers 48010037SARM gem5 Developers // responses are never express snoops 48110474Sandreas.hansson@arm.com assert(!pkt->isExpressSnoop()); 48210474Sandreas.hansson@arm.com 48310474Sandreas.hansson@arm.com // a snoop response sees the snoop response latency, and if it is 48410474Sandreas.hansson@arm.com // forwarded as a normal response, the response latency 48510474Sandreas.hansson@arm.com Tick xbar_delay = 48610037SARM gem5 Developers (forwardAsSnoop ? snoopResponseLatency : responseLatency) * 48710474Sandreas.hansson@arm.com clockPeriod(); 48810474Sandreas.hansson@arm.com 48910474Sandreas.hansson@arm.com // set the packet header and payload delay 49010474Sandreas.hansson@arm.com calcPacketTiming(pkt, xbar_delay); 49110474Sandreas.hansson@arm.com 49210037SARM gem5 Developers // determine how long to be crossbar layer is busy 49310037SARM gem5 Developers Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 49410037SARM gem5 Developers 49510037SARM gem5 Developers // forward it either as a snoop response or a normal response 4967404SAli.Saidi@ARM.com if (forwardAsSnoop) { 4977404SAli.Saidi@ARM.com // this is a snoop response to a snoop request we forwarded, 49810037SARM gem5 Developers // e.g. coming from the L1 and going to the L2, and it should 49910037SARM gem5 Developers // be forwarded as a snoop response 50010037SARM gem5 Developers 50110037SARM gem5 Developers if (snoopFilter) { 5027404SAli.Saidi@ARM.com // update the probe filter so that it can properly track the line 5037404SAli.Saidi@ARM.com snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 5047439Sdam.sunwoo@arm.com *masterPorts[dest_port_id]); 50510037SARM gem5 Developers } 50610037SARM gem5 Developers 50710037SARM gem5 Developers bool success M5_VAR_USED = 5087439Sdam.sunwoo@arm.com masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 50910037SARM gem5 Developers pktCount[slave_port_id][dest_port_id]++; 5107579Sminkyu.jeong@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 5117728SAli.Saidi@ARM.com assert(success); 5127728SAli.Saidi@ARM.com 5137579Sminkyu.jeong@arm.com snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 5147579Sminkyu.jeong@arm.com } else { 5157579Sminkyu.jeong@arm.com // we got a snoop response on one of our slave ports, 5167579Sminkyu.jeong@arm.com // i.e. from a coherent master connected to the crossbar, and 5177579Sminkyu.jeong@arm.com // since we created the snoop request as part of recvTiming, 5187579Sminkyu.jeong@arm.com // this should now be a normal response again 5197404SAli.Saidi@ARM.com outstandingSnoop.erase(pkt->req); 5207404SAli.Saidi@ARM.com 52110836Sandreas.hansson@arm.com // this is a snoop response from a coherent master, hence it 5227946SGiacomo.Gabrielli@arm.com // should never go back to where the snoop response came from, 52310836Sandreas.hansson@arm.com // but instead to where the original request came from 5247946SGiacomo.Gabrielli@arm.com assert(slave_port_id != dest_port_id); 5257946SGiacomo.Gabrielli@arm.com 52611181Snathananel.premillieu@arm.com if (snoopFilter) { 52711181Snathananel.premillieu@arm.com // update the probe filter so that it can properly track the line 52811181Snathananel.premillieu@arm.com snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 52911181Snathananel.premillieu@arm.com *slavePorts[dest_port_id]); 53010037SARM gem5 Developers } 53110037SARM gem5 Developers 53210037SARM gem5 Developers DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 53310037SARM gem5 Developers " FWD RESP\n", src_port->name(), pkt->cmdString(), 53410037SARM gem5 Developers pkt->getAddr()); 53510037SARM gem5 Developers 53610037SARM gem5 Developers // as a normal response, it should go back to a master through 53710037SARM gem5 Developers // one of our slave ports, we also pay for any outstanding 53810037SARM gem5 Developers // header latency 53910037SARM gem5 Developers Tick latency = pkt->headerDelay; 54010037SARM gem5 Developers pkt->headerDelay = 0; 54110037SARM gem5 Developers slavePorts[dest_port_id]->schedTimingResp(pkt, curTick() + latency); 54210037SARM gem5 Developers 54310037SARM gem5 Developers respLayers[dest_port_id]->succeededTiming(packetFinishTime); 54410037SARM gem5 Developers } 54510037SARM gem5 Developers 54610037SARM gem5 Developers // remove the request from the routing table 54710037SARM gem5 Developers routeTo.erase(route_lookup); 54810037SARM gem5 Developers 54910037SARM gem5 Developers // stats updates 55010037SARM gem5 Developers transDist[pkt_cmd]++; 55110621SCurtis.Dunham@arm.com snoops++; 55210621SCurtis.Dunham@arm.com 55310836Sandreas.hansson@arm.com return true; 55410037SARM gem5 Developers} 55510037SARM gem5 Developers 55610037SARM gem5 Developers 55710037SARM gem5 Developersvoid 55810037SARM gem5 DevelopersCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 55910037SARM gem5 Developers const std::vector<QueuedSlavePort*>& dests) 56010037SARM gem5 Developers{ 56110037SARM gem5 Developers DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 56210037SARM gem5 Developers pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 56310037SARM gem5 Developers 56410037SARM gem5 Developers // snoops should only happen if the system isn't bypassing caches 56510037SARM gem5 Developers assert(!system->bypassCaches()); 56610037SARM gem5 Developers 56710037SARM gem5 Developers unsigned fanout = 0; 56810037SARM gem5 Developers 56910037SARM gem5 Developers for (const auto& p: dests) { 57010037SARM gem5 Developers // we could have gotten this request from a snooping master 57110037SARM gem5 Developers // (corresponding to our own slave port that is also in 57210037SARM gem5 Developers // snoopPorts) and should not send it back to where it came 57310037SARM gem5 Developers // from 57410037SARM gem5 Developers if (exclude_slave_port_id == InvalidPortID || 57510037SARM gem5 Developers p->getId() != exclude_slave_port_id) { 57610037SARM gem5 Developers // cache is not allowed to refuse snoop 57710037SARM gem5 Developers p->sendTimingSnoopReq(pkt); 57810037SARM gem5 Developers fanout++; 57910037SARM gem5 Developers } 58010037SARM gem5 Developers } 58110037SARM gem5 Developers 58210037SARM gem5 Developers // Stats for fanout of this forward operation 58310037SARM gem5 Developers snoopFanout.sample(fanout); 58410037SARM gem5 Developers} 58510037SARM gem5 Developers 58610037SARM gem5 Developersvoid 58710037SARM gem5 DevelopersCoherentXBar::recvReqRetry(PortID master_port_id) 58810037SARM gem5 Developers{ 58910037SARM gem5 Developers // responses and snoop responses never block on forwarding them, 59010037SARM gem5 Developers // so the retry will always be coming from a port to which we 59110037SARM gem5 Developers // tried to forward a request 59210037SARM gem5 Developers reqLayers[master_port_id]->recvRetry(); 59310474Sandreas.hansson@arm.com} 59410474Sandreas.hansson@arm.com 59510474Sandreas.hansson@arm.comTick 59610474Sandreas.hansson@arm.comCoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 59710474Sandreas.hansson@arm.com{ 59810037SARM gem5 Developers DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 59910474Sandreas.hansson@arm.com slavePorts[slave_port_id]->name(), pkt->getAddr(), 60010474Sandreas.hansson@arm.com pkt->cmdString()); 60110474Sandreas.hansson@arm.com 60210474Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 60310474Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 60410474Sandreas.hansson@arm.com 60510474Sandreas.hansson@arm.com MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 60610037SARM gem5 Developers Tick snoop_response_latency = 0; 60710037SARM gem5 Developers 60810037SARM gem5 Developers if (!system->bypassCaches()) { 60910037SARM gem5 Developers // forward to all snoopers but the source 61010037SARM gem5 Developers std::pair<MemCmd, Tick> snoop_result; 61110037SARM gem5 Developers if (snoopFilter) { 61210037SARM gem5 Developers // check with the snoop filter where to forward this packet 61310037SARM gem5 Developers auto sf_res = 61410037SARM gem5 Developers snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 61510037SARM gem5 Developers snoop_response_latency += sf_res.second * clockPeriod(); 61610037SARM gem5 Developers DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 61710474Sandreas.hansson@arm.com " SF size: %i lat: %i\n", __func__, 61810474Sandreas.hansson@arm.com slavePorts[slave_port_id]->name(), pkt->cmdString(), 61910474Sandreas.hansson@arm.com pkt->getAddr(), sf_res.first.size(), sf_res.second); 62010474Sandreas.hansson@arm.com 62110474Sandreas.hansson@arm.com // let the snoop filter know about the success of the send 62210037SARM gem5 Developers // operation, and do it even before sending it onwards to 62310474Sandreas.hansson@arm.com // avoid situations where atomic upward snoops sneak in 62410474Sandreas.hansson@arm.com // between and change the filter state 62510474Sandreas.hansson@arm.com snoopFilter->finishRequest(false, pkt); 62610474Sandreas.hansson@arm.com 62710474Sandreas.hansson@arm.com snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 62810474Sandreas.hansson@arm.com sf_res.first); 62910474Sandreas.hansson@arm.com } else { 63010037SARM gem5 Developers snoop_result = forwardAtomic(pkt, slave_port_id); 63110037SARM gem5 Developers } 63210037SARM gem5 Developers snoop_response_cmd = snoop_result.first; 63310037SARM gem5 Developers snoop_response_latency += snoop_result.second; 63410037SARM gem5 Developers } 63510037SARM gem5 Developers 63610037SARM gem5 Developers // forwardAtomic snooped into peer caches of the sender, and if 63710037SARM gem5 Developers // this is a clean evict, but the packet is found in a cache, do 63810037SARM gem5 Developers // not forward it 63910474Sandreas.hansson@arm.com if ((pkt->cmd == MemCmd::CleanEvict || 64010474Sandreas.hansson@arm.com pkt->cmd == MemCmd::WritebackClean) && pkt->isBlockCached()) { 64110474Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "Clean evict/writeback %#llx still cached, " 64210474Sandreas.hansson@arm.com "not forwarding\n", pkt->getAddr()); 64310474Sandreas.hansson@arm.com return 0; 64410037SARM gem5 Developers } 64510474Sandreas.hansson@arm.com 64610474Sandreas.hansson@arm.com // even if we had a snoop response, we must continue and also 64710474Sandreas.hansson@arm.com // perform the actual request at the destination 64810474Sandreas.hansson@arm.com PortID master_port_id = findPort(pkt->getAddr()); 64910474Sandreas.hansson@arm.com 65010037SARM gem5 Developers // stats updates for the request 65110037SARM gem5 Developers pktCount[slave_port_id][master_port_id]++; 65210037SARM gem5 Developers pktSize[slave_port_id][master_port_id] += pkt_size; 65310037SARM gem5 Developers transDist[pkt_cmd]++; 65410037SARM gem5 Developers 65510037SARM gem5 Developers // forward the request to the appropriate destination 65610037SARM gem5 Developers Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 65710037SARM gem5 Developers 65810037SARM gem5 Developers // if lower levels have replied, tell the snoop filter 65910037SARM gem5 Developers if (!system->bypassCaches() && snoopFilter && pkt->isResponse()) { 66010037SARM gem5 Developers snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 66110037SARM gem5 Developers } 66210037SARM gem5 Developers 66310037SARM gem5 Developers // if we got a response from a snooper, restore it here 66410037SARM gem5 Developers if (snoop_response_cmd != MemCmd::InvalidCmd) { 66510037SARM gem5 Developers // no one else should have responded 66610037SARM gem5 Developers assert(!pkt->isResponse()); 66710037SARM gem5 Developers pkt->cmd = snoop_response_cmd; 66810037SARM gem5 Developers response_latency = snoop_response_latency; 66910037SARM gem5 Developers } 67010037SARM gem5 Developers 67110037SARM gem5 Developers // add the response data 67210037SARM gem5 Developers if (pkt->isResponse()) { 67310037SARM gem5 Developers pkt_size = pkt->hasData() ? pkt->getSize() : 0; 67410037SARM gem5 Developers pkt_cmd = pkt->cmdToIndex(); 67510037SARM gem5 Developers 67610037SARM gem5 Developers // stats updates 67710037SARM gem5 Developers pktCount[slave_port_id][master_port_id]++; 67810037SARM gem5 Developers pktSize[slave_port_id][master_port_id] += pkt_size; 67910037SARM gem5 Developers transDist[pkt_cmd]++; 68010037SARM gem5 Developers } 68110037SARM gem5 Developers 68210037SARM gem5 Developers // @todo: Not setting header time 68310037SARM gem5 Developers pkt->payloadDelay = response_latency; 68410037SARM gem5 Developers return response_latency; 68510037SARM gem5 Developers} 68610037SARM gem5 Developers 68710037SARM gem5 DevelopersTick 68810037SARM gem5 DevelopersCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 68910836Sandreas.hansson@arm.com{ 69010037SARM gem5 Developers DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 69110037SARM gem5 Developers masterPorts[master_port_id]->name(), pkt->getAddr(), 69210037SARM gem5 Developers pkt->cmdString()); 69310037SARM gem5 Developers 69410324SCurtis.Dunham@arm.com // add the request snoop data 69510037SARM gem5 Developers snoops++; 69610037SARM gem5 Developers 69710037SARM gem5 Developers // forward to all snoopers 69810037SARM gem5 Developers std::pair<MemCmd, Tick> snoop_result; 69910037SARM gem5 Developers Tick snoop_response_latency = 0; 70010037SARM gem5 Developers if (snoopFilter) { 70110037SARM gem5 Developers auto sf_res = snoopFilter->lookupSnoop(pkt); 70210037SARM gem5 Developers snoop_response_latency += sf_res.second * clockPeriod(); 70310037SARM gem5 Developers DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 70410037SARM gem5 Developers __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 70510037SARM gem5 Developers pkt->getAddr(), sf_res.first.size(), sf_res.second); 70610037SARM gem5 Developers snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 70710037SARM gem5 Developers sf_res.first); 70810037SARM gem5 Developers } else { 70910037SARM gem5 Developers snoop_result = forwardAtomic(pkt, InvalidPortID); 71010037SARM gem5 Developers } 71110037SARM gem5 Developers MemCmd snoop_response_cmd = snoop_result.first; 71210037SARM gem5 Developers snoop_response_latency += snoop_result.second; 71310037SARM gem5 Developers 71410037SARM gem5 Developers if (snoop_response_cmd != MemCmd::InvalidCmd) 71510037SARM gem5 Developers pkt->cmd = snoop_response_cmd; 71610037SARM gem5 Developers 71710037SARM gem5 Developers // add the response snoop data 71810037SARM gem5 Developers if (pkt->isResponse()) { 71910037SARM gem5 Developers snoops++; 72010037SARM gem5 Developers } 72110037SARM gem5 Developers 72210037SARM gem5 Developers // @todo: Not setting header time 72310037SARM gem5 Developers pkt->payloadDelay = snoop_response_latency; 72410037SARM gem5 Developers return snoop_response_latency; 72510037SARM gem5 Developers} 72610037SARM gem5 Developers 72710037SARM gem5 Developersstd::pair<MemCmd, Tick> 72810037SARM gem5 DevelopersCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 72910037SARM gem5 Developers PortID source_master_port_id, 73010037SARM gem5 Developers const std::vector<QueuedSlavePort*>& dests) 73110324SCurtis.Dunham@arm.com{ 73210324SCurtis.Dunham@arm.com // the packet may be changed on snoops, record the original 73310324SCurtis.Dunham@arm.com // command to enable us to restore it between snoops so that 73410324SCurtis.Dunham@arm.com // additional snoops can take place properly 73510324SCurtis.Dunham@arm.com MemCmd orig_cmd = pkt->cmd; 73610324SCurtis.Dunham@arm.com MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 73710324SCurtis.Dunham@arm.com Tick snoop_response_latency = 0; 73810037SARM gem5 Developers 73910621SCurtis.Dunham@arm.com // snoops should only happen if the system isn't bypassing caches 74010621SCurtis.Dunham@arm.com assert(!system->bypassCaches()); 74110037SARM gem5 Developers 74210037SARM gem5 Developers unsigned fanout = 0; 74310037SARM gem5 Developers 74410324SCurtis.Dunham@arm.com for (const auto& p: dests) { 74510037SARM gem5 Developers // we could have gotten this request from a snooping master 74610037SARM gem5 Developers // (corresponding to our own slave port that is also in 74710037SARM gem5 Developers // snoopPorts) and should not send it back to where it came 74810037SARM gem5 Developers // from 74910037SARM gem5 Developers if (exclude_slave_port_id != InvalidPortID && 75010037SARM gem5 Developers p->getId() == exclude_slave_port_id) 75110037SARM gem5 Developers continue; 75210037SARM gem5 Developers 75310324SCurtis.Dunham@arm.com Tick latency = p->sendAtomicSnoop(pkt); 75410324SCurtis.Dunham@arm.com fanout++; 75510037SARM gem5 Developers 75610324SCurtis.Dunham@arm.com // in contrast to a functional access, we have to keep on 75710037SARM gem5 Developers // going as all snoopers must be updated even if we get a 75810037SARM gem5 Developers // response 75910037SARM gem5 Developers if (!pkt->isResponse()) 76010037SARM gem5 Developers continue; 76110037SARM gem5 Developers 76210324SCurtis.Dunham@arm.com // response from snoop agent 76310324SCurtis.Dunham@arm.com assert(pkt->cmd != orig_cmd); 76410037SARM gem5 Developers assert(pkt->cacheResponding()); 76510324SCurtis.Dunham@arm.com // should only happen once 76610037SARM gem5 Developers assert(snoop_response_cmd == MemCmd::InvalidCmd); 76710037SARM gem5 Developers // save response state 76810037SARM gem5 Developers snoop_response_cmd = pkt->cmd; 76910037SARM gem5 Developers snoop_response_latency = latency; 77010037SARM gem5 Developers 77110037SARM gem5 Developers if (snoopFilter) { 77210324SCurtis.Dunham@arm.com // Handle responses by the snoopers and differentiate between 77310037SARM gem5 Developers // responses to requests from above and snoops from below 77410037SARM gem5 Developers if (source_master_port_id != InvalidPortID) { 77510037SARM gem5 Developers // Getting a response for a snoop from below 77610037SARM gem5 Developers assert(exclude_slave_port_id == InvalidPortID); 77710037SARM gem5 Developers snoopFilter->updateSnoopForward(pkt, *p, 77810324SCurtis.Dunham@arm.com *masterPorts[source_master_port_id]); 77910324SCurtis.Dunham@arm.com } else { 78010324SCurtis.Dunham@arm.com // Getting a response for a request from above 78110324SCurtis.Dunham@arm.com assert(source_master_port_id == InvalidPortID); 78210324SCurtis.Dunham@arm.com snoopFilter->updateSnoopResponse(pkt, *p, 78310324SCurtis.Dunham@arm.com *slavePorts[exclude_slave_port_id]); 78410324SCurtis.Dunham@arm.com } 78510037SARM gem5 Developers } 78610037SARM gem5 Developers // restore original packet state for remaining snoopers 78710037SARM gem5 Developers pkt->cmd = orig_cmd; 78810324SCurtis.Dunham@arm.com } 78910037SARM gem5 Developers 79010324SCurtis.Dunham@arm.com // Stats for fanout 79110037SARM gem5 Developers snoopFanout.sample(fanout); 79210037SARM gem5 Developers 79310037SARM gem5 Developers // the packet is restored as part of the loop and any potential 79410037SARM gem5 Developers // snoop response is part of the returned pair 79510037SARM gem5 Developers return std::make_pair(snoop_response_cmd, snoop_response_latency); 79610037SARM gem5 Developers} 79710474Sandreas.hansson@arm.com 79810474Sandreas.hansson@arm.comvoid 79910474Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 80010474Sandreas.hansson@arm.com{ 80110037SARM gem5 Developers if (!pkt->isPrint()) { 80210474Sandreas.hansson@arm.com // don't do DPRINTFs on PrintReq as it clutters up the output 80310474Sandreas.hansson@arm.com DPRINTF(CoherentXBar, 80410474Sandreas.hansson@arm.com "recvFunctional: packet src %s addr 0x%x cmd %s\n", 80510474Sandreas.hansson@arm.com slavePorts[slave_port_id]->name(), pkt->getAddr(), 80610474Sandreas.hansson@arm.com pkt->cmdString()); 80710474Sandreas.hansson@arm.com } 80810037SARM gem5 Developers 80910037SARM gem5 Developers if (!system->bypassCaches()) { 81010037SARM gem5 Developers // forward to all snoopers but the source 81110037SARM gem5 Developers forwardFunctional(pkt, slave_port_id); 81210037SARM gem5 Developers } 81310037SARM gem5 Developers 81410037SARM gem5 Developers // there is no need to continue if the snooping has found what we 81510037SARM gem5 Developers // were looking for and the packet is already a response 81610037SARM gem5 Developers if (!pkt->isResponse()) { 81710037SARM gem5 Developers // since our slave ports are queued ports we need to check them as well 81810037SARM gem5 Developers for (const auto& p : slavePorts) { 81910037SARM gem5 Developers // if we find a response that has the data, then the 82010037SARM gem5 Developers // downstream caches/memories may be out of date, so simply stop 82110324SCurtis.Dunham@arm.com // here 82210324SCurtis.Dunham@arm.com if (p->checkFunctional(pkt)) { 82310324SCurtis.Dunham@arm.com if (pkt->needsResponse()) 82410324SCurtis.Dunham@arm.com pkt->makeResponse(); 82510324SCurtis.Dunham@arm.com return; 82610324SCurtis.Dunham@arm.com } 82710324SCurtis.Dunham@arm.com } 82810324SCurtis.Dunham@arm.com 82910324SCurtis.Dunham@arm.com PortID dest_id = findPort(pkt->getAddr()); 83010037SARM gem5 Developers 83110324SCurtis.Dunham@arm.com masterPorts[dest_id]->sendFunctional(pkt); 83210324SCurtis.Dunham@arm.com } 83310324SCurtis.Dunham@arm.com} 83410324SCurtis.Dunham@arm.com 83510324SCurtis.Dunham@arm.comvoid 83610324SCurtis.Dunham@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 83710324SCurtis.Dunham@arm.com{ 83810324SCurtis.Dunham@arm.com if (!pkt->isPrint()) { 83910324SCurtis.Dunham@arm.com // don't do DPRINTFs on PrintReq as it clutters up the output 84010324SCurtis.Dunham@arm.com DPRINTF(CoherentXBar, 84110324SCurtis.Dunham@arm.com "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 84210324SCurtis.Dunham@arm.com masterPorts[master_port_id]->name(), pkt->getAddr(), 84310324SCurtis.Dunham@arm.com pkt->cmdString()); 84410324SCurtis.Dunham@arm.com } 84510324SCurtis.Dunham@arm.com 84610324SCurtis.Dunham@arm.com for (const auto& p : slavePorts) { 84710324SCurtis.Dunham@arm.com if (p->checkFunctional(pkt)) { 84810324SCurtis.Dunham@arm.com if (pkt->needsResponse()) 84910324SCurtis.Dunham@arm.com pkt->makeResponse(); 85010324SCurtis.Dunham@arm.com return; 85110324SCurtis.Dunham@arm.com } 85210324SCurtis.Dunham@arm.com } 85310324SCurtis.Dunham@arm.com 85410324SCurtis.Dunham@arm.com // forward to all snoopers 85510324SCurtis.Dunham@arm.com forwardFunctional(pkt, InvalidPortID); 85610324SCurtis.Dunham@arm.com} 85710324SCurtis.Dunham@arm.com 85810324SCurtis.Dunham@arm.comvoid 85910324SCurtis.Dunham@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 86010324SCurtis.Dunham@arm.com{ 86110324SCurtis.Dunham@arm.com // snoops should only happen if the system isn't bypassing caches 86210324SCurtis.Dunham@arm.com assert(!system->bypassCaches()); 86310324SCurtis.Dunham@arm.com 86410324SCurtis.Dunham@arm.com for (const auto& p: snoopPorts) { 86510037SARM gem5 Developers // we could have gotten this request from a snooping master 86610037SARM gem5 Developers // (corresponding to our own slave port that is also in 86710037SARM gem5 Developers // snoopPorts) and should not send it back to where it came 86810324SCurtis.Dunham@arm.com // from 86910037SARM gem5 Developers if (exclude_slave_port_id == InvalidPortID || 87010037SARM gem5 Developers p->getId() != exclude_slave_port_id) 87110037SARM gem5 Developers p->sendFunctionalSnoop(pkt); 87210037SARM gem5 Developers 87310037SARM gem5 Developers // if we get a response we are done 87410037SARM gem5 Developers if (pkt->isResponse()) { 87510037SARM gem5 Developers break; 87610037SARM gem5 Developers } 87710037SARM gem5 Developers } 87810037SARM gem5 Developers} 87910037SARM gem5 Developers 88010037SARM gem5 Developersvoid 88110037SARM gem5 DevelopersCoherentXBar::regStats() 88210037SARM gem5 Developers{ 88310474Sandreas.hansson@arm.com // register the stats of the base class and our layers 88410474Sandreas.hansson@arm.com BaseXBar::regStats(); 88510474Sandreas.hansson@arm.com for (auto l: reqLayers) 88610474Sandreas.hansson@arm.com l->regStats(); 88710474Sandreas.hansson@arm.com for (auto l: respLayers) 88810037SARM gem5 Developers l->regStats(); 88910474Sandreas.hansson@arm.com for (auto l: snoopLayers) 89010474Sandreas.hansson@arm.com l->regStats(); 89110474Sandreas.hansson@arm.com 89210474Sandreas.hansson@arm.com snoops 89310474Sandreas.hansson@arm.com .name(name() + ".snoops") 89410474Sandreas.hansson@arm.com .desc("Total snoops (count)") 89510474Sandreas.hansson@arm.com ; 89610037SARM gem5 Developers 89710037SARM gem5 Developers snoopFanout 89810037SARM gem5 Developers .init(0, snoopPorts.size(), 1) 89910037SARM gem5 Developers .name(name() + ".snoop_fanout") 90010037SARM gem5 Developers .desc("Request fanout histogram") 90110037SARM gem5 Developers ; 90210037SARM gem5 Developers} 90310037SARM gem5 Developers 90410037SARM gem5 DevelopersCoherentXBar * 90510037SARM gem5 DevelopersCoherentXBarParams::create() 90610037SARM gem5 Developers{ 90710037SARM gem5 Developers return new CoherentXBar(this); 90810037SARM gem5 Developers} 90910037SARM gem5 Developers