coherent_xbar.cc revision 12241
12497SN/A/* 212241Snikos.nikoleris@arm.com * Copyright (c) 2011-2017 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142497SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152497SN/A * All rights reserved. 162497SN/A * 172497SN/A * Redistribution and use in source and binary forms, with or without 182497SN/A * modification, are permitted provided that the following conditions are 192497SN/A * met: redistributions of source code must retain the above copyright 202497SN/A * notice, this list of conditions and the following disclaimer; 212497SN/A * redistributions in binary form must reproduce the above copyright 222497SN/A * notice, this list of conditions and the following disclaimer in the 232497SN/A * documentation and/or other materials provided with the distribution; 242497SN/A * neither the name of the copyright holders nor the names of its 252497SN/A * contributors may be used to endorse or promote products derived from 262497SN/A * this software without specific prior written permission. 272497SN/A * 282497SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292497SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302497SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312497SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322497SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332497SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342497SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352497SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362497SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372497SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382497SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 418715SN/A * Andreas Hansson 428922SN/A * William Wang 432497SN/A */ 442497SN/A 452497SN/A/** 462982SN/A * @file 4710405Sandreas.hansson@arm.com * Definition of a crossbar object. 482497SN/A */ 492497SN/A 5011793Sbrandon.potter@amd.com#include "mem/coherent_xbar.hh" 5111793Sbrandon.potter@amd.com 522846SN/A#include "base/misc.hh" 532548SN/A#include "base/trace.hh" 5410405Sandreas.hansson@arm.com#include "debug/AddrRanges.hh" 5510405Sandreas.hansson@arm.com#include "debug/CoherentXBar.hh" 569524SN/A#include "sim/system.hh" 572497SN/A 5810405Sandreas.hansson@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 5910719SMarco.Balboni@ARM.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter), 6011334Sandreas.hansson@arm.com snoopResponseLatency(p->snoop_response_latency), 6111334Sandreas.hansson@arm.com pointOfCoherency(p->point_of_coherency) 627523SN/A{ 638851SN/A // create the ports based on the size of the master and slave 648948SN/A // vector ports, and the presence of the default port, the ports 658948SN/A // are enumerated starting from zero 668851SN/A for (int i = 0; i < p->port_master_connection_count; ++i) { 679095SN/A std::string portName = csprintf("%s.master[%d]", name(), i); 6810405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 698922SN/A masterPorts.push_back(bp); 709715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 719715SN/A csprintf(".reqLayer%d", i))); 7210713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 7310713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", i))); 748851SN/A } 758851SN/A 768948SN/A // see if we have a default slave device connected and if so add 778948SN/A // our corresponding master port 788915SN/A if (p->port_default_connection_count) { 799031SN/A defaultPortID = masterPorts.size(); 809095SN/A std::string portName = name() + ".default"; 8110405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 829036SN/A defaultPortID); 838922SN/A masterPorts.push_back(bp); 849715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 859715SN/A defaultPortID))); 8610713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 8710713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", 8810713Sandreas.hansson@arm.com defaultPortID))); 898915SN/A } 908915SN/A 918948SN/A // create the slave ports, once again starting at zero 928851SN/A for (int i = 0; i < p->port_slave_connection_count; ++i) { 939095SN/A std::string portName = csprintf("%s.slave[%d]", name(), i); 9410888Sandreas.hansson@arm.com QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 958922SN/A slavePorts.push_back(bp); 969715SN/A respLayers.push_back(new RespLayer(*bp, *this, 979715SN/A csprintf(".respLayer%d", i))); 989716SN/A snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 998851SN/A } 1008851SN/A 1017523SN/A clearPortCache(); 1027523SN/A} 1037523SN/A 10410405Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 1059715SN/A{ 10610405Sandreas.hansson@arm.com for (auto l: reqLayers) 10710405Sandreas.hansson@arm.com delete l; 10810405Sandreas.hansson@arm.com for (auto l: respLayers) 10910405Sandreas.hansson@arm.com delete l; 11010405Sandreas.hansson@arm.com for (auto l: snoopLayers) 11110405Sandreas.hansson@arm.com delete l; 11210405Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11310405Sandreas.hansson@arm.com delete p; 1149715SN/A} 1159715SN/A 1162568SN/Avoid 11710405Sandreas.hansson@arm.comCoherentXBar::init() 1182568SN/A{ 11910405Sandreas.hansson@arm.com BaseXBar::init(); 1209278SN/A 1218948SN/A // iterate over our slave ports and determine which of our 1228948SN/A // neighbouring master ports are snooping and add them as snoopers 12310405Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 1249088SN/A // check if the connected master port is snooping 12510405Sandreas.hansson@arm.com if (p->isSnooping()) { 12610405Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12710405Sandreas.hansson@arm.com p->getMasterPort().name()); 12810405Sandreas.hansson@arm.com snoopPorts.push_back(p); 1298711SN/A } 1308711SN/A } 1312568SN/A 1329036SN/A if (snoopPorts.empty()) 13310405Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 13411133Sandreas.hansson@arm.com 13511133Sandreas.hansson@arm.com // inform the snoop filter about the slave ports so it can create 13611133Sandreas.hansson@arm.com // its own internal representation 13711133Sandreas.hansson@arm.com if (snoopFilter) 13811133Sandreas.hansson@arm.com snoopFilter->setSlavePorts(slavePorts); 1393244SN/A} 1403244SN/A 1418948SN/Abool 14210405Sandreas.hansson@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1433244SN/A{ 1448975SN/A // determine the source port based on the id 1459032SN/A SlavePort *src_port = slavePorts[slave_port_id]; 1463244SN/A 1479091SN/A // remember if the packet is an express snoop 1489091SN/A bool is_express_snoop = pkt->isExpressSnoop(); 14911284Sandreas.hansson@arm.com bool cache_responding = pkt->cacheResponding(); 15010656Sandreas.hansson@arm.com // for normal requests, going downstream, the express snoop flag 15111284Sandreas.hansson@arm.com // and the cache responding flag should always be the same 15211284Sandreas.hansson@arm.com assert(is_express_snoop == cache_responding); 1539091SN/A 1549612SN/A // determine the destination based on the address 1559712SN/A PortID master_port_id = findPort(pkt->getAddr()); 1569612SN/A 15710405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the current 1589033SN/A // port, and exclude express snoops from the check 1599715SN/A if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 16011744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 16111744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1623244SN/A return false; 1633244SN/A } 1643244SN/A 16511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 16611744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1675197SN/A 1689712SN/A // store size and command as they might be modified when 1699712SN/A // forwarding the packet 1709712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1719712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 1729712SN/A 17310719SMarco.Balboni@ARM.com // store the old header delay so we can restore it if needed 17410719SMarco.Balboni@ARM.com Tick old_header_delay = pkt->headerDelay; 17510719SMarco.Balboni@ARM.com 17610719SMarco.Balboni@ARM.com // a request sees the frontend and forward latency 17710719SMarco.Balboni@ARM.com Tick xbar_delay = (frontendLatency + forwardLatency) * clockPeriod(); 17810719SMarco.Balboni@ARM.com 17910719SMarco.Balboni@ARM.com // set the packet header and payload delay 18010719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 18110719SMarco.Balboni@ARM.com 18210719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 18310719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 1844912SN/A 18510821Sandreas.hansson@arm.com if (!system->bypassCaches()) { 18611127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 18711127Sandreas.hansson@arm.com 1888979SN/A // the packet is a memory-mapped request and should be 1898979SN/A // broadcasted to our snoopers but the source 19010402SN/A if (snoopFilter) { 19110402SN/A // check with the snoop filter where to forward this packet 19210402SN/A auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 19311126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 19411126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 19511126Sandreas.hansson@arm.com // of the snoop filter 19610719SMarco.Balboni@ARM.com pkt->headerDelay += sf_res.second * clockPeriod(); 19711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 19811744Snikos.nikoleris@arm.com __func__, src_port->name(), pkt->print(), 19911744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 20011196Sali.jafri@arm.com 20111199Sandreas.hansson@arm.com if (pkt->isEviction()) { 20211196Sali.jafri@arm.com // for block-evicting packets, i.e. writebacks and 20311196Sali.jafri@arm.com // clean evictions, there is no need to snoop up, as 20411196Sali.jafri@arm.com // all we do is determine if the block is cached or 20511196Sali.jafri@arm.com // not, instead just set it here based on the snoop 20611196Sali.jafri@arm.com // filter result 20711196Sali.jafri@arm.com if (!sf_res.first.empty()) 20811196Sali.jafri@arm.com pkt->setBlockCached(); 20911196Sali.jafri@arm.com } else { 21011196Sali.jafri@arm.com forwardTiming(pkt, slave_port_id, sf_res.first); 21111196Sali.jafri@arm.com } 21210402SN/A } else { 21310402SN/A forwardTiming(pkt, slave_port_id); 21410402SN/A } 21511127Sandreas.hansson@arm.com 21611127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 21711127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 21811127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 2198979SN/A } 2208948SN/A 22111334Sandreas.hansson@arm.com // set up a sensible starting point 22211334Sandreas.hansson@arm.com bool success = true; 22310883Sali.jafri@arm.com 22411284Sandreas.hansson@arm.com // remember if the packet will generate a snoop response by 22511284Sandreas.hansson@arm.com // checking if a cache set the cacheResponding flag during the 22611284Sandreas.hansson@arm.com // snooping above 22711284Sandreas.hansson@arm.com const bool expect_snoop_resp = !cache_responding && pkt->cacheResponding(); 22811334Sandreas.hansson@arm.com bool expect_response = pkt->needsResponse() && !pkt->cacheResponding(); 2298915SN/A 23011334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 23111334Sandreas.hansson@arm.com 23211334Sandreas.hansson@arm.com // in certain cases the crossbar is responsible for responding 23311334Sandreas.hansson@arm.com bool respond_directly = false; 23411544Snikos.nikoleris@arm.com // store the original address as an address mapper could possibly 23511544Snikos.nikoleris@arm.com // modify the address upon a sendTimingRequest 23611544Snikos.nikoleris@arm.com const Addr addr(pkt->getAddr()); 23711334Sandreas.hansson@arm.com if (sink_packet) { 23811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 23911744Snikos.nikoleris@arm.com pkt->print()); 24011334Sandreas.hansson@arm.com } else { 24111334Sandreas.hansson@arm.com // determine if we are forwarding the packet, or responding to 24211334Sandreas.hansson@arm.com // it 24311334Sandreas.hansson@arm.com if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) { 24411334Sandreas.hansson@arm.com // if we are passing on, rather than sinking, a packet to 24511334Sandreas.hansson@arm.com // which an upstream cache has committed to responding, 24611334Sandreas.hansson@arm.com // the line was needs writable, and the responding only 24711334Sandreas.hansson@arm.com // had an Owned copy, so we need to immidiately let the 24811334Sandreas.hansson@arm.com // downstream caches know, bypass any flow control 24911334Sandreas.hansson@arm.com if (pkt->cacheResponding()) { 25011334Sandreas.hansson@arm.com pkt->setExpressSnoop(); 25111334Sandreas.hansson@arm.com } 25211334Sandreas.hansson@arm.com 25311334Sandreas.hansson@arm.com // since it is a normal request, attempt to send the packet 25411334Sandreas.hansson@arm.com success = masterPorts[master_port_id]->sendTimingReq(pkt); 25511334Sandreas.hansson@arm.com } else { 25611334Sandreas.hansson@arm.com // no need to forward, turn this packet around and respond 25711334Sandreas.hansson@arm.com // directly 25811334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 25911334Sandreas.hansson@arm.com 26011334Sandreas.hansson@arm.com respond_directly = true; 26111334Sandreas.hansson@arm.com assert(!expect_snoop_resp); 26211334Sandreas.hansson@arm.com expect_response = false; 26311334Sandreas.hansson@arm.com } 26411334Sandreas.hansson@arm.com } 2658948SN/A 26610821Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 26710402SN/A // Let the snoop filter know about the success of the send operation 26811605Snikos.nikoleris@arm.com snoopFilter->finishRequest(!success, addr, pkt->isSecure()); 26910402SN/A } 27010402SN/A 27110656Sandreas.hansson@arm.com // check if we were successful in sending the packet onwards 27210656Sandreas.hansson@arm.com if (!success) { 27311284Sandreas.hansson@arm.com // express snoops should never be forced to retry 27410656Sandreas.hansson@arm.com assert(!is_express_snoop); 27510656Sandreas.hansson@arm.com 27610719SMarco.Balboni@ARM.com // restore the header delay 27710719SMarco.Balboni@ARM.com pkt->headerDelay = old_header_delay; 27810656Sandreas.hansson@arm.com 27911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__, 28011744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 28110656Sandreas.hansson@arm.com 28210656Sandreas.hansson@arm.com // update the layer state and schedule an idle event 28310656Sandreas.hansson@arm.com reqLayers[master_port_id]->failedTiming(src_port, 28410719SMarco.Balboni@ARM.com clockEdge(Cycles(1))); 2859091SN/A } else { 28610656Sandreas.hansson@arm.com // express snoops currently bypass the crossbar state entirely 28710656Sandreas.hansson@arm.com if (!is_express_snoop) { 28810656Sandreas.hansson@arm.com // if this particular request will generate a snoop 28910656Sandreas.hansson@arm.com // response 29010656Sandreas.hansson@arm.com if (expect_snoop_resp) { 29110656Sandreas.hansson@arm.com // we should never have an exsiting request outstanding 29210656Sandreas.hansson@arm.com assert(outstandingSnoop.find(pkt->req) == 29310656Sandreas.hansson@arm.com outstandingSnoop.end()); 29410656Sandreas.hansson@arm.com outstandingSnoop.insert(pkt->req); 2958948SN/A 29610656Sandreas.hansson@arm.com // basic sanity check on the outstanding snoops 29710656Sandreas.hansson@arm.com panic_if(outstandingSnoop.size() > 512, 29810656Sandreas.hansson@arm.com "Outstanding snoop requests exceeded 512\n"); 29910656Sandreas.hansson@arm.com } 3008948SN/A 30110656Sandreas.hansson@arm.com // remember where to route the normal response to 30210656Sandreas.hansson@arm.com if (expect_response || expect_snoop_resp) { 30310656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 30410656Sandreas.hansson@arm.com routeTo[pkt->req] = slave_port_id; 3059549SN/A 30610656Sandreas.hansson@arm.com panic_if(routeTo.size() > 512, 30710656Sandreas.hansson@arm.com "Routing table exceeds 512 packets\n"); 30810656Sandreas.hansson@arm.com } 3098948SN/A 31010405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 3119715SN/A reqLayers[master_port_id]->succeededTiming(packetFinishTime); 3129091SN/A } 3138975SN/A 31410656Sandreas.hansson@arm.com // stats updates only consider packets that were successfully sent 3159712SN/A pktCount[slave_port_id][master_port_id]++; 31610405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 3179712SN/A transDist[pkt_cmd]++; 31810656Sandreas.hansson@arm.com 31911564Sdavid.guillen@arm.com if (is_express_snoop) { 32010656Sandreas.hansson@arm.com snoops++; 32111564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 32211564Sdavid.guillen@arm.com } 3239712SN/A } 3249712SN/A 32511334Sandreas.hansson@arm.com if (sink_packet) 32611334Sandreas.hansson@arm.com // queue the packet for deletion 32711334Sandreas.hansson@arm.com pendingDelete.reset(pkt); 32811334Sandreas.hansson@arm.com 32911334Sandreas.hansson@arm.com if (respond_directly) { 33011334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 33111334Sandreas.hansson@arm.com assert(success); 33211334Sandreas.hansson@arm.com 33311334Sandreas.hansson@arm.com pkt->makeResponse(); 33411334Sandreas.hansson@arm.com 33511334Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 33611334Sandreas.hansson@arm.com // let the snoop filter inspect the response and update its state 33711334Sandreas.hansson@arm.com snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 33811334Sandreas.hansson@arm.com } 33911334Sandreas.hansson@arm.com 34011334Sandreas.hansson@arm.com Tick response_time = clockEdge() + pkt->headerDelay; 34111334Sandreas.hansson@arm.com pkt->headerDelay = 0; 34211334Sandreas.hansson@arm.com 34311334Sandreas.hansson@arm.com slavePorts[slave_port_id]->schedTimingResp(pkt, response_time); 34411334Sandreas.hansson@arm.com } 34511334Sandreas.hansson@arm.com 3469091SN/A return success; 3478975SN/A} 3488975SN/A 3498975SN/Abool 35010405Sandreas.hansson@arm.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 3518975SN/A{ 3528975SN/A // determine the source port based on the id 3539032SN/A MasterPort *src_port = masterPorts[master_port_id]; 3548975SN/A 35510656Sandreas.hansson@arm.com // determine the destination 35610656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 35710656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 35810656Sandreas.hansson@arm.com const PortID slave_port_id = route_lookup->second; 35910572Sandreas.hansson@arm.com assert(slave_port_id != InvalidPortID); 36010572Sandreas.hansson@arm.com assert(slave_port_id < respLayers.size()); 3619713SN/A 36210405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 36310405Sandreas.hansson@arm.com // current port 3649715SN/A if (!respLayers[slave_port_id]->tryTiming(src_port)) { 36511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 36611744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 3678975SN/A return false; 3688975SN/A } 3698975SN/A 37011744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 37111744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 3728975SN/A 3739712SN/A // store size and command as they might be modified when 3749712SN/A // forwarding the packet 3759712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 3769712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 3779712SN/A 37810719SMarco.Balboni@ARM.com // a response sees the response latency 37910719SMarco.Balboni@ARM.com Tick xbar_delay = responseLatency * clockPeriod(); 38010719SMarco.Balboni@ARM.com 38110719SMarco.Balboni@ARM.com // set the packet header and payload delay 38210719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 38310719SMarco.Balboni@ARM.com 38410719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 38510719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 3868975SN/A 38710821Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 38810402SN/A // let the snoop filter inspect the response and update its state 38910402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 39010402SN/A } 39110402SN/A 39210888Sandreas.hansson@arm.com // send the packet through the destination slave port and pay for 39310888Sandreas.hansson@arm.com // any outstanding header delay 39410888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 39510888Sandreas.hansson@arm.com pkt->headerDelay = 0; 39610888Sandreas.hansson@arm.com slavePorts[slave_port_id]->schedTimingResp(pkt, curTick() + latency); 3978975SN/A 39810656Sandreas.hansson@arm.com // remove the request from the routing table 39910656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 40010656Sandreas.hansson@arm.com 4019715SN/A respLayers[slave_port_id]->succeededTiming(packetFinishTime); 4028975SN/A 4039712SN/A // stats updates 4049712SN/A pktCount[slave_port_id][master_port_id]++; 40510405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 4069712SN/A transDist[pkt_cmd]++; 4079712SN/A 4088975SN/A return true; 4098975SN/A} 4108975SN/A 4118975SN/Avoid 41210405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 4138975SN/A{ 41411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 41511744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 4168975SN/A 4179712SN/A // update stats here as we know the forwarding will succeed 41811564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 4199712SN/A transDist[pkt->cmdToIndex()]++; 42010405Sandreas.hansson@arm.com snoops++; 42111564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 4229712SN/A 4238975SN/A // we should only see express snoops from caches 4248975SN/A assert(pkt->isExpressSnoop()); 4258975SN/A 42611127Sandreas.hansson@arm.com // set the packet header and payload delay, for now use forward latency 42711127Sandreas.hansson@arm.com // @todo Assess the choice of latency further 42811127Sandreas.hansson@arm.com calcPacketTiming(pkt, forwardLatency * clockPeriod()); 42911127Sandreas.hansson@arm.com 43011284Sandreas.hansson@arm.com // remember if a cache has already committed to responding so we 43111284Sandreas.hansson@arm.com // can see if it changes during the snooping 43211284Sandreas.hansson@arm.com const bool cache_responding = pkt->cacheResponding(); 4339032SN/A 43411127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 43511127Sandreas.hansson@arm.com 43610402SN/A if (snoopFilter) { 43710402SN/A // let the Snoop Filter work its magic and guide probing 43810402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 43911126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 44011126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 44111126Sandreas.hansson@arm.com // of the snoop filter 44211126Sandreas.hansson@arm.com pkt->headerDelay += sf_res.second * clockPeriod(); 44311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 44411744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 44511744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 44610402SN/A 44710402SN/A // forward to all snoopers 44810402SN/A forwardTiming(pkt, InvalidPortID, sf_res.first); 44910402SN/A } else { 45010402SN/A forwardTiming(pkt, InvalidPortID); 45110402SN/A } 4528975SN/A 45311127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 45411127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 45511127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 45611127Sandreas.hansson@arm.com 45710656Sandreas.hansson@arm.com // if we can expect a response, remember how to route it 45811284Sandreas.hansson@arm.com if (!cache_responding && pkt->cacheResponding()) { 45910656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 46010656Sandreas.hansson@arm.com routeTo[pkt->req] = master_port_id; 46110656Sandreas.hansson@arm.com } 46210656Sandreas.hansson@arm.com 4638975SN/A // a snoop request came from a connected slave device (one of 4648975SN/A // our master ports), and if it is not coming from the slave 4658975SN/A // device responsible for the address range something is 4668975SN/A // wrong, hence there is nothing further to do as the packet 4678975SN/A // would be going back to where it came from 4689032SN/A assert(master_port_id == findPort(pkt->getAddr())); 4698975SN/A} 4708975SN/A 4718975SN/Abool 47210405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 4738975SN/A{ 4748975SN/A // determine the source port based on the id 4759032SN/A SlavePort* src_port = slavePorts[slave_port_id]; 4768975SN/A 47710656Sandreas.hansson@arm.com // get the destination 47810656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 47910656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 48010656Sandreas.hansson@arm.com const PortID dest_port_id = route_lookup->second; 48110572Sandreas.hansson@arm.com assert(dest_port_id != InvalidPortID); 4829714SN/A 4839714SN/A // determine if the response is from a snoop request we 4849714SN/A // created as the result of a normal request (in which case it 48510656Sandreas.hansson@arm.com // should be in the outstandingSnoop), or if we merely forwarded 4869714SN/A // someone else's snoop request 48710656Sandreas.hansson@arm.com const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 48810656Sandreas.hansson@arm.com outstandingSnoop.end(); 4899714SN/A 49010405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 49110405Sandreas.hansson@arm.com // current port, note that the check is bypassed if the response 49210405Sandreas.hansson@arm.com // is being passed on as a normal response since this is occupying 49310405Sandreas.hansson@arm.com // the response layer rather than the snoop response layer 4949715SN/A if (forwardAsSnoop) { 49510572Sandreas.hansson@arm.com assert(dest_port_id < snoopLayers.size()); 4969715SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 49711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 49811744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 4999715SN/A return false; 5009715SN/A } 5019716SN/A } else { 5029716SN/A // get the master port that mirrors this slave port internally 5039716SN/A MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 50410572Sandreas.hansson@arm.com assert(dest_port_id < respLayers.size()); 5059716SN/A if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 50611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 50711744Snikos.nikoleris@arm.com snoop_port->name(), pkt->print()); 5089716SN/A return false; 5099716SN/A } 5108975SN/A } 5118975SN/A 51211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 51311744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 5148975SN/A 5159712SN/A // store size and command as they might be modified when 5169712SN/A // forwarding the packet 5179712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 5189712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 5199712SN/A 5208975SN/A // responses are never express snoops 5218975SN/A assert(!pkt->isExpressSnoop()); 5228975SN/A 52310719SMarco.Balboni@ARM.com // a snoop response sees the snoop response latency, and if it is 52410719SMarco.Balboni@ARM.com // forwarded as a normal response, the response latency 52510719SMarco.Balboni@ARM.com Tick xbar_delay = 52610719SMarco.Balboni@ARM.com (forwardAsSnoop ? snoopResponseLatency : responseLatency) * 52710719SMarco.Balboni@ARM.com clockPeriod(); 52810719SMarco.Balboni@ARM.com 52910719SMarco.Balboni@ARM.com // set the packet header and payload delay 53010719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 53110719SMarco.Balboni@ARM.com 53210719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 53310719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 5348975SN/A 5359714SN/A // forward it either as a snoop response or a normal response 5369714SN/A if (forwardAsSnoop) { 5379714SN/A // this is a snoop response to a snoop request we forwarded, 5389714SN/A // e.g. coming from the L1 and going to the L2, and it should 5399714SN/A // be forwarded as a snoop response 54010402SN/A 54110402SN/A if (snoopFilter) { 54210402SN/A // update the probe filter so that it can properly track the line 54310402SN/A snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 54410402SN/A *masterPorts[dest_port_id]); 54510402SN/A } 54610402SN/A 5479712SN/A bool success M5_VAR_USED = 5489712SN/A masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 5499712SN/A pktCount[slave_port_id][dest_port_id]++; 55010405Sandreas.hansson@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 5518975SN/A assert(success); 5529714SN/A 5539715SN/A snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 5543244SN/A } else { 5558975SN/A // we got a snoop response on one of our slave ports, 55610405Sandreas.hansson@arm.com // i.e. from a coherent master connected to the crossbar, and 55710405Sandreas.hansson@arm.com // since we created the snoop request as part of recvTiming, 55810405Sandreas.hansson@arm.com // this should now be a normal response again 55910656Sandreas.hansson@arm.com outstandingSnoop.erase(pkt->req); 5608948SN/A 56110656Sandreas.hansson@arm.com // this is a snoop response from a coherent master, hence it 56210656Sandreas.hansson@arm.com // should never go back to where the snoop response came from, 56310656Sandreas.hansson@arm.com // but instead to where the original request came from 5649712SN/A assert(slave_port_id != dest_port_id); 5658948SN/A 56610402SN/A if (snoopFilter) { 56710402SN/A // update the probe filter so that it can properly track the line 56810402SN/A snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 56910402SN/A *slavePorts[dest_port_id]); 57010402SN/A } 57110402SN/A 57211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s FWD RESP\n", __func__, 57311744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 57410402SN/A 5759714SN/A // as a normal response, it should go back to a master through 57610888Sandreas.hansson@arm.com // one of our slave ports, we also pay for any outstanding 57710888Sandreas.hansson@arm.com // header latency 57810888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 57910888Sandreas.hansson@arm.com pkt->headerDelay = 0; 58010888Sandreas.hansson@arm.com slavePorts[dest_port_id]->schedTimingResp(pkt, curTick() + latency); 5819716SN/A 5829716SN/A respLayers[dest_port_id]->succeededTiming(packetFinishTime); 5833244SN/A } 5843244SN/A 58510656Sandreas.hansson@arm.com // remove the request from the routing table 58610656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 58710656Sandreas.hansson@arm.com 5889712SN/A // stats updates 5899712SN/A transDist[pkt_cmd]++; 59010405Sandreas.hansson@arm.com snoops++; 59111564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 5929712SN/A 5938948SN/A return true; 5948948SN/A} 5958948SN/A 5963210SN/A 5978948SN/Avoid 59810405Sandreas.hansson@arm.comCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 59910888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 6008948SN/A{ 60111744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s for %s\n", __func__, pkt->print()); 6029663SN/A 6039524SN/A // snoops should only happen if the system isn't bypassing caches 6049524SN/A assert(!system->bypassCaches()); 6059524SN/A 60610401SN/A unsigned fanout = 0; 60710401SN/A 60810405Sandreas.hansson@arm.com for (const auto& p: dests) { 6098948SN/A // we could have gotten this request from a snooping master 6108948SN/A // (corresponding to our own slave port that is also in 6118948SN/A // snoopPorts) and should not send it back to where it came 6128948SN/A // from 6139031SN/A if (exclude_slave_port_id == InvalidPortID || 6148948SN/A p->getId() != exclude_slave_port_id) { 6158948SN/A // cache is not allowed to refuse snoop 6168975SN/A p->sendTimingSnoopReq(pkt); 61710401SN/A fanout++; 6188948SN/A } 6198948SN/A } 62010401SN/A 62110401SN/A // Stats for fanout of this forward operation 62210401SN/A snoopFanout.sample(fanout); 6232497SN/A} 6242497SN/A 6259092SN/Avoid 62610713Sandreas.hansson@arm.comCoherentXBar::recvReqRetry(PortID master_port_id) 6279092SN/A{ 6289093SN/A // responses and snoop responses never block on forwarding them, 6299093SN/A // so the retry will always be coming from a port to which we 6309093SN/A // tried to forward a request 6319715SN/A reqLayers[master_port_id]->recvRetry(); 6329092SN/A} 6339092SN/A 6349036SN/ATick 63510405Sandreas.hansson@arm.comCoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 6362657SN/A{ 63711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 63811744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 6398915SN/A 64010405Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 64110405Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 6429712SN/A 6438979SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 6448979SN/A Tick snoop_response_latency = 0; 6458979SN/A 64610821Sandreas.hansson@arm.com if (!system->bypassCaches()) { 6478979SN/A // forward to all snoopers but the source 64810402SN/A std::pair<MemCmd, Tick> snoop_result; 64910402SN/A if (snoopFilter) { 65010402SN/A // check with the snoop filter where to forward this packet 65110402SN/A auto sf_res = 65210402SN/A snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 65310402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 65411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 65511744Snikos.nikoleris@arm.com __func__, slavePorts[slave_port_id]->name(), pkt->print(), 65611744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 65711130Sali.jafri@arm.com 65811130Sali.jafri@arm.com // let the snoop filter know about the success of the send 65911130Sali.jafri@arm.com // operation, and do it even before sending it onwards to 66011130Sali.jafri@arm.com // avoid situations where atomic upward snoops sneak in 66111130Sali.jafri@arm.com // between and change the filter state 66211605Snikos.nikoleris@arm.com snoopFilter->finishRequest(false, pkt->getAddr(), pkt->isSecure()); 66311130Sali.jafri@arm.com 66412241Snikos.nikoleris@arm.com if (pkt->isEviction()) { 66512241Snikos.nikoleris@arm.com // for block-evicting packets, i.e. writebacks and 66612241Snikos.nikoleris@arm.com // clean evictions, there is no need to snoop up, as 66712241Snikos.nikoleris@arm.com // all we do is determine if the block is cached or 66812241Snikos.nikoleris@arm.com // not, instead just set it here based on the snoop 66912241Snikos.nikoleris@arm.com // filter result 67012241Snikos.nikoleris@arm.com if (!sf_res.first.empty()) 67112241Snikos.nikoleris@arm.com pkt->setBlockCached(); 67212241Snikos.nikoleris@arm.com } else { 67312241Snikos.nikoleris@arm.com snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 67412241Snikos.nikoleris@arm.com sf_res.first); 67512241Snikos.nikoleris@arm.com } 67610402SN/A } else { 67710402SN/A snoop_result = forwardAtomic(pkt, slave_port_id); 67810402SN/A } 6798979SN/A snoop_response_cmd = snoop_result.first; 68010402SN/A snoop_response_latency += snoop_result.second; 6818979SN/A } 6828915SN/A 68311334Sandreas.hansson@arm.com // set up a sensible default value 68411334Sandreas.hansson@arm.com Tick response_latency = 0; 68511334Sandreas.hansson@arm.com 68611334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 68711130Sali.jafri@arm.com 6888948SN/A // even if we had a snoop response, we must continue and also 6898948SN/A // perform the actual request at the destination 69010405Sandreas.hansson@arm.com PortID master_port_id = findPort(pkt->getAddr()); 69110405Sandreas.hansson@arm.com 69211334Sandreas.hansson@arm.com if (sink_packet) { 69311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 69411744Snikos.nikoleris@arm.com pkt->print()); 69511334Sandreas.hansson@arm.com } else { 69611334Sandreas.hansson@arm.com if (!pointOfCoherency || pkt->isRead() || pkt->isWrite()) { 69711334Sandreas.hansson@arm.com // forward the request to the appropriate destination 69811334Sandreas.hansson@arm.com response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 69911334Sandreas.hansson@arm.com } else { 70011334Sandreas.hansson@arm.com // if it does not need a response we sink the packet above 70111334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 70211334Sandreas.hansson@arm.com 70311334Sandreas.hansson@arm.com pkt->makeResponse(); 70411334Sandreas.hansson@arm.com } 70511334Sandreas.hansson@arm.com } 70611334Sandreas.hansson@arm.com 70710405Sandreas.hansson@arm.com // stats updates for the request 70810405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 70910405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 71010405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 7118948SN/A 7128948SN/A 71311130Sali.jafri@arm.com // if lower levels have replied, tell the snoop filter 71411130Sali.jafri@arm.com if (!system->bypassCaches() && snoopFilter && pkt->isResponse()) { 71510402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 71610402SN/A } 71710402SN/A 7188948SN/A // if we got a response from a snooper, restore it here 7198948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) { 7208948SN/A // no one else should have responded 7218948SN/A assert(!pkt->isResponse()); 7228948SN/A pkt->cmd = snoop_response_cmd; 7238948SN/A response_latency = snoop_response_latency; 7248948SN/A } 7258948SN/A 7269712SN/A // add the response data 72710405Sandreas.hansson@arm.com if (pkt->isResponse()) { 72810405Sandreas.hansson@arm.com pkt_size = pkt->hasData() ? pkt->getSize() : 0; 72910405Sandreas.hansson@arm.com pkt_cmd = pkt->cmdToIndex(); 73010405Sandreas.hansson@arm.com 73110405Sandreas.hansson@arm.com // stats updates 73210405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 73310405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 73410405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 73510405Sandreas.hansson@arm.com } 7369712SN/A 73710694SMarco.Balboni@ARM.com // @todo: Not setting header time 73810694SMarco.Balboni@ARM.com pkt->payloadDelay = response_latency; 7398948SN/A return response_latency; 7408948SN/A} 7418948SN/A 7428948SN/ATick 74310405Sandreas.hansson@arm.comCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 7448948SN/A{ 74511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 74611744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 7478948SN/A 7489712SN/A // add the request snoop data 74911564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 75010405Sandreas.hansson@arm.com snoops++; 75111564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 7529712SN/A 7538948SN/A // forward to all snoopers 75410402SN/A std::pair<MemCmd, Tick> snoop_result; 75510402SN/A Tick snoop_response_latency = 0; 75610402SN/A if (snoopFilter) { 75710402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 75810402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 75911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 76011744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 76111744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 76210402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 76310402SN/A sf_res.first); 76410402SN/A } else { 76510402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID); 76610402SN/A } 7678948SN/A MemCmd snoop_response_cmd = snoop_result.first; 76810402SN/A snoop_response_latency += snoop_result.second; 7698948SN/A 7708948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) 7718948SN/A pkt->cmd = snoop_response_cmd; 7728948SN/A 7739712SN/A // add the response snoop data 77410401SN/A if (pkt->isResponse()) { 77510405Sandreas.hansson@arm.com snoops++; 77610401SN/A } 7779712SN/A 77810694SMarco.Balboni@ARM.com // @todo: Not setting header time 77910694SMarco.Balboni@ARM.com pkt->payloadDelay = snoop_response_latency; 7808948SN/A return snoop_response_latency; 7818948SN/A} 7828948SN/A 7838948SN/Astd::pair<MemCmd, Tick> 78410405Sandreas.hansson@arm.comCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 78510402SN/A PortID source_master_port_id, 78610888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 7878948SN/A{ 7889032SN/A // the packet may be changed on snoops, record the original 7899032SN/A // command to enable us to restore it between snoops so that 7908948SN/A // additional snoops can take place properly 7914626SN/A MemCmd orig_cmd = pkt->cmd; 7924879SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 7934879SN/A Tick snoop_response_latency = 0; 7943662SN/A 7959524SN/A // snoops should only happen if the system isn't bypassing caches 7969524SN/A assert(!system->bypassCaches()); 7979524SN/A 79810401SN/A unsigned fanout = 0; 79910401SN/A 80010405Sandreas.hansson@arm.com for (const auto& p: dests) { 8018915SN/A // we could have gotten this request from a snooping master 8028915SN/A // (corresponding to our own slave port that is also in 8038915SN/A // snoopPorts) and should not send it back to where it came 8048915SN/A // from 80510402SN/A if (exclude_slave_port_id != InvalidPortID && 80610402SN/A p->getId() == exclude_slave_port_id) 80710402SN/A continue; 80810401SN/A 80910402SN/A Tick latency = p->sendAtomicSnoop(pkt); 81010402SN/A fanout++; 81110402SN/A 81210402SN/A // in contrast to a functional access, we have to keep on 81310402SN/A // going as all snoopers must be updated even if we get a 81410402SN/A // response 81510402SN/A if (!pkt->isResponse()) 81610402SN/A continue; 81710402SN/A 81810402SN/A // response from snoop agent 81910402SN/A assert(pkt->cmd != orig_cmd); 82011284Sandreas.hansson@arm.com assert(pkt->cacheResponding()); 82110402SN/A // should only happen once 82210402SN/A assert(snoop_response_cmd == MemCmd::InvalidCmd); 82310402SN/A // save response state 82410402SN/A snoop_response_cmd = pkt->cmd; 82510402SN/A snoop_response_latency = latency; 82610402SN/A 82710402SN/A if (snoopFilter) { 82810402SN/A // Handle responses by the snoopers and differentiate between 82910402SN/A // responses to requests from above and snoops from below 83010402SN/A if (source_master_port_id != InvalidPortID) { 83110402SN/A // Getting a response for a snoop from below 83210402SN/A assert(exclude_slave_port_id == InvalidPortID); 83310402SN/A snoopFilter->updateSnoopForward(pkt, *p, 83410402SN/A *masterPorts[source_master_port_id]); 83510402SN/A } else { 83610402SN/A // Getting a response for a request from above 83710402SN/A assert(source_master_port_id == InvalidPortID); 83810402SN/A snoopFilter->updateSnoopResponse(pkt, *p, 83910402SN/A *slavePorts[exclude_slave_port_id]); 8404626SN/A } 8414626SN/A } 84210402SN/A // restore original packet state for remaining snoopers 84310402SN/A pkt->cmd = orig_cmd; 8444626SN/A } 8454626SN/A 84610401SN/A // Stats for fanout 84710401SN/A snoopFanout.sample(fanout); 84810401SN/A 8498948SN/A // the packet is restored as part of the loop and any potential 8508948SN/A // snoop response is part of the returned pair 8518948SN/A return std::make_pair(snoop_response_cmd, snoop_response_latency); 8522497SN/A} 8532497SN/A 8542497SN/Avoid 85510405Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 8562497SN/A{ 8578663SN/A if (!pkt->isPrint()) { 8588663SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 85911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 86011744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 8618663SN/A } 8628663SN/A 86310821Sandreas.hansson@arm.com if (!system->bypassCaches()) { 8648979SN/A // forward to all snoopers but the source 8659032SN/A forwardFunctional(pkt, slave_port_id); 8668979SN/A } 8674912SN/A 8688948SN/A // there is no need to continue if the snooping has found what we 8698948SN/A // were looking for and the packet is already a response 8708948SN/A if (!pkt->isResponse()) { 87110888Sandreas.hansson@arm.com // since our slave ports are queued ports we need to check them as well 87210888Sandreas.hansson@arm.com for (const auto& p : slavePorts) { 87310888Sandreas.hansson@arm.com // if we find a response that has the data, then the 87410888Sandreas.hansson@arm.com // downstream caches/memories may be out of date, so simply stop 87510888Sandreas.hansson@arm.com // here 87610888Sandreas.hansson@arm.com if (p->checkFunctional(pkt)) { 87710888Sandreas.hansson@arm.com if (pkt->needsResponse()) 87810888Sandreas.hansson@arm.com pkt->makeResponse(); 87910888Sandreas.hansson@arm.com return; 88010888Sandreas.hansson@arm.com } 88110888Sandreas.hansson@arm.com } 88210888Sandreas.hansson@arm.com 8839031SN/A PortID dest_id = findPort(pkt->getAddr()); 8848948SN/A 8858948SN/A masterPorts[dest_id]->sendFunctional(pkt); 8868948SN/A } 8878948SN/A} 8888948SN/A 8898948SN/Avoid 89010405Sandreas.hansson@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 8918948SN/A{ 8928948SN/A if (!pkt->isPrint()) { 8938948SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 89411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 89511744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 8968948SN/A } 8978948SN/A 89811188Sandreas.sandberg@arm.com for (const auto& p : slavePorts) { 89911188Sandreas.sandberg@arm.com if (p->checkFunctional(pkt)) { 90011188Sandreas.sandberg@arm.com if (pkt->needsResponse()) 90111188Sandreas.sandberg@arm.com pkt->makeResponse(); 90211188Sandreas.sandberg@arm.com return; 90311188Sandreas.sandberg@arm.com } 90411188Sandreas.sandberg@arm.com } 90511188Sandreas.sandberg@arm.com 9068948SN/A // forward to all snoopers 9079031SN/A forwardFunctional(pkt, InvalidPortID); 9088948SN/A} 9098948SN/A 9108948SN/Avoid 91110405Sandreas.hansson@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 9128948SN/A{ 9139524SN/A // snoops should only happen if the system isn't bypassing caches 9149524SN/A assert(!system->bypassCaches()); 9159524SN/A 91610405Sandreas.hansson@arm.com for (const auto& p: snoopPorts) { 9178915SN/A // we could have gotten this request from a snooping master 9188915SN/A // (corresponding to our own slave port that is also in 9198915SN/A // snoopPorts) and should not send it back to where it came 9208915SN/A // from 9219031SN/A if (exclude_slave_port_id == InvalidPortID || 9228948SN/A p->getId() != exclude_slave_port_id) 9238948SN/A p->sendFunctionalSnoop(pkt); 9248915SN/A 9258948SN/A // if we get a response we are done 9268948SN/A if (pkt->isResponse()) { 9278948SN/A break; 9288915SN/A } 9293650SN/A } 9302497SN/A} 9312497SN/A 93211334Sandreas.hansson@arm.combool 93311334Sandreas.hansson@arm.comCoherentXBar::sinkPacket(const PacketPtr pkt) const 93411334Sandreas.hansson@arm.com{ 93511334Sandreas.hansson@arm.com // we can sink the packet if: 93611334Sandreas.hansson@arm.com // 1) the crossbar is the point of coherency, and a cache is 93711334Sandreas.hansson@arm.com // responding after being snooped 93811334Sandreas.hansson@arm.com // 2) the crossbar is the point of coherency, and the packet is a 93911334Sandreas.hansson@arm.com // coherency packet (not a read or a write) that does not 94011334Sandreas.hansson@arm.com // require a response 94111334Sandreas.hansson@arm.com // 3) this is a clean evict or clean writeback, but the packet is 94211334Sandreas.hansson@arm.com // found in a cache above this crossbar 94311334Sandreas.hansson@arm.com // 4) a cache is responding after being snooped, and the packet 94411334Sandreas.hansson@arm.com // either does not need the block to be writable, or the cache 94511334Sandreas.hansson@arm.com // that has promised to respond (setting the cache responding 94611334Sandreas.hansson@arm.com // flag) is providing writable and thus had a Modified block, 94711334Sandreas.hansson@arm.com // and no further action is needed 94811334Sandreas.hansson@arm.com return (pointOfCoherency && pkt->cacheResponding()) || 94911334Sandreas.hansson@arm.com (pointOfCoherency && !(pkt->isRead() || pkt->isWrite()) && 95011334Sandreas.hansson@arm.com !pkt->needsResponse()) || 95111334Sandreas.hansson@arm.com (pkt->isCleanEviction() && pkt->isBlockCached()) || 95211334Sandreas.hansson@arm.com (pkt->cacheResponding() && 95311334Sandreas.hansson@arm.com (!pkt->needsWritable() || pkt->responderHadWritable())); 95411334Sandreas.hansson@arm.com} 95511334Sandreas.hansson@arm.com 9569712SN/Avoid 95710405Sandreas.hansson@arm.comCoherentXBar::regStats() 9589712SN/A{ 95910405Sandreas.hansson@arm.com // register the stats of the base class and our layers 96010405Sandreas.hansson@arm.com BaseXBar::regStats(); 96110405Sandreas.hansson@arm.com for (auto l: reqLayers) 96210405Sandreas.hansson@arm.com l->regStats(); 96310405Sandreas.hansson@arm.com for (auto l: respLayers) 96410405Sandreas.hansson@arm.com l->regStats(); 96510405Sandreas.hansson@arm.com for (auto l: snoopLayers) 96610405Sandreas.hansson@arm.com l->regStats(); 9679712SN/A 96810405Sandreas.hansson@arm.com snoops 96910405Sandreas.hansson@arm.com .name(name() + ".snoops") 97010401SN/A .desc("Total snoops (count)") 97110401SN/A ; 97210401SN/A 97311564Sdavid.guillen@arm.com snoopTraffic 97411564Sdavid.guillen@arm.com .name(name() + ".snoopTraffic") 97511564Sdavid.guillen@arm.com .desc("Total snoop traffic (bytes)") 97611564Sdavid.guillen@arm.com ; 97711564Sdavid.guillen@arm.com 97810401SN/A snoopFanout 97910401SN/A .init(0, snoopPorts.size(), 1) 98010401SN/A .name(name() + ".snoop_fanout") 98110401SN/A .desc("Request fanout histogram") 98210401SN/A ; 9839712SN/A} 9849712SN/A 98510405Sandreas.hansson@arm.comCoherentXBar * 98610405Sandreas.hansson@arm.comCoherentXBarParams::create() 9872497SN/A{ 98810405Sandreas.hansson@arm.com return new CoherentXBar(this); 9892497SN/A} 990