coherent_xbar.cc revision 10713
12497SN/A/* 210405Sandreas.hansson@arm.com * Copyright (c) 2011-2014 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142497SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152497SN/A * All rights reserved. 162497SN/A * 172497SN/A * Redistribution and use in source and binary forms, with or without 182497SN/A * modification, are permitted provided that the following conditions are 192497SN/A * met: redistributions of source code must retain the above copyright 202497SN/A * notice, this list of conditions and the following disclaimer; 212497SN/A * redistributions in binary form must reproduce the above copyright 222497SN/A * notice, this list of conditions and the following disclaimer in the 232497SN/A * documentation and/or other materials provided with the distribution; 242497SN/A * neither the name of the copyright holders nor the names of its 252497SN/A * contributors may be used to endorse or promote products derived from 262497SN/A * this software without specific prior written permission. 272497SN/A * 282497SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292497SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302497SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312497SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322497SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332497SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342497SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352497SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362497SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372497SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382497SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 418715SN/A * Andreas Hansson 428922SN/A * William Wang 432497SN/A */ 442497SN/A 452497SN/A/** 462982SN/A * @file 4710405Sandreas.hansson@arm.com * Definition of a crossbar object. 482497SN/A */ 492497SN/A 502846SN/A#include "base/misc.hh" 512548SN/A#include "base/trace.hh" 5210405Sandreas.hansson@arm.com#include "debug/AddrRanges.hh" 5310405Sandreas.hansson@arm.com#include "debug/CoherentXBar.hh" 5410405Sandreas.hansson@arm.com#include "mem/coherent_xbar.hh" 559524SN/A#include "sim/system.hh" 562497SN/A 5710405Sandreas.hansson@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 5810405Sandreas.hansson@arm.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 597523SN/A{ 608851SN/A // create the ports based on the size of the master and slave 618948SN/A // vector ports, and the presence of the default port, the ports 628948SN/A // are enumerated starting from zero 638851SN/A for (int i = 0; i < p->port_master_connection_count; ++i) { 649095SN/A std::string portName = csprintf("%s.master[%d]", name(), i); 6510405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 668922SN/A masterPorts.push_back(bp); 679715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 689715SN/A csprintf(".reqLayer%d", i))); 6910713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 7010713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", i))); 718851SN/A } 728851SN/A 738948SN/A // see if we have a default slave device connected and if so add 748948SN/A // our corresponding master port 758915SN/A if (p->port_default_connection_count) { 769031SN/A defaultPortID = masterPorts.size(); 779095SN/A std::string portName = name() + ".default"; 7810405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 799036SN/A defaultPortID); 808922SN/A masterPorts.push_back(bp); 819715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 829715SN/A defaultPortID))); 8310713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 8410713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", 8510713Sandreas.hansson@arm.com defaultPortID))); 868915SN/A } 878915SN/A 888948SN/A // create the slave ports, once again starting at zero 898851SN/A for (int i = 0; i < p->port_slave_connection_count; ++i) { 909095SN/A std::string portName = csprintf("%s.slave[%d]", name(), i); 9110405Sandreas.hansson@arm.com SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 928922SN/A slavePorts.push_back(bp); 939715SN/A respLayers.push_back(new RespLayer(*bp, *this, 949715SN/A csprintf(".respLayer%d", i))); 959716SN/A snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 968851SN/A } 978851SN/A 9810402SN/A if (snoopFilter) 9910402SN/A snoopFilter->setSlavePorts(slavePorts); 10010402SN/A 1017523SN/A clearPortCache(); 1027523SN/A} 1037523SN/A 10410405Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 1059715SN/A{ 10610405Sandreas.hansson@arm.com for (auto l: reqLayers) 10710405Sandreas.hansson@arm.com delete l; 10810405Sandreas.hansson@arm.com for (auto l: respLayers) 10910405Sandreas.hansson@arm.com delete l; 11010405Sandreas.hansson@arm.com for (auto l: snoopLayers) 11110405Sandreas.hansson@arm.com delete l; 11210405Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11310405Sandreas.hansson@arm.com delete p; 1149715SN/A} 1159715SN/A 1162568SN/Avoid 11710405Sandreas.hansson@arm.comCoherentXBar::init() 1182568SN/A{ 1199278SN/A // the base class is responsible for determining the block size 12010405Sandreas.hansson@arm.com BaseXBar::init(); 1219278SN/A 1228948SN/A // iterate over our slave ports and determine which of our 1238948SN/A // neighbouring master ports are snooping and add them as snoopers 12410405Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 1259088SN/A // check if the connected master port is snooping 12610405Sandreas.hansson@arm.com if (p->isSnooping()) { 12710405Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12810405Sandreas.hansson@arm.com p->getMasterPort().name()); 12910405Sandreas.hansson@arm.com snoopPorts.push_back(p); 1308711SN/A } 1318711SN/A } 1322568SN/A 1339036SN/A if (snoopPorts.empty()) 13410405Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 1353244SN/A} 1363244SN/A 1378948SN/Abool 13810405Sandreas.hansson@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1393244SN/A{ 1408975SN/A // determine the source port based on the id 1419032SN/A SlavePort *src_port = slavePorts[slave_port_id]; 1423244SN/A 1439091SN/A // remember if the packet is an express snoop 1449091SN/A bool is_express_snoop = pkt->isExpressSnoop(); 14510656Sandreas.hansson@arm.com bool is_inhibited = pkt->memInhibitAsserted(); 14610656Sandreas.hansson@arm.com // for normal requests, going downstream, the express snoop flag 14710656Sandreas.hansson@arm.com // and the inhibited flag should always be the same 14810656Sandreas.hansson@arm.com assert(is_express_snoop == is_inhibited); 1499091SN/A 1509612SN/A // determine the destination based on the address 1519712SN/A PortID master_port_id = findPort(pkt->getAddr()); 1529612SN/A 15310405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the current 1549033SN/A // port, and exclude express snoops from the check 1559715SN/A if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 15610405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 1578949SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 1583244SN/A return false; 1593244SN/A } 1603244SN/A 16110405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 1629091SN/A src_port->name(), pkt->cmdString(), is_express_snoop, 1639091SN/A pkt->getAddr()); 1645197SN/A 1659712SN/A // store size and command as they might be modified when 1669712SN/A // forwarding the packet 1679712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1689712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 1699712SN/A 1709547SN/A calcPacketTiming(pkt); 17110694SMarco.Balboni@ARM.com Tick packetFinishTime = curTick() + pkt->payloadDelay; 1724912SN/A 1738979SN/A // uncacheable requests need never be snooped 1749524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 1758979SN/A // the packet is a memory-mapped request and should be 1768979SN/A // broadcasted to our snoopers but the source 17710402SN/A if (snoopFilter) { 17810402SN/A // check with the snoop filter where to forward this packet 17910402SN/A auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 18010402SN/A packetFinishTime += sf_res.second * clockPeriod(); 18110405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 18210402SN/A " SF size: %i lat: %i\n", src_port->name(), 18310402SN/A pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 18410402SN/A sf_res.second); 18510402SN/A forwardTiming(pkt, slave_port_id, sf_res.first); 18610402SN/A } else { 18710402SN/A forwardTiming(pkt, slave_port_id); 18810402SN/A } 1898979SN/A } 1908948SN/A 19110656Sandreas.hansson@arm.com // remember if the packet will generate a snoop response 19210656Sandreas.hansson@arm.com const bool expect_snoop_resp = !is_inhibited && pkt->memInhibitAsserted(); 19310656Sandreas.hansson@arm.com const bool expect_response = pkt->needsResponse() && 19410656Sandreas.hansson@arm.com !pkt->memInhibitAsserted(); 1958915SN/A 19610402SN/A // Note: Cannot create a copy of the full packet, here. 19710402SN/A MemCmd orig_cmd(pkt->cmd); 19810402SN/A 1999612SN/A // since it is a normal request, attempt to send the packet 2009712SN/A bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 2018948SN/A 20210402SN/A if (snoopFilter && !pkt->req->isUncacheable() 20310402SN/A && !system->bypassCaches()) { 20410402SN/A // The packet may already be overwritten by the sendTimingReq function. 20510402SN/A // The snoop filter needs to see the original request *and* the return 20610402SN/A // status of the send operation, so we need to recreate the original 20710402SN/A // request. Atomic mode does not have the issue, as there the send 20810402SN/A // operation and the response happen instantaneously and don't need two 20910402SN/A // phase tracking. 21010402SN/A MemCmd tmp_cmd(pkt->cmd); 21110402SN/A pkt->cmd = orig_cmd; 21210402SN/A // Let the snoop filter know about the success of the send operation 21310402SN/A snoopFilter->updateRequest(pkt, *src_port, !success); 21410402SN/A pkt->cmd = tmp_cmd; 21510402SN/A } 21610402SN/A 21710656Sandreas.hansson@arm.com // check if we were successful in sending the packet onwards 21810656Sandreas.hansson@arm.com if (!success) { 21910656Sandreas.hansson@arm.com // express snoops and inhibited packets should never be forced 22010656Sandreas.hansson@arm.com // to retry 22110656Sandreas.hansson@arm.com assert(!is_express_snoop); 22210656Sandreas.hansson@arm.com assert(!pkt->memInhibitAsserted()); 22310656Sandreas.hansson@arm.com 22410656Sandreas.hansson@arm.com // undo the calculation so we can check for 0 again 22510694SMarco.Balboni@ARM.com pkt->headerDelay = pkt->payloadDelay = 0; 22610656Sandreas.hansson@arm.com 22710656Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 22810656Sandreas.hansson@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 22910656Sandreas.hansson@arm.com 23010656Sandreas.hansson@arm.com // update the layer state and schedule an idle event 23110656Sandreas.hansson@arm.com reqLayers[master_port_id]->failedTiming(src_port, 23210656Sandreas.hansson@arm.com clockEdge(headerCycles)); 2339091SN/A } else { 23410656Sandreas.hansson@arm.com // express snoops currently bypass the crossbar state entirely 23510656Sandreas.hansson@arm.com if (!is_express_snoop) { 23610656Sandreas.hansson@arm.com // if this particular request will generate a snoop 23710656Sandreas.hansson@arm.com // response 23810656Sandreas.hansson@arm.com if (expect_snoop_resp) { 23910656Sandreas.hansson@arm.com // we should never have an exsiting request outstanding 24010656Sandreas.hansson@arm.com assert(outstandingSnoop.find(pkt->req) == 24110656Sandreas.hansson@arm.com outstandingSnoop.end()); 24210656Sandreas.hansson@arm.com outstandingSnoop.insert(pkt->req); 2438948SN/A 24410656Sandreas.hansson@arm.com // basic sanity check on the outstanding snoops 24510656Sandreas.hansson@arm.com panic_if(outstandingSnoop.size() > 512, 24610656Sandreas.hansson@arm.com "Outstanding snoop requests exceeded 512\n"); 24710656Sandreas.hansson@arm.com } 2488948SN/A 24910656Sandreas.hansson@arm.com // remember where to route the normal response to 25010656Sandreas.hansson@arm.com if (expect_response || expect_snoop_resp) { 25110656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 25210656Sandreas.hansson@arm.com routeTo[pkt->req] = slave_port_id; 2539549SN/A 25410656Sandreas.hansson@arm.com panic_if(routeTo.size() > 512, 25510656Sandreas.hansson@arm.com "Routing table exceeds 512 packets\n"); 25610656Sandreas.hansson@arm.com } 2578948SN/A 25810405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 2599715SN/A reqLayers[master_port_id]->succeededTiming(packetFinishTime); 2609091SN/A } 2618975SN/A 26210656Sandreas.hansson@arm.com // stats updates only consider packets that were successfully sent 2639712SN/A pktCount[slave_port_id][master_port_id]++; 26410405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 2659712SN/A transDist[pkt_cmd]++; 26610656Sandreas.hansson@arm.com 26710656Sandreas.hansson@arm.com if (is_express_snoop) 26810656Sandreas.hansson@arm.com snoops++; 2699712SN/A } 2709712SN/A 2719091SN/A return success; 2728975SN/A} 2738975SN/A 2748975SN/Abool 27510405Sandreas.hansson@arm.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 2768975SN/A{ 2778975SN/A // determine the source port based on the id 2789032SN/A MasterPort *src_port = masterPorts[master_port_id]; 2798975SN/A 28010656Sandreas.hansson@arm.com // determine the destination 28110656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 28210656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 28310656Sandreas.hansson@arm.com const PortID slave_port_id = route_lookup->second; 28410572Sandreas.hansson@arm.com assert(slave_port_id != InvalidPortID); 28510572Sandreas.hansson@arm.com assert(slave_port_id < respLayers.size()); 2869713SN/A 28710405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 28810405Sandreas.hansson@arm.com // current port 2899715SN/A if (!respLayers[slave_port_id]->tryTiming(src_port)) { 29010405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 2918975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 2928975SN/A return false; 2938975SN/A } 2948975SN/A 29510405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 2968975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 2978975SN/A 2989712SN/A // store size and command as they might be modified when 2999712SN/A // forwarding the packet 3009712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 3019712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 3029712SN/A 3038975SN/A calcPacketTiming(pkt); 30410694SMarco.Balboni@ARM.com Tick packetFinishTime = curTick() + pkt->payloadDelay; 3058975SN/A 30610402SN/A if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 30710402SN/A // let the snoop filter inspect the response and update its state 30810402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 30910402SN/A } 31010402SN/A 3119712SN/A // send the packet through the destination slave port 3129712SN/A bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 3138975SN/A 3148975SN/A // currently it is illegal to block responses... can lead to 3158975SN/A // deadlock 3168975SN/A assert(success); 3178975SN/A 31810656Sandreas.hansson@arm.com // remove the request from the routing table 31910656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 32010656Sandreas.hansson@arm.com 3219715SN/A respLayers[slave_port_id]->succeededTiming(packetFinishTime); 3228975SN/A 3239712SN/A // stats updates 3249712SN/A pktCount[slave_port_id][master_port_id]++; 32510405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 3269712SN/A transDist[pkt_cmd]++; 3279712SN/A 3288975SN/A return true; 3298975SN/A} 3308975SN/A 3318975SN/Avoid 33210405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 3338975SN/A{ 33410405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 3359032SN/A masterPorts[master_port_id]->name(), pkt->cmdString(), 3368975SN/A pkt->getAddr()); 3378975SN/A 3389712SN/A // update stats here as we know the forwarding will succeed 3399712SN/A transDist[pkt->cmdToIndex()]++; 34010405Sandreas.hansson@arm.com snoops++; 3419712SN/A 3428975SN/A // we should only see express snoops from caches 3438975SN/A assert(pkt->isExpressSnoop()); 3448975SN/A 34510656Sandreas.hansson@arm.com // remeber if the packet is inhibited so we can see if it changes 34610656Sandreas.hansson@arm.com const bool is_inhibited = pkt->memInhibitAsserted(); 3479032SN/A 34810402SN/A if (snoopFilter) { 34910402SN/A // let the Snoop Filter work its magic and guide probing 35010402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 35110402SN/A // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 35210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 35310402SN/A " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 35410402SN/A pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 35510402SN/A sf_res.second); 35610402SN/A 35710402SN/A // forward to all snoopers 35810402SN/A forwardTiming(pkt, InvalidPortID, sf_res.first); 35910402SN/A } else { 36010402SN/A forwardTiming(pkt, InvalidPortID); 36110402SN/A } 3628975SN/A 36310656Sandreas.hansson@arm.com // if we can expect a response, remember how to route it 36410656Sandreas.hansson@arm.com if (!is_inhibited && pkt->memInhibitAsserted()) { 36510656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 36610656Sandreas.hansson@arm.com routeTo[pkt->req] = master_port_id; 36710656Sandreas.hansson@arm.com } 36810656Sandreas.hansson@arm.com 3698975SN/A // a snoop request came from a connected slave device (one of 3708975SN/A // our master ports), and if it is not coming from the slave 3718975SN/A // device responsible for the address range something is 3728975SN/A // wrong, hence there is nothing further to do as the packet 3738975SN/A // would be going back to where it came from 3749032SN/A assert(master_port_id == findPort(pkt->getAddr())); 3758975SN/A} 3768975SN/A 3778975SN/Abool 37810405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 3798975SN/A{ 3808975SN/A // determine the source port based on the id 3819032SN/A SlavePort* src_port = slavePorts[slave_port_id]; 3828975SN/A 38310656Sandreas.hansson@arm.com // get the destination 38410656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 38510656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 38610656Sandreas.hansson@arm.com const PortID dest_port_id = route_lookup->second; 38710572Sandreas.hansson@arm.com assert(dest_port_id != InvalidPortID); 3889714SN/A 3899714SN/A // determine if the response is from a snoop request we 3909714SN/A // created as the result of a normal request (in which case it 39110656Sandreas.hansson@arm.com // should be in the outstandingSnoop), or if we merely forwarded 3929714SN/A // someone else's snoop request 39310656Sandreas.hansson@arm.com const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 39410656Sandreas.hansson@arm.com outstandingSnoop.end(); 3959714SN/A 39610405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 39710405Sandreas.hansson@arm.com // current port, note that the check is bypassed if the response 39810405Sandreas.hansson@arm.com // is being passed on as a normal response since this is occupying 39910405Sandreas.hansson@arm.com // the response layer rather than the snoop response layer 4009715SN/A if (forwardAsSnoop) { 40110572Sandreas.hansson@arm.com assert(dest_port_id < snoopLayers.size()); 4029715SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 40310405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 4049715SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 4059715SN/A return false; 4069715SN/A } 4079716SN/A } else { 4089716SN/A // get the master port that mirrors this slave port internally 4099716SN/A MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 41010572Sandreas.hansson@arm.com assert(dest_port_id < respLayers.size()); 4119716SN/A if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 41210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 4139716SN/A snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 4149716SN/A return false; 4159716SN/A } 4168975SN/A } 4178975SN/A 41810405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 4198975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 4208975SN/A 4219712SN/A // store size and command as they might be modified when 4229712SN/A // forwarding the packet 4239712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 4249712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 4259712SN/A 4268975SN/A // responses are never express snoops 4278975SN/A assert(!pkt->isExpressSnoop()); 4288975SN/A 4298975SN/A calcPacketTiming(pkt); 43010694SMarco.Balboni@ARM.com Tick packetFinishTime = curTick() + pkt->payloadDelay; 4318975SN/A 4329714SN/A // forward it either as a snoop response or a normal response 4339714SN/A if (forwardAsSnoop) { 4349714SN/A // this is a snoop response to a snoop request we forwarded, 4359714SN/A // e.g. coming from the L1 and going to the L2, and it should 4369714SN/A // be forwarded as a snoop response 43710402SN/A 43810402SN/A if (snoopFilter) { 43910402SN/A // update the probe filter so that it can properly track the line 44010402SN/A snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 44110402SN/A *masterPorts[dest_port_id]); 44210402SN/A } 44310402SN/A 4449712SN/A bool success M5_VAR_USED = 4459712SN/A masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 4469712SN/A pktCount[slave_port_id][dest_port_id]++; 44710405Sandreas.hansson@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 4488975SN/A assert(success); 4499714SN/A 4509715SN/A snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 4513244SN/A } else { 4528975SN/A // we got a snoop response on one of our slave ports, 45310405Sandreas.hansson@arm.com // i.e. from a coherent master connected to the crossbar, and 45410405Sandreas.hansson@arm.com // since we created the snoop request as part of recvTiming, 45510405Sandreas.hansson@arm.com // this should now be a normal response again 45610656Sandreas.hansson@arm.com outstandingSnoop.erase(pkt->req); 4578948SN/A 45810656Sandreas.hansson@arm.com // this is a snoop response from a coherent master, hence it 45910656Sandreas.hansson@arm.com // should never go back to where the snoop response came from, 46010656Sandreas.hansson@arm.com // but instead to where the original request came from 4619712SN/A assert(slave_port_id != dest_port_id); 4628948SN/A 46310402SN/A if (snoopFilter) { 46410402SN/A // update the probe filter so that it can properly track the line 46510402SN/A snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 46610402SN/A *slavePorts[dest_port_id]); 46710402SN/A } 46810402SN/A 46910405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 47010402SN/A " FWD RESP\n", src_port->name(), pkt->cmdString(), 47110402SN/A pkt->getAddr()); 47210402SN/A 4739714SN/A // as a normal response, it should go back to a master through 4749714SN/A // one of our slave ports, at this point we are ignoring the 4759714SN/A // fact that the response layer could be busy and do not touch 4769714SN/A // its state 4779712SN/A bool success M5_VAR_USED = 4789712SN/A slavePorts[dest_port_id]->sendTimingResp(pkt); 4798975SN/A 4809714SN/A // @todo Put the response in an internal FIFO and pass it on 4819714SN/A // to the response layer from there 4829714SN/A 4838975SN/A // currently it is illegal to block responses... can lead 4848975SN/A // to deadlock 4858948SN/A assert(success); 4869716SN/A 4879716SN/A respLayers[dest_port_id]->succeededTiming(packetFinishTime); 4883244SN/A } 4893244SN/A 49010656Sandreas.hansson@arm.com // remove the request from the routing table 49110656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 49210656Sandreas.hansson@arm.com 4939712SN/A // stats updates 4949712SN/A transDist[pkt_cmd]++; 49510405Sandreas.hansson@arm.com snoops++; 4969712SN/A 4978948SN/A return true; 4988948SN/A} 4998948SN/A 5003210SN/A 5018948SN/Avoid 50210405Sandreas.hansson@arm.comCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 50310402SN/A const std::vector<SlavePort*>& dests) 5048948SN/A{ 50510405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 5069663SN/A pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 5079663SN/A 5089524SN/A // snoops should only happen if the system isn't bypassing caches 5099524SN/A assert(!system->bypassCaches()); 5109524SN/A 51110401SN/A unsigned fanout = 0; 51210401SN/A 51310405Sandreas.hansson@arm.com for (const auto& p: dests) { 5148948SN/A // we could have gotten this request from a snooping master 5158948SN/A // (corresponding to our own slave port that is also in 5168948SN/A // snoopPorts) and should not send it back to where it came 5178948SN/A // from 5189031SN/A if (exclude_slave_port_id == InvalidPortID || 5198948SN/A p->getId() != exclude_slave_port_id) { 5208948SN/A // cache is not allowed to refuse snoop 5218975SN/A p->sendTimingSnoopReq(pkt); 52210401SN/A fanout++; 5238948SN/A } 5248948SN/A } 52510401SN/A 52610401SN/A // Stats for fanout of this forward operation 52710401SN/A snoopFanout.sample(fanout); 5282497SN/A} 5292497SN/A 5309092SN/Avoid 53110713Sandreas.hansson@arm.comCoherentXBar::recvReqRetry(PortID master_port_id) 5329092SN/A{ 5339093SN/A // responses and snoop responses never block on forwarding them, 5349093SN/A // so the retry will always be coming from a port to which we 5359093SN/A // tried to forward a request 5369715SN/A reqLayers[master_port_id]->recvRetry(); 5379092SN/A} 5389092SN/A 5399036SN/ATick 54010405Sandreas.hansson@arm.comCoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 5412657SN/A{ 54210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 5439032SN/A slavePorts[slave_port_id]->name(), pkt->getAddr(), 5448949SN/A pkt->cmdString()); 5458915SN/A 54610405Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 54710405Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 5489712SN/A 5498979SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 5508979SN/A Tick snoop_response_latency = 0; 5518979SN/A 5528979SN/A // uncacheable requests need never be snooped 5539524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 5548979SN/A // forward to all snoopers but the source 55510402SN/A std::pair<MemCmd, Tick> snoop_result; 55610402SN/A if (snoopFilter) { 55710402SN/A // check with the snoop filter where to forward this packet 55810402SN/A auto sf_res = 55910402SN/A snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 56010402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 56110405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 56210402SN/A " SF size: %i lat: %i\n", __func__, 56310402SN/A slavePorts[slave_port_id]->name(), pkt->cmdString(), 56410402SN/A pkt->getAddr(), sf_res.first.size(), sf_res.second); 56510402SN/A snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 56610402SN/A sf_res.first); 56710402SN/A } else { 56810402SN/A snoop_result = forwardAtomic(pkt, slave_port_id); 56910402SN/A } 5708979SN/A snoop_response_cmd = snoop_result.first; 57110402SN/A snoop_response_latency += snoop_result.second; 5728979SN/A } 5738915SN/A 5748948SN/A // even if we had a snoop response, we must continue and also 5758948SN/A // perform the actual request at the destination 57610405Sandreas.hansson@arm.com PortID master_port_id = findPort(pkt->getAddr()); 57710405Sandreas.hansson@arm.com 57810405Sandreas.hansson@arm.com // stats updates for the request 57910405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 58010405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 58110405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 5828948SN/A 5838948SN/A // forward the request to the appropriate destination 58410405Sandreas.hansson@arm.com Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 5858948SN/A 58610402SN/A // Lower levels have replied, tell the snoop filter 58710402SN/A if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 58810402SN/A pkt->isResponse()) { 58910402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 59010402SN/A } 59110402SN/A 5928948SN/A // if we got a response from a snooper, restore it here 5938948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) { 5948948SN/A // no one else should have responded 5958948SN/A assert(!pkt->isResponse()); 5968948SN/A pkt->cmd = snoop_response_cmd; 5978948SN/A response_latency = snoop_response_latency; 5988948SN/A } 5998948SN/A 6009712SN/A // add the response data 60110405Sandreas.hansson@arm.com if (pkt->isResponse()) { 60210405Sandreas.hansson@arm.com pkt_size = pkt->hasData() ? pkt->getSize() : 0; 60310405Sandreas.hansson@arm.com pkt_cmd = pkt->cmdToIndex(); 60410405Sandreas.hansson@arm.com 60510405Sandreas.hansson@arm.com // stats updates 60610405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 60710405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 60810405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 60910405Sandreas.hansson@arm.com } 6109712SN/A 61110694SMarco.Balboni@ARM.com // @todo: Not setting header time 61210694SMarco.Balboni@ARM.com pkt->payloadDelay = response_latency; 6138948SN/A return response_latency; 6148948SN/A} 6158948SN/A 6168948SN/ATick 61710405Sandreas.hansson@arm.comCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 6188948SN/A{ 61910405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 6209032SN/A masterPorts[master_port_id]->name(), pkt->getAddr(), 6218949SN/A pkt->cmdString()); 6228948SN/A 6239712SN/A // add the request snoop data 62410405Sandreas.hansson@arm.com snoops++; 6259712SN/A 6268948SN/A // forward to all snoopers 62710402SN/A std::pair<MemCmd, Tick> snoop_result; 62810402SN/A Tick snoop_response_latency = 0; 62910402SN/A if (snoopFilter) { 63010402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 63110402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 63210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 63310402SN/A __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 63410402SN/A pkt->getAddr(), sf_res.first.size(), sf_res.second); 63510402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 63610402SN/A sf_res.first); 63710402SN/A } else { 63810402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID); 63910402SN/A } 6408948SN/A MemCmd snoop_response_cmd = snoop_result.first; 64110402SN/A snoop_response_latency += snoop_result.second; 6428948SN/A 6438948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) 6448948SN/A pkt->cmd = snoop_response_cmd; 6458948SN/A 6469712SN/A // add the response snoop data 64710401SN/A if (pkt->isResponse()) { 64810405Sandreas.hansson@arm.com snoops++; 64910401SN/A } 6509712SN/A 65110694SMarco.Balboni@ARM.com // @todo: Not setting header time 65210694SMarco.Balboni@ARM.com pkt->payloadDelay = snoop_response_latency; 6538948SN/A return snoop_response_latency; 6548948SN/A} 6558948SN/A 6568948SN/Astd::pair<MemCmd, Tick> 65710405Sandreas.hansson@arm.comCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 65810402SN/A PortID source_master_port_id, 65910402SN/A const std::vector<SlavePort*>& dests) 6608948SN/A{ 6619032SN/A // the packet may be changed on snoops, record the original 6629032SN/A // command to enable us to restore it between snoops so that 6638948SN/A // additional snoops can take place properly 6644626SN/A MemCmd orig_cmd = pkt->cmd; 6654879SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 6664879SN/A Tick snoop_response_latency = 0; 6673662SN/A 6689524SN/A // snoops should only happen if the system isn't bypassing caches 6699524SN/A assert(!system->bypassCaches()); 6709524SN/A 67110401SN/A unsigned fanout = 0; 67210401SN/A 67310405Sandreas.hansson@arm.com for (const auto& p: dests) { 6748915SN/A // we could have gotten this request from a snooping master 6758915SN/A // (corresponding to our own slave port that is also in 6768915SN/A // snoopPorts) and should not send it back to where it came 6778915SN/A // from 67810402SN/A if (exclude_slave_port_id != InvalidPortID && 67910402SN/A p->getId() == exclude_slave_port_id) 68010402SN/A continue; 68110401SN/A 68210402SN/A Tick latency = p->sendAtomicSnoop(pkt); 68310402SN/A fanout++; 68410402SN/A 68510402SN/A // in contrast to a functional access, we have to keep on 68610402SN/A // going as all snoopers must be updated even if we get a 68710402SN/A // response 68810402SN/A if (!pkt->isResponse()) 68910402SN/A continue; 69010402SN/A 69110402SN/A // response from snoop agent 69210402SN/A assert(pkt->cmd != orig_cmd); 69310402SN/A assert(pkt->memInhibitAsserted()); 69410402SN/A // should only happen once 69510402SN/A assert(snoop_response_cmd == MemCmd::InvalidCmd); 69610402SN/A // save response state 69710402SN/A snoop_response_cmd = pkt->cmd; 69810402SN/A snoop_response_latency = latency; 69910402SN/A 70010402SN/A if (snoopFilter) { 70110402SN/A // Handle responses by the snoopers and differentiate between 70210402SN/A // responses to requests from above and snoops from below 70310402SN/A if (source_master_port_id != InvalidPortID) { 70410402SN/A // Getting a response for a snoop from below 70510402SN/A assert(exclude_slave_port_id == InvalidPortID); 70610402SN/A snoopFilter->updateSnoopForward(pkt, *p, 70710402SN/A *masterPorts[source_master_port_id]); 70810402SN/A } else { 70910402SN/A // Getting a response for a request from above 71010402SN/A assert(source_master_port_id == InvalidPortID); 71110402SN/A snoopFilter->updateSnoopResponse(pkt, *p, 71210402SN/A *slavePorts[exclude_slave_port_id]); 7134626SN/A } 7144626SN/A } 71510402SN/A // restore original packet state for remaining snoopers 71610402SN/A pkt->cmd = orig_cmd; 7174626SN/A } 7184626SN/A 71910401SN/A // Stats for fanout 72010401SN/A snoopFanout.sample(fanout); 72110401SN/A 7228948SN/A // the packet is restored as part of the loop and any potential 7238948SN/A // snoop response is part of the returned pair 7248948SN/A return std::make_pair(snoop_response_cmd, snoop_response_latency); 7252497SN/A} 7262497SN/A 7272497SN/Avoid 72810405Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 7292497SN/A{ 7308663SN/A if (!pkt->isPrint()) { 7318663SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 73210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, 7338949SN/A "recvFunctional: packet src %s addr 0x%x cmd %s\n", 7349032SN/A slavePorts[slave_port_id]->name(), pkt->getAddr(), 7358663SN/A pkt->cmdString()); 7368663SN/A } 7378663SN/A 7388979SN/A // uncacheable requests need never be snooped 7399524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 7408979SN/A // forward to all snoopers but the source 7419032SN/A forwardFunctional(pkt, slave_port_id); 7428979SN/A } 7434912SN/A 7448948SN/A // there is no need to continue if the snooping has found what we 7458948SN/A // were looking for and the packet is already a response 7468948SN/A if (!pkt->isResponse()) { 7479031SN/A PortID dest_id = findPort(pkt->getAddr()); 7488948SN/A 7498948SN/A masterPorts[dest_id]->sendFunctional(pkt); 7508948SN/A } 7518948SN/A} 7528948SN/A 7538948SN/Avoid 75410405Sandreas.hansson@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 7558948SN/A{ 7568948SN/A if (!pkt->isPrint()) { 7578948SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 75810405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, 7598949SN/A "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 7609032SN/A masterPorts[master_port_id]->name(), pkt->getAddr(), 7618948SN/A pkt->cmdString()); 7628948SN/A } 7638948SN/A 7648948SN/A // forward to all snoopers 7659031SN/A forwardFunctional(pkt, InvalidPortID); 7668948SN/A} 7678948SN/A 7688948SN/Avoid 76910405Sandreas.hansson@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 7708948SN/A{ 7719524SN/A // snoops should only happen if the system isn't bypassing caches 7729524SN/A assert(!system->bypassCaches()); 7739524SN/A 77410405Sandreas.hansson@arm.com for (const auto& p: snoopPorts) { 7758915SN/A // we could have gotten this request from a snooping master 7768915SN/A // (corresponding to our own slave port that is also in 7778915SN/A // snoopPorts) and should not send it back to where it came 7788915SN/A // from 7799031SN/A if (exclude_slave_port_id == InvalidPortID || 7808948SN/A p->getId() != exclude_slave_port_id) 7818948SN/A p->sendFunctionalSnoop(pkt); 7828915SN/A 7838948SN/A // if we get a response we are done 7848948SN/A if (pkt->isResponse()) { 7858948SN/A break; 7868915SN/A } 7873650SN/A } 7882497SN/A} 7892497SN/A 7909092SN/Aunsigned int 79110405Sandreas.hansson@arm.comCoherentXBar::drain(DrainManager *dm) 7929092SN/A{ 7939093SN/A // sum up the individual layers 7949715SN/A unsigned int total = 0; 79510405Sandreas.hansson@arm.com for (auto l: reqLayers) 79610405Sandreas.hansson@arm.com total += l->drain(dm); 79710405Sandreas.hansson@arm.com for (auto l: respLayers) 79810405Sandreas.hansson@arm.com total += l->drain(dm); 79910405Sandreas.hansson@arm.com for (auto l: snoopLayers) 80010405Sandreas.hansson@arm.com total += l->drain(dm); 8019715SN/A return total; 8029092SN/A} 8039092SN/A 8049712SN/Avoid 80510405Sandreas.hansson@arm.comCoherentXBar::regStats() 8069712SN/A{ 80710405Sandreas.hansson@arm.com // register the stats of the base class and our layers 80810405Sandreas.hansson@arm.com BaseXBar::regStats(); 80910405Sandreas.hansson@arm.com for (auto l: reqLayers) 81010405Sandreas.hansson@arm.com l->regStats(); 81110405Sandreas.hansson@arm.com for (auto l: respLayers) 81210405Sandreas.hansson@arm.com l->regStats(); 81310405Sandreas.hansson@arm.com for (auto l: snoopLayers) 81410405Sandreas.hansson@arm.com l->regStats(); 8159712SN/A 81610405Sandreas.hansson@arm.com snoops 81710405Sandreas.hansson@arm.com .name(name() + ".snoops") 81810401SN/A .desc("Total snoops (count)") 81910401SN/A ; 82010401SN/A 82110401SN/A snoopFanout 82210401SN/A .init(0, snoopPorts.size(), 1) 82310401SN/A .name(name() + ".snoop_fanout") 82410401SN/A .desc("Request fanout histogram") 82510401SN/A ; 8269712SN/A} 8279712SN/A 82810405Sandreas.hansson@arm.comCoherentXBar * 82910405Sandreas.hansson@arm.comCoherentXBarParams::create() 8302497SN/A{ 83110405Sandreas.hansson@arm.com return new CoherentXBar(this); 8322497SN/A} 833