coherent_xbar.cc revision 10405
12497SN/A/* 210405Sandreas.hansson@arm.com * Copyright (c) 2011-2014 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142497SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152497SN/A * All rights reserved. 162497SN/A * 172497SN/A * Redistribution and use in source and binary forms, with or without 182497SN/A * modification, are permitted provided that the following conditions are 192497SN/A * met: redistributions of source code must retain the above copyright 202497SN/A * notice, this list of conditions and the following disclaimer; 212497SN/A * redistributions in binary form must reproduce the above copyright 222497SN/A * notice, this list of conditions and the following disclaimer in the 232497SN/A * documentation and/or other materials provided with the distribution; 242497SN/A * neither the name of the copyright holders nor the names of its 252497SN/A * contributors may be used to endorse or promote products derived from 262497SN/A * this software without specific prior written permission. 272497SN/A * 282497SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292497SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302497SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312497SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322497SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332497SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342497SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352497SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362497SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372497SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382497SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 418715SN/A * Andreas Hansson 428922SN/A * William Wang 432497SN/A */ 442497SN/A 452497SN/A/** 462982SN/A * @file 4710405Sandreas.hansson@arm.com * Definition of a crossbar object. 482497SN/A */ 492497SN/A 502846SN/A#include "base/misc.hh" 512548SN/A#include "base/trace.hh" 5210405Sandreas.hansson@arm.com#include "debug/AddrRanges.hh" 5310405Sandreas.hansson@arm.com#include "debug/CoherentXBar.hh" 5410405Sandreas.hansson@arm.com#include "mem/coherent_xbar.hh" 559524SN/A#include "sim/system.hh" 562497SN/A 5710405Sandreas.hansson@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 5810405Sandreas.hansson@arm.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 597523SN/A{ 608851SN/A // create the ports based on the size of the master and slave 618948SN/A // vector ports, and the presence of the default port, the ports 628948SN/A // are enumerated starting from zero 638851SN/A for (int i = 0; i < p->port_master_connection_count; ++i) { 649095SN/A std::string portName = csprintf("%s.master[%d]", name(), i); 6510405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 668922SN/A masterPorts.push_back(bp); 679715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 689715SN/A csprintf(".reqLayer%d", i))); 699715SN/A snoopLayers.push_back(new SnoopLayer(*bp, *this, 709715SN/A csprintf(".snoopLayer%d", i))); 718851SN/A } 728851SN/A 738948SN/A // see if we have a default slave device connected and if so add 748948SN/A // our corresponding master port 758915SN/A if (p->port_default_connection_count) { 769031SN/A defaultPortID = masterPorts.size(); 779095SN/A std::string portName = name() + ".default"; 7810405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 799036SN/A defaultPortID); 808922SN/A masterPorts.push_back(bp); 819715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 829715SN/A defaultPortID))); 839715SN/A snoopLayers.push_back(new SnoopLayer(*bp, *this, 849715SN/A csprintf(".snoopLayer%d", 859715SN/A defaultPortID))); 868915SN/A } 878915SN/A 888948SN/A // create the slave ports, once again starting at zero 898851SN/A for (int i = 0; i < p->port_slave_connection_count; ++i) { 909095SN/A std::string portName = csprintf("%s.slave[%d]", name(), i); 9110405Sandreas.hansson@arm.com SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 928922SN/A slavePorts.push_back(bp); 939715SN/A respLayers.push_back(new RespLayer(*bp, *this, 949715SN/A csprintf(".respLayer%d", i))); 959716SN/A snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 968851SN/A } 978851SN/A 9810402SN/A if (snoopFilter) 9910402SN/A snoopFilter->setSlavePorts(slavePorts); 10010402SN/A 1017523SN/A clearPortCache(); 1027523SN/A} 1037523SN/A 10410405Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 1059715SN/A{ 10610405Sandreas.hansson@arm.com for (auto l: reqLayers) 10710405Sandreas.hansson@arm.com delete l; 10810405Sandreas.hansson@arm.com for (auto l: respLayers) 10910405Sandreas.hansson@arm.com delete l; 11010405Sandreas.hansson@arm.com for (auto l: snoopLayers) 11110405Sandreas.hansson@arm.com delete l; 11210405Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11310405Sandreas.hansson@arm.com delete p; 1149715SN/A} 1159715SN/A 1162568SN/Avoid 11710405Sandreas.hansson@arm.comCoherentXBar::init() 1182568SN/A{ 1199278SN/A // the base class is responsible for determining the block size 12010405Sandreas.hansson@arm.com BaseXBar::init(); 1219278SN/A 1228948SN/A // iterate over our slave ports and determine which of our 1238948SN/A // neighbouring master ports are snooping and add them as snoopers 12410405Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 1259088SN/A // check if the connected master port is snooping 12610405Sandreas.hansson@arm.com if (p->isSnooping()) { 12710405Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12810405Sandreas.hansson@arm.com p->getMasterPort().name()); 12910405Sandreas.hansson@arm.com snoopPorts.push_back(p); 1308711SN/A } 1318711SN/A } 1322568SN/A 1339036SN/A if (snoopPorts.empty()) 13410405Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 1353244SN/A} 1363244SN/A 1378948SN/Abool 13810405Sandreas.hansson@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1393244SN/A{ 1408975SN/A // determine the source port based on the id 1419032SN/A SlavePort *src_port = slavePorts[slave_port_id]; 1423244SN/A 1439091SN/A // remember if the packet is an express snoop 1449091SN/A bool is_express_snoop = pkt->isExpressSnoop(); 1459091SN/A 1469612SN/A // determine the destination based on the address 1479712SN/A PortID master_port_id = findPort(pkt->getAddr()); 1489612SN/A 14910405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the current 1509033SN/A // port, and exclude express snoops from the check 1519715SN/A if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 15210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 1538949SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 1543244SN/A return false; 1553244SN/A } 1563244SN/A 15710405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 1589091SN/A src_port->name(), pkt->cmdString(), is_express_snoop, 1599091SN/A pkt->getAddr()); 1605197SN/A 1619712SN/A // store size and command as they might be modified when 1629712SN/A // forwarding the packet 1639712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1649712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 1659712SN/A 1669032SN/A // set the source port for routing of the response 1679032SN/A pkt->setSrc(slave_port_id); 1689032SN/A 1699547SN/A calcPacketTiming(pkt); 17010405Sandreas.hansson@arm.com Tick packetFinishTime = pkt->lastWordDelay + curTick(); 1714912SN/A 1728979SN/A // uncacheable requests need never be snooped 1739524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 1748979SN/A // the packet is a memory-mapped request and should be 1758979SN/A // broadcasted to our snoopers but the source 17610402SN/A if (snoopFilter) { 17710402SN/A // check with the snoop filter where to forward this packet 17810402SN/A auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 17910402SN/A packetFinishTime += sf_res.second * clockPeriod(); 18010405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 18110402SN/A " SF size: %i lat: %i\n", src_port->name(), 18210402SN/A pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 18310402SN/A sf_res.second); 18410402SN/A forwardTiming(pkt, slave_port_id, sf_res.first); 18510402SN/A } else { 18610402SN/A forwardTiming(pkt, slave_port_id); 18710402SN/A } 1888979SN/A } 1898948SN/A 1908975SN/A // remember if we add an outstanding req so we can undo it if 1918975SN/A // necessary, if the packet needs a response, we should add it 1928975SN/A // as outstanding and express snoops never fail so there is 1938975SN/A // not need to worry about them 1949091SN/A bool add_outstanding = !is_express_snoop && pkt->needsResponse(); 1958948SN/A 1968975SN/A // keep track that we have an outstanding request packet 1978975SN/A // matching this request, this is used by the coherency 1988975SN/A // mechanism in determining what to do with snoop responses 1998975SN/A // (in recvTimingSnoop) 2008975SN/A if (add_outstanding) { 2018975SN/A // we should never have an exsiting request outstanding 2028975SN/A assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 2038975SN/A outstandingReq.insert(pkt->req); 2048975SN/A } 2058915SN/A 20610402SN/A // Note: Cannot create a copy of the full packet, here. 20710402SN/A MemCmd orig_cmd(pkt->cmd); 20810402SN/A 2099612SN/A // since it is a normal request, attempt to send the packet 2109712SN/A bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 2118948SN/A 21210402SN/A if (snoopFilter && !pkt->req->isUncacheable() 21310402SN/A && !system->bypassCaches()) { 21410402SN/A // The packet may already be overwritten by the sendTimingReq function. 21510402SN/A // The snoop filter needs to see the original request *and* the return 21610402SN/A // status of the send operation, so we need to recreate the original 21710402SN/A // request. Atomic mode does not have the issue, as there the send 21810402SN/A // operation and the response happen instantaneously and don't need two 21910402SN/A // phase tracking. 22010402SN/A MemCmd tmp_cmd(pkt->cmd); 22110402SN/A pkt->cmd = orig_cmd; 22210402SN/A // Let the snoop filter know about the success of the send operation 22310402SN/A snoopFilter->updateRequest(pkt, *src_port, !success); 22410402SN/A pkt->cmd = tmp_cmd; 22510402SN/A } 22610402SN/A 2279091SN/A // if this is an express snoop, we are done at this point 2289091SN/A if (is_express_snoop) { 2299091SN/A assert(success); 23010405Sandreas.hansson@arm.com snoops++; 2319091SN/A } else { 2329091SN/A // for normal requests, check if successful 2339091SN/A if (!success) { 2349091SN/A // inhibited packets should never be forced to retry 2359091SN/A assert(!pkt->memInhibitAsserted()); 2368948SN/A 2379091SN/A // if it was added as outstanding and the send failed, then 2389091SN/A // erase it again 2399091SN/A if (add_outstanding) 2409091SN/A outstandingReq.erase(pkt->req); 2418948SN/A 2429549SN/A // undo the calculation so we can check for 0 again 24310405Sandreas.hansson@arm.com pkt->firstWordDelay = pkt->lastWordDelay = 0; 2449549SN/A 24510405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 2469091SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 2478948SN/A 24810405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 2499715SN/A reqLayers[master_port_id]->failedTiming(src_port, 2509715SN/A clockEdge(headerCycles)); 2519091SN/A } else { 25210405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 2539715SN/A reqLayers[master_port_id]->succeededTiming(packetFinishTime); 2549091SN/A } 2558975SN/A } 2568975SN/A 2579712SN/A // stats updates only consider packets that were successfully sent 2589712SN/A if (success) { 2599712SN/A pktCount[slave_port_id][master_port_id]++; 26010405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 2619712SN/A transDist[pkt_cmd]++; 2629712SN/A } 2639712SN/A 2649091SN/A return success; 2658975SN/A} 2668975SN/A 2678975SN/Abool 26810405Sandreas.hansson@arm.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 2698975SN/A{ 2708975SN/A // determine the source port based on the id 2719032SN/A MasterPort *src_port = masterPorts[master_port_id]; 2728975SN/A 2739713SN/A // determine the destination based on what is stored in the packet 2749713SN/A PortID slave_port_id = pkt->getDest(); 2759713SN/A 27610405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 27710405Sandreas.hansson@arm.com // current port 2789715SN/A if (!respLayers[slave_port_id]->tryTiming(src_port)) { 27910405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 2808975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 2818975SN/A return false; 2828975SN/A } 2838975SN/A 28410405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 2858975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 2868975SN/A 2879712SN/A // store size and command as they might be modified when 2889712SN/A // forwarding the packet 2899712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 2909712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 2919712SN/A 2928975SN/A calcPacketTiming(pkt); 29310405Sandreas.hansson@arm.com Tick packetFinishTime = pkt->lastWordDelay + curTick(); 2948975SN/A 2958975SN/A // the packet is a normal response to a request that we should 29610405Sandreas.hansson@arm.com // have seen passing through the crossbar 2978975SN/A assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 2988975SN/A 29910402SN/A if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 30010402SN/A // let the snoop filter inspect the response and update its state 30110402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 30210402SN/A } 30310402SN/A 3048975SN/A // remove it as outstanding 3058975SN/A outstandingReq.erase(pkt->req); 3068975SN/A 3079712SN/A // send the packet through the destination slave port 3089712SN/A bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 3098975SN/A 3108975SN/A // currently it is illegal to block responses... can lead to 3118975SN/A // deadlock 3128975SN/A assert(success); 3138975SN/A 3149715SN/A respLayers[slave_port_id]->succeededTiming(packetFinishTime); 3158975SN/A 3169712SN/A // stats updates 3179712SN/A pktCount[slave_port_id][master_port_id]++; 31810405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 3199712SN/A transDist[pkt_cmd]++; 3209712SN/A 3218975SN/A return true; 3228975SN/A} 3238975SN/A 3248975SN/Avoid 32510405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 3268975SN/A{ 32710405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 3289032SN/A masterPorts[master_port_id]->name(), pkt->cmdString(), 3298975SN/A pkt->getAddr()); 3308975SN/A 3319712SN/A // update stats here as we know the forwarding will succeed 3329712SN/A transDist[pkt->cmdToIndex()]++; 33310405Sandreas.hansson@arm.com snoops++; 3349712SN/A 3358975SN/A // we should only see express snoops from caches 3368975SN/A assert(pkt->isExpressSnoop()); 3378975SN/A 3389032SN/A // set the source port for routing of the response 3399032SN/A pkt->setSrc(master_port_id); 3409032SN/A 34110402SN/A if (snoopFilter) { 34210402SN/A // let the Snoop Filter work its magic and guide probing 34310402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 34410402SN/A // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 34510405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 34610402SN/A " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 34710402SN/A pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 34810402SN/A sf_res.second); 34910402SN/A 35010402SN/A // forward to all snoopers 35110402SN/A forwardTiming(pkt, InvalidPortID, sf_res.first); 35210402SN/A } else { 35310402SN/A forwardTiming(pkt, InvalidPortID); 35410402SN/A } 3558975SN/A 3568975SN/A // a snoop request came from a connected slave device (one of 3578975SN/A // our master ports), and if it is not coming from the slave 3588975SN/A // device responsible for the address range something is 3598975SN/A // wrong, hence there is nothing further to do as the packet 3608975SN/A // would be going back to where it came from 3619032SN/A assert(master_port_id == findPort(pkt->getAddr())); 3628975SN/A} 3638975SN/A 3648975SN/Abool 36510405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 3668975SN/A{ 3678975SN/A // determine the source port based on the id 3689032SN/A SlavePort* src_port = slavePorts[slave_port_id]; 3698975SN/A 3709714SN/A // get the destination from the packet 3719714SN/A PortID dest_port_id = pkt->getDest(); 3729714SN/A 3739714SN/A // determine if the response is from a snoop request we 3749714SN/A // created as the result of a normal request (in which case it 3759714SN/A // should be in the outstandingReq), or if we merely forwarded 3769714SN/A // someone else's snoop request 3779714SN/A bool forwardAsSnoop = outstandingReq.find(pkt->req) == 3789714SN/A outstandingReq.end(); 3799714SN/A 38010405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 38110405Sandreas.hansson@arm.com // current port, note that the check is bypassed if the response 38210405Sandreas.hansson@arm.com // is being passed on as a normal response since this is occupying 38310405Sandreas.hansson@arm.com // the response layer rather than the snoop response layer 3849715SN/A if (forwardAsSnoop) { 3859715SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 38610405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 3879715SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 3889715SN/A return false; 3899715SN/A } 3909716SN/A } else { 3919716SN/A // get the master port that mirrors this slave port internally 3929716SN/A MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 3939716SN/A if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 39410405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 3959716SN/A snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 3969716SN/A return false; 3979716SN/A } 3988975SN/A } 3998975SN/A 40010405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 4018975SN/A src_port->name(), pkt->cmdString(), pkt->getAddr()); 4028975SN/A 4039712SN/A // store size and command as they might be modified when 4049712SN/A // forwarding the packet 4059712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 4069712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 4079712SN/A 4088975SN/A // responses are never express snoops 4098975SN/A assert(!pkt->isExpressSnoop()); 4108975SN/A 4118975SN/A calcPacketTiming(pkt); 41210405Sandreas.hansson@arm.com Tick packetFinishTime = pkt->lastWordDelay + curTick(); 4138975SN/A 4149714SN/A // forward it either as a snoop response or a normal response 4159714SN/A if (forwardAsSnoop) { 4169714SN/A // this is a snoop response to a snoop request we forwarded, 4179714SN/A // e.g. coming from the L1 and going to the L2, and it should 4189714SN/A // be forwarded as a snoop response 41910402SN/A 42010402SN/A if (snoopFilter) { 42110402SN/A // update the probe filter so that it can properly track the line 42210402SN/A snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 42310402SN/A *masterPorts[dest_port_id]); 42410402SN/A } 42510402SN/A 4269712SN/A bool success M5_VAR_USED = 4279712SN/A masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 4289712SN/A pktCount[slave_port_id][dest_port_id]++; 42910405Sandreas.hansson@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 4308975SN/A assert(success); 4319714SN/A 4329715SN/A snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 4333244SN/A } else { 4348975SN/A // we got a snoop response on one of our slave ports, 43510405Sandreas.hansson@arm.com // i.e. from a coherent master connected to the crossbar, and 43610405Sandreas.hansson@arm.com // since we created the snoop request as part of recvTiming, 43710405Sandreas.hansson@arm.com // this should now be a normal response again 4388948SN/A outstandingReq.erase(pkt->req); 4398948SN/A 4408975SN/A // this is a snoop response from a coherent master, with a 44110405Sandreas.hansson@arm.com // destination field set on its way through the crossbar as 44210405Sandreas.hansson@arm.com // request, hence it should never go back to where the snoop 44310405Sandreas.hansson@arm.com // response came from, but instead to where the original 44410405Sandreas.hansson@arm.com // request came from 4459712SN/A assert(slave_port_id != dest_port_id); 4468948SN/A 44710402SN/A if (snoopFilter) { 44810402SN/A // update the probe filter so that it can properly track the line 44910402SN/A snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 45010402SN/A *slavePorts[dest_port_id]); 45110402SN/A } 45210402SN/A 45310405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 45410402SN/A " FWD RESP\n", src_port->name(), pkt->cmdString(), 45510402SN/A pkt->getAddr()); 45610402SN/A 4579714SN/A // as a normal response, it should go back to a master through 4589714SN/A // one of our slave ports, at this point we are ignoring the 4599714SN/A // fact that the response layer could be busy and do not touch 4609714SN/A // its state 4619712SN/A bool success M5_VAR_USED = 4629712SN/A slavePorts[dest_port_id]->sendTimingResp(pkt); 4638975SN/A 4649714SN/A // @todo Put the response in an internal FIFO and pass it on 4659714SN/A // to the response layer from there 4669714SN/A 4678975SN/A // currently it is illegal to block responses... can lead 4688975SN/A // to deadlock 4698948SN/A assert(success); 4709716SN/A 4719716SN/A respLayers[dest_port_id]->succeededTiming(packetFinishTime); 4723244SN/A } 4733244SN/A 4749712SN/A // stats updates 4759712SN/A transDist[pkt_cmd]++; 47610405Sandreas.hansson@arm.com snoops++; 4779712SN/A 4788948SN/A return true; 4798948SN/A} 4808948SN/A 4813210SN/A 4828948SN/Avoid 48310405Sandreas.hansson@arm.comCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 48410402SN/A const std::vector<SlavePort*>& dests) 4858948SN/A{ 48610405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 4879663SN/A pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 4889663SN/A 4899524SN/A // snoops should only happen if the system isn't bypassing caches 4909524SN/A assert(!system->bypassCaches()); 4919524SN/A 49210401SN/A unsigned fanout = 0; 49310401SN/A 49410405Sandreas.hansson@arm.com for (const auto& p: dests) { 4958948SN/A // we could have gotten this request from a snooping master 4968948SN/A // (corresponding to our own slave port that is also in 4978948SN/A // snoopPorts) and should not send it back to where it came 4988948SN/A // from 4999031SN/A if (exclude_slave_port_id == InvalidPortID || 5008948SN/A p->getId() != exclude_slave_port_id) { 5018948SN/A // cache is not allowed to refuse snoop 5028975SN/A p->sendTimingSnoopReq(pkt); 50310401SN/A fanout++; 5048948SN/A } 5058948SN/A } 50610401SN/A 50710401SN/A // Stats for fanout of this forward operation 50810401SN/A snoopFanout.sample(fanout); 5092497SN/A} 5102497SN/A 5119092SN/Avoid 51210405Sandreas.hansson@arm.comCoherentXBar::recvRetry(PortID master_port_id) 5139092SN/A{ 5149093SN/A // responses and snoop responses never block on forwarding them, 5159093SN/A // so the retry will always be coming from a port to which we 5169093SN/A // tried to forward a request 5179715SN/A reqLayers[master_port_id]->recvRetry(); 5189092SN/A} 5199092SN/A 5209036SN/ATick 52110405Sandreas.hansson@arm.comCoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 5222657SN/A{ 52310405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 5249032SN/A slavePorts[slave_port_id]->name(), pkt->getAddr(), 5258949SN/A pkt->cmdString()); 5268915SN/A 52710405Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 52810405Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 5299712SN/A 5308979SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 5318979SN/A Tick snoop_response_latency = 0; 5328979SN/A 5338979SN/A // uncacheable requests need never be snooped 5349524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 5358979SN/A // forward to all snoopers but the source 53610402SN/A std::pair<MemCmd, Tick> snoop_result; 53710402SN/A if (snoopFilter) { 53810402SN/A // check with the snoop filter where to forward this packet 53910402SN/A auto sf_res = 54010402SN/A snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 54110402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 54210405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 54310402SN/A " SF size: %i lat: %i\n", __func__, 54410402SN/A slavePorts[slave_port_id]->name(), pkt->cmdString(), 54510402SN/A pkt->getAddr(), sf_res.first.size(), sf_res.second); 54610402SN/A snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 54710402SN/A sf_res.first); 54810402SN/A } else { 54910402SN/A snoop_result = forwardAtomic(pkt, slave_port_id); 55010402SN/A } 5518979SN/A snoop_response_cmd = snoop_result.first; 55210402SN/A snoop_response_latency += snoop_result.second; 5538979SN/A } 5548915SN/A 5558948SN/A // even if we had a snoop response, we must continue and also 5568948SN/A // perform the actual request at the destination 55710405Sandreas.hansson@arm.com PortID master_port_id = findPort(pkt->getAddr()); 55810405Sandreas.hansson@arm.com 55910405Sandreas.hansson@arm.com // stats updates for the request 56010405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 56110405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 56210405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 5638948SN/A 5648948SN/A // forward the request to the appropriate destination 56510405Sandreas.hansson@arm.com Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 5668948SN/A 56710402SN/A // Lower levels have replied, tell the snoop filter 56810402SN/A if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 56910402SN/A pkt->isResponse()) { 57010402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 57110402SN/A } 57210402SN/A 5738948SN/A // if we got a response from a snooper, restore it here 5748948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) { 5758948SN/A // no one else should have responded 5768948SN/A assert(!pkt->isResponse()); 5778948SN/A pkt->cmd = snoop_response_cmd; 5788948SN/A response_latency = snoop_response_latency; 5798948SN/A } 5808948SN/A 5819712SN/A // add the response data 58210405Sandreas.hansson@arm.com if (pkt->isResponse()) { 58310405Sandreas.hansson@arm.com pkt_size = pkt->hasData() ? pkt->getSize() : 0; 58410405Sandreas.hansson@arm.com pkt_cmd = pkt->cmdToIndex(); 58510405Sandreas.hansson@arm.com 58610405Sandreas.hansson@arm.com // stats updates 58710405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 58810405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 58910405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 59010405Sandreas.hansson@arm.com } 5919712SN/A 5929547SN/A // @todo: Not setting first-word time 59310405Sandreas.hansson@arm.com pkt->lastWordDelay = response_latency; 5948948SN/A return response_latency; 5958948SN/A} 5968948SN/A 5978948SN/ATick 59810405Sandreas.hansson@arm.comCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 5998948SN/A{ 60010405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 6019032SN/A masterPorts[master_port_id]->name(), pkt->getAddr(), 6028949SN/A pkt->cmdString()); 6038948SN/A 6049712SN/A // add the request snoop data 60510405Sandreas.hansson@arm.com snoops++; 6069712SN/A 6078948SN/A // forward to all snoopers 60810402SN/A std::pair<MemCmd, Tick> snoop_result; 60910402SN/A Tick snoop_response_latency = 0; 61010402SN/A if (snoopFilter) { 61110402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 61210402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 61310405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 61410402SN/A __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 61510402SN/A pkt->getAddr(), sf_res.first.size(), sf_res.second); 61610402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 61710402SN/A sf_res.first); 61810402SN/A } else { 61910402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID); 62010402SN/A } 6218948SN/A MemCmd snoop_response_cmd = snoop_result.first; 62210402SN/A snoop_response_latency += snoop_result.second; 6238948SN/A 6248948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) 6258948SN/A pkt->cmd = snoop_response_cmd; 6268948SN/A 6279712SN/A // add the response snoop data 62810401SN/A if (pkt->isResponse()) { 62910405Sandreas.hansson@arm.com snoops++; 63010401SN/A } 6319712SN/A 6329547SN/A // @todo: Not setting first-word time 63310405Sandreas.hansson@arm.com pkt->lastWordDelay = snoop_response_latency; 6348948SN/A return snoop_response_latency; 6358948SN/A} 6368948SN/A 6378948SN/Astd::pair<MemCmd, Tick> 63810405Sandreas.hansson@arm.comCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 63910402SN/A PortID source_master_port_id, 64010402SN/A const std::vector<SlavePort*>& dests) 6418948SN/A{ 6429032SN/A // the packet may be changed on snoops, record the original 6439032SN/A // command to enable us to restore it between snoops so that 6448948SN/A // additional snoops can take place properly 6454626SN/A MemCmd orig_cmd = pkt->cmd; 6464879SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 6474879SN/A Tick snoop_response_latency = 0; 6483662SN/A 6499524SN/A // snoops should only happen if the system isn't bypassing caches 6509524SN/A assert(!system->bypassCaches()); 6519524SN/A 65210401SN/A unsigned fanout = 0; 65310401SN/A 65410405Sandreas.hansson@arm.com for (const auto& p: dests) { 6558915SN/A // we could have gotten this request from a snooping master 6568915SN/A // (corresponding to our own slave port that is also in 6578915SN/A // snoopPorts) and should not send it back to where it came 6588915SN/A // from 65910402SN/A if (exclude_slave_port_id != InvalidPortID && 66010402SN/A p->getId() == exclude_slave_port_id) 66110402SN/A continue; 66210401SN/A 66310402SN/A Tick latency = p->sendAtomicSnoop(pkt); 66410402SN/A fanout++; 66510402SN/A 66610402SN/A // in contrast to a functional access, we have to keep on 66710402SN/A // going as all snoopers must be updated even if we get a 66810402SN/A // response 66910402SN/A if (!pkt->isResponse()) 67010402SN/A continue; 67110402SN/A 67210402SN/A // response from snoop agent 67310402SN/A assert(pkt->cmd != orig_cmd); 67410402SN/A assert(pkt->memInhibitAsserted()); 67510402SN/A // should only happen once 67610402SN/A assert(snoop_response_cmd == MemCmd::InvalidCmd); 67710402SN/A // save response state 67810402SN/A snoop_response_cmd = pkt->cmd; 67910402SN/A snoop_response_latency = latency; 68010402SN/A 68110402SN/A if (snoopFilter) { 68210402SN/A // Handle responses by the snoopers and differentiate between 68310402SN/A // responses to requests from above and snoops from below 68410402SN/A if (source_master_port_id != InvalidPortID) { 68510402SN/A // Getting a response for a snoop from below 68610402SN/A assert(exclude_slave_port_id == InvalidPortID); 68710402SN/A snoopFilter->updateSnoopForward(pkt, *p, 68810402SN/A *masterPorts[source_master_port_id]); 68910402SN/A } else { 69010402SN/A // Getting a response for a request from above 69110402SN/A assert(source_master_port_id == InvalidPortID); 69210402SN/A snoopFilter->updateSnoopResponse(pkt, *p, 69310402SN/A *slavePorts[exclude_slave_port_id]); 6944626SN/A } 6954626SN/A } 69610402SN/A // restore original packet state for remaining snoopers 69710402SN/A pkt->cmd = orig_cmd; 6984626SN/A } 6994626SN/A 70010401SN/A // Stats for fanout 70110401SN/A snoopFanout.sample(fanout); 70210401SN/A 7038948SN/A // the packet is restored as part of the loop and any potential 7048948SN/A // snoop response is part of the returned pair 7058948SN/A return std::make_pair(snoop_response_cmd, snoop_response_latency); 7062497SN/A} 7072497SN/A 7082497SN/Avoid 70910405Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 7102497SN/A{ 7118663SN/A if (!pkt->isPrint()) { 7128663SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 71310405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, 7148949SN/A "recvFunctional: packet src %s addr 0x%x cmd %s\n", 7159032SN/A slavePorts[slave_port_id]->name(), pkt->getAddr(), 7168663SN/A pkt->cmdString()); 7178663SN/A } 7188663SN/A 7198979SN/A // uncacheable requests need never be snooped 7209524SN/A if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 7218979SN/A // forward to all snoopers but the source 7229032SN/A forwardFunctional(pkt, slave_port_id); 7238979SN/A } 7244912SN/A 7258948SN/A // there is no need to continue if the snooping has found what we 7268948SN/A // were looking for and the packet is already a response 7278948SN/A if (!pkt->isResponse()) { 7289031SN/A PortID dest_id = findPort(pkt->getAddr()); 7298948SN/A 7308948SN/A masterPorts[dest_id]->sendFunctional(pkt); 7318948SN/A } 7328948SN/A} 7338948SN/A 7348948SN/Avoid 73510405Sandreas.hansson@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 7368948SN/A{ 7378948SN/A if (!pkt->isPrint()) { 7388948SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 73910405Sandreas.hansson@arm.com DPRINTF(CoherentXBar, 7408949SN/A "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 7419032SN/A masterPorts[master_port_id]->name(), pkt->getAddr(), 7428948SN/A pkt->cmdString()); 7438948SN/A } 7448948SN/A 7458948SN/A // forward to all snoopers 7469031SN/A forwardFunctional(pkt, InvalidPortID); 7478948SN/A} 7488948SN/A 7498948SN/Avoid 75010405Sandreas.hansson@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 7518948SN/A{ 7529524SN/A // snoops should only happen if the system isn't bypassing caches 7539524SN/A assert(!system->bypassCaches()); 7549524SN/A 75510405Sandreas.hansson@arm.com for (const auto& p: snoopPorts) { 7568915SN/A // we could have gotten this request from a snooping master 7578915SN/A // (corresponding to our own slave port that is also in 7588915SN/A // snoopPorts) and should not send it back to where it came 7598915SN/A // from 7609031SN/A if (exclude_slave_port_id == InvalidPortID || 7618948SN/A p->getId() != exclude_slave_port_id) 7628948SN/A p->sendFunctionalSnoop(pkt); 7638915SN/A 7648948SN/A // if we get a response we are done 7658948SN/A if (pkt->isResponse()) { 7668948SN/A break; 7678915SN/A } 7683650SN/A } 7692497SN/A} 7702497SN/A 7719092SN/Aunsigned int 77210405Sandreas.hansson@arm.comCoherentXBar::drain(DrainManager *dm) 7739092SN/A{ 7749093SN/A // sum up the individual layers 7759715SN/A unsigned int total = 0; 77610405Sandreas.hansson@arm.com for (auto l: reqLayers) 77710405Sandreas.hansson@arm.com total += l->drain(dm); 77810405Sandreas.hansson@arm.com for (auto l: respLayers) 77910405Sandreas.hansson@arm.com total += l->drain(dm); 78010405Sandreas.hansson@arm.com for (auto l: snoopLayers) 78110405Sandreas.hansson@arm.com total += l->drain(dm); 7829715SN/A return total; 7839092SN/A} 7849092SN/A 7859712SN/Avoid 78610405Sandreas.hansson@arm.comCoherentXBar::regStats() 7879712SN/A{ 78810405Sandreas.hansson@arm.com // register the stats of the base class and our layers 78910405Sandreas.hansson@arm.com BaseXBar::regStats(); 79010405Sandreas.hansson@arm.com for (auto l: reqLayers) 79110405Sandreas.hansson@arm.com l->regStats(); 79210405Sandreas.hansson@arm.com for (auto l: respLayers) 79310405Sandreas.hansson@arm.com l->regStats(); 79410405Sandreas.hansson@arm.com for (auto l: snoopLayers) 79510405Sandreas.hansson@arm.com l->regStats(); 7969712SN/A 79710405Sandreas.hansson@arm.com snoops 79810405Sandreas.hansson@arm.com .name(name() + ".snoops") 79910401SN/A .desc("Total snoops (count)") 80010401SN/A ; 80110401SN/A 80210401SN/A snoopFanout 80310401SN/A .init(0, snoopPorts.size(), 1) 80410401SN/A .name(name() + ".snoop_fanout") 80510401SN/A .desc("Request fanout histogram") 80610401SN/A ; 8079712SN/A} 8089712SN/A 80910405Sandreas.hansson@arm.comCoherentXBar * 81010405Sandreas.hansson@arm.comCoherentXBarParams::create() 8112497SN/A{ 81210405Sandreas.hansson@arm.com return new CoherentXBar(this); 8132497SN/A} 814