coherent_xbar.cc revision 14006
12497SN/A/* 214006Stiago.muck@arm.com * Copyright (c) 2011-2019 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142497SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152497SN/A * All rights reserved. 162497SN/A * 172497SN/A * Redistribution and use in source and binary forms, with or without 182497SN/A * modification, are permitted provided that the following conditions are 192497SN/A * met: redistributions of source code must retain the above copyright 202497SN/A * notice, this list of conditions and the following disclaimer; 212497SN/A * redistributions in binary form must reproduce the above copyright 222497SN/A * notice, this list of conditions and the following disclaimer in the 232497SN/A * documentation and/or other materials provided with the distribution; 242497SN/A * neither the name of the copyright holders nor the names of its 252497SN/A * contributors may be used to endorse or promote products derived from 262497SN/A * this software without specific prior written permission. 272497SN/A * 282497SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292497SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302497SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312497SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322497SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332497SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342497SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352497SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362497SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372497SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382497SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 418715SN/A * Andreas Hansson 428922SN/A * William Wang 4312351Snikos.nikoleris@arm.com * Nikos Nikoleris 442497SN/A */ 452497SN/A 462497SN/A/** 472982SN/A * @file 4810405Sandreas.hansson@arm.com * Definition of a crossbar object. 492497SN/A */ 502497SN/A 5111793Sbrandon.potter@amd.com#include "mem/coherent_xbar.hh" 5211793Sbrandon.potter@amd.com 5312334Sgabeblack@google.com#include "base/logging.hh" 542548SN/A#include "base/trace.hh" 5510405Sandreas.hansson@arm.com#include "debug/AddrRanges.hh" 5610405Sandreas.hansson@arm.com#include "debug/CoherentXBar.hh" 579524SN/A#include "sim/system.hh" 582497SN/A 5910405Sandreas.hansson@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 6010719SMarco.Balboni@ARM.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter), 6111334Sandreas.hansson@arm.com snoopResponseLatency(p->snoop_response_latency), 6214006Stiago.muck@arm.com maxOutstandingSnoopCheck(p->max_outstanding_snoops), 6314006Stiago.muck@arm.com maxRoutingTableSizeCheck(p->max_routing_table_size), 6412341Snikos.nikoleris@arm.com pointOfCoherency(p->point_of_coherency), 6512341Snikos.nikoleris@arm.com pointOfUnification(p->point_of_unification) 667523SN/A{ 678851SN/A // create the ports based on the size of the master and slave 688948SN/A // vector ports, and the presence of the default port, the ports 698948SN/A // are enumerated starting from zero 708851SN/A for (int i = 0; i < p->port_master_connection_count; ++i) { 719095SN/A std::string portName = csprintf("%s.master[%d]", name(), i); 7210405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 738922SN/A masterPorts.push_back(bp); 749715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 759715SN/A csprintf(".reqLayer%d", i))); 7613808Sgabeblack@google.com snoopLayers.push_back( 7713808Sgabeblack@google.com new SnoopRespLayer(*bp, *this, csprintf(".snoopLayer%d", i))); 788851SN/A } 798851SN/A 808948SN/A // see if we have a default slave device connected and if so add 818948SN/A // our corresponding master port 828915SN/A if (p->port_default_connection_count) { 839031SN/A defaultPortID = masterPorts.size(); 849095SN/A std::string portName = name() + ".default"; 8510405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 8613808Sgabeblack@google.com defaultPortID); 878922SN/A masterPorts.push_back(bp); 889715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 8913808Sgabeblack@google.com defaultPortID))); 9010713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 9110713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", 9210713Sandreas.hansson@arm.com defaultPortID))); 938915SN/A } 948915SN/A 958948SN/A // create the slave ports, once again starting at zero 968851SN/A for (int i = 0; i < p->port_slave_connection_count; ++i) { 979095SN/A std::string portName = csprintf("%s.slave[%d]", name(), i); 9810888Sandreas.hansson@arm.com QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 998922SN/A slavePorts.push_back(bp); 1009715SN/A respLayers.push_back(new RespLayer(*bp, *this, 1019715SN/A csprintf(".respLayer%d", i))); 1029716SN/A snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 1038851SN/A } 1047523SN/A} 1057523SN/A 10610405Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 1079715SN/A{ 10810405Sandreas.hansson@arm.com for (auto l: reqLayers) 10910405Sandreas.hansson@arm.com delete l; 11010405Sandreas.hansson@arm.com for (auto l: respLayers) 11110405Sandreas.hansson@arm.com delete l; 11210405Sandreas.hansson@arm.com for (auto l: snoopLayers) 11310405Sandreas.hansson@arm.com delete l; 11410405Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11510405Sandreas.hansson@arm.com delete p; 1169715SN/A} 1179715SN/A 1182568SN/Avoid 11910405Sandreas.hansson@arm.comCoherentXBar::init() 1202568SN/A{ 12110405Sandreas.hansson@arm.com BaseXBar::init(); 1229278SN/A 1238948SN/A // iterate over our slave ports and determine which of our 1248948SN/A // neighbouring master ports are snooping and add them as snoopers 12510405Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 1269088SN/A // check if the connected master port is snooping 12710405Sandreas.hansson@arm.com if (p->isSnooping()) { 12810405Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12910405Sandreas.hansson@arm.com p->getMasterPort().name()); 13010405Sandreas.hansson@arm.com snoopPorts.push_back(p); 1318711SN/A } 1328711SN/A } 1332568SN/A 1349036SN/A if (snoopPorts.empty()) 13510405Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 13611133Sandreas.hansson@arm.com 13711133Sandreas.hansson@arm.com // inform the snoop filter about the slave ports so it can create 13811133Sandreas.hansson@arm.com // its own internal representation 13911133Sandreas.hansson@arm.com if (snoopFilter) 14011133Sandreas.hansson@arm.com snoopFilter->setSlavePorts(slavePorts); 1413244SN/A} 1423244SN/A 1438948SN/Abool 14410405Sandreas.hansson@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1453244SN/A{ 1468975SN/A // determine the source port based on the id 1479032SN/A SlavePort *src_port = slavePorts[slave_port_id]; 1483244SN/A 1499091SN/A // remember if the packet is an express snoop 1509091SN/A bool is_express_snoop = pkt->isExpressSnoop(); 15111284Sandreas.hansson@arm.com bool cache_responding = pkt->cacheResponding(); 15210656Sandreas.hansson@arm.com // for normal requests, going downstream, the express snoop flag 15311284Sandreas.hansson@arm.com // and the cache responding flag should always be the same 15411284Sandreas.hansson@arm.com assert(is_express_snoop == cache_responding); 1559091SN/A 15612780Snikos.nikoleris@arm.com // determine the destination based on the destination address range 15713856Sodanrc@yahoo.com.br PortID master_port_id = findPort(pkt->getAddrRange()); 1589612SN/A 15910405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the current 1609033SN/A // port, and exclude express snoops from the check 1619715SN/A if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 16211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 16311744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1643244SN/A return false; 1653244SN/A } 1663244SN/A 16711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 16811744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1695197SN/A 1709712SN/A // store size and command as they might be modified when 1719712SN/A // forwarding the packet 1729712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1739712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 1749712SN/A 17510719SMarco.Balboni@ARM.com // store the old header delay so we can restore it if needed 17610719SMarco.Balboni@ARM.com Tick old_header_delay = pkt->headerDelay; 17710719SMarco.Balboni@ARM.com 17810719SMarco.Balboni@ARM.com // a request sees the frontend and forward latency 17910719SMarco.Balboni@ARM.com Tick xbar_delay = (frontendLatency + forwardLatency) * clockPeriod(); 18010719SMarco.Balboni@ARM.com 18110719SMarco.Balboni@ARM.com // set the packet header and payload delay 18210719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 18310719SMarco.Balboni@ARM.com 18410719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 18510719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 1864912SN/A 18712346Snikos.nikoleris@arm.com // is this the destination point for this packet? (e.g. true if 18812346Snikos.nikoleris@arm.com // this xbar is the PoC for a cache maintenance operation to the 18912346Snikos.nikoleris@arm.com // PoC) otherwise the destination is any cache that can satisfy 19012346Snikos.nikoleris@arm.com // the request 19112346Snikos.nikoleris@arm.com const bool is_destination = isDestination(pkt); 19212346Snikos.nikoleris@arm.com 19312345Snikos.nikoleris@arm.com const bool snoop_caches = !system->bypassCaches() && 19412345Snikos.nikoleris@arm.com pkt->cmd != MemCmd::WriteClean; 19512345Snikos.nikoleris@arm.com if (snoop_caches) { 19611127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 19711127Sandreas.hansson@arm.com 19812351Snikos.nikoleris@arm.com if (pkt->isClean() && !is_destination) { 19912351Snikos.nikoleris@arm.com // before snooping we need to make sure that the memory 20012351Snikos.nikoleris@arm.com // below is not busy and the cache clean request can be 20112351Snikos.nikoleris@arm.com // forwarded to it 20212351Snikos.nikoleris@arm.com if (!masterPorts[master_port_id]->tryTiming(pkt)) { 20312351Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__, 20412351Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 20512351Snikos.nikoleris@arm.com 20612351Snikos.nikoleris@arm.com // update the layer state and schedule an idle event 20712351Snikos.nikoleris@arm.com reqLayers[master_port_id]->failedTiming(src_port, 20812351Snikos.nikoleris@arm.com clockEdge(Cycles(1))); 20912351Snikos.nikoleris@arm.com return false; 21012351Snikos.nikoleris@arm.com } 21112351Snikos.nikoleris@arm.com } 21212351Snikos.nikoleris@arm.com 21312351Snikos.nikoleris@arm.com 2148979SN/A // the packet is a memory-mapped request and should be 2158979SN/A // broadcasted to our snoopers but the source 21610402SN/A if (snoopFilter) { 21710402SN/A // check with the snoop filter where to forward this packet 21810402SN/A auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 21911126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 22011126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 22111126Sandreas.hansson@arm.com // of the snoop filter 22210719SMarco.Balboni@ARM.com pkt->headerDelay += sf_res.second * clockPeriod(); 22311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 22411744Snikos.nikoleris@arm.com __func__, src_port->name(), pkt->print(), 22511744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 22611196Sali.jafri@arm.com 22711199Sandreas.hansson@arm.com if (pkt->isEviction()) { 22811196Sali.jafri@arm.com // for block-evicting packets, i.e. writebacks and 22911196Sali.jafri@arm.com // clean evictions, there is no need to snoop up, as 23011196Sali.jafri@arm.com // all we do is determine if the block is cached or 23111196Sali.jafri@arm.com // not, instead just set it here based on the snoop 23211196Sali.jafri@arm.com // filter result 23311196Sali.jafri@arm.com if (!sf_res.first.empty()) 23411196Sali.jafri@arm.com pkt->setBlockCached(); 23511196Sali.jafri@arm.com } else { 23611196Sali.jafri@arm.com forwardTiming(pkt, slave_port_id, sf_res.first); 23711196Sali.jafri@arm.com } 23810402SN/A } else { 23910402SN/A forwardTiming(pkt, slave_port_id); 24010402SN/A } 24111127Sandreas.hansson@arm.com 24211127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 24311127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 24411127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 2458979SN/A } 2468948SN/A 24711334Sandreas.hansson@arm.com // set up a sensible starting point 24811334Sandreas.hansson@arm.com bool success = true; 24910883Sali.jafri@arm.com 25011284Sandreas.hansson@arm.com // remember if the packet will generate a snoop response by 25111284Sandreas.hansson@arm.com // checking if a cache set the cacheResponding flag during the 25211284Sandreas.hansson@arm.com // snooping above 25311284Sandreas.hansson@arm.com const bool expect_snoop_resp = !cache_responding && pkt->cacheResponding(); 25411334Sandreas.hansson@arm.com bool expect_response = pkt->needsResponse() && !pkt->cacheResponding(); 2558915SN/A 25611334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 25711334Sandreas.hansson@arm.com 25811334Sandreas.hansson@arm.com // in certain cases the crossbar is responsible for responding 25911334Sandreas.hansson@arm.com bool respond_directly = false; 26011544Snikos.nikoleris@arm.com // store the original address as an address mapper could possibly 26111544Snikos.nikoleris@arm.com // modify the address upon a sendTimingRequest 26211544Snikos.nikoleris@arm.com const Addr addr(pkt->getAddr()); 26311334Sandreas.hansson@arm.com if (sink_packet) { 26411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 26511744Snikos.nikoleris@arm.com pkt->print()); 26611334Sandreas.hansson@arm.com } else { 26711334Sandreas.hansson@arm.com // determine if we are forwarding the packet, or responding to 26811334Sandreas.hansson@arm.com // it 26912346Snikos.nikoleris@arm.com if (forwardPacket(pkt)) { 27011334Sandreas.hansson@arm.com // if we are passing on, rather than sinking, a packet to 27111334Sandreas.hansson@arm.com // which an upstream cache has committed to responding, 27211334Sandreas.hansson@arm.com // the line was needs writable, and the responding only 27311334Sandreas.hansson@arm.com // had an Owned copy, so we need to immidiately let the 27411334Sandreas.hansson@arm.com // downstream caches know, bypass any flow control 27511334Sandreas.hansson@arm.com if (pkt->cacheResponding()) { 27611334Sandreas.hansson@arm.com pkt->setExpressSnoop(); 27711334Sandreas.hansson@arm.com } 27811334Sandreas.hansson@arm.com 27912346Snikos.nikoleris@arm.com // make sure that the write request (e.g., WriteClean) 28012346Snikos.nikoleris@arm.com // will stop at the memory below if this crossbar is its 28112346Snikos.nikoleris@arm.com // destination 28212346Snikos.nikoleris@arm.com if (pkt->isWrite() && is_destination) { 28312346Snikos.nikoleris@arm.com pkt->clearWriteThrough(); 28412346Snikos.nikoleris@arm.com } 28512346Snikos.nikoleris@arm.com 28611334Sandreas.hansson@arm.com // since it is a normal request, attempt to send the packet 28711334Sandreas.hansson@arm.com success = masterPorts[master_port_id]->sendTimingReq(pkt); 28811334Sandreas.hansson@arm.com } else { 28911334Sandreas.hansson@arm.com // no need to forward, turn this packet around and respond 29011334Sandreas.hansson@arm.com // directly 29111334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 29211334Sandreas.hansson@arm.com 29311334Sandreas.hansson@arm.com respond_directly = true; 29411334Sandreas.hansson@arm.com assert(!expect_snoop_resp); 29511334Sandreas.hansson@arm.com expect_response = false; 29611334Sandreas.hansson@arm.com } 29711334Sandreas.hansson@arm.com } 2988948SN/A 29912345Snikos.nikoleris@arm.com if (snoopFilter && snoop_caches) { 30010402SN/A // Let the snoop filter know about the success of the send operation 30111605Snikos.nikoleris@arm.com snoopFilter->finishRequest(!success, addr, pkt->isSecure()); 30210402SN/A } 30310402SN/A 30410656Sandreas.hansson@arm.com // check if we were successful in sending the packet onwards 30510656Sandreas.hansson@arm.com if (!success) { 30611284Sandreas.hansson@arm.com // express snoops should never be forced to retry 30710656Sandreas.hansson@arm.com assert(!is_express_snoop); 30810656Sandreas.hansson@arm.com 30910719SMarco.Balboni@ARM.com // restore the header delay 31010719SMarco.Balboni@ARM.com pkt->headerDelay = old_header_delay; 31110656Sandreas.hansson@arm.com 31211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__, 31311744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 31410656Sandreas.hansson@arm.com 31510656Sandreas.hansson@arm.com // update the layer state and schedule an idle event 31610656Sandreas.hansson@arm.com reqLayers[master_port_id]->failedTiming(src_port, 31710719SMarco.Balboni@ARM.com clockEdge(Cycles(1))); 3189091SN/A } else { 31910656Sandreas.hansson@arm.com // express snoops currently bypass the crossbar state entirely 32010656Sandreas.hansson@arm.com if (!is_express_snoop) { 32110656Sandreas.hansson@arm.com // if this particular request will generate a snoop 32210656Sandreas.hansson@arm.com // response 32310656Sandreas.hansson@arm.com if (expect_snoop_resp) { 32410656Sandreas.hansson@arm.com // we should never have an exsiting request outstanding 32510656Sandreas.hansson@arm.com assert(outstandingSnoop.find(pkt->req) == 32610656Sandreas.hansson@arm.com outstandingSnoop.end()); 32710656Sandreas.hansson@arm.com outstandingSnoop.insert(pkt->req); 3288948SN/A 32910656Sandreas.hansson@arm.com // basic sanity check on the outstanding snoops 33014006Stiago.muck@arm.com panic_if(outstandingSnoop.size() > maxOutstandingSnoopCheck, 33114006Stiago.muck@arm.com "%s: Outstanding snoop requests exceeded %d\n", 33214006Stiago.muck@arm.com name(), maxOutstandingSnoopCheck); 33310656Sandreas.hansson@arm.com } 3348948SN/A 33510656Sandreas.hansson@arm.com // remember where to route the normal response to 33610656Sandreas.hansson@arm.com if (expect_response || expect_snoop_resp) { 33710656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 33810656Sandreas.hansson@arm.com routeTo[pkt->req] = slave_port_id; 3399549SN/A 34014006Stiago.muck@arm.com panic_if(routeTo.size() > maxRoutingTableSizeCheck, 34114006Stiago.muck@arm.com "%s: Routing table exceeds %d packets\n", 34214006Stiago.muck@arm.com name(), maxRoutingTableSizeCheck); 34310656Sandreas.hansson@arm.com } 3448948SN/A 34510405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 3469715SN/A reqLayers[master_port_id]->succeededTiming(packetFinishTime); 3479091SN/A } 3488975SN/A 34910656Sandreas.hansson@arm.com // stats updates only consider packets that were successfully sent 3509712SN/A pktCount[slave_port_id][master_port_id]++; 35110405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 3529712SN/A transDist[pkt_cmd]++; 35310656Sandreas.hansson@arm.com 35411564Sdavid.guillen@arm.com if (is_express_snoop) { 35510656Sandreas.hansson@arm.com snoops++; 35611564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 35711564Sdavid.guillen@arm.com } 3589712SN/A } 3599712SN/A 36011334Sandreas.hansson@arm.com if (sink_packet) 36111334Sandreas.hansson@arm.com // queue the packet for deletion 36211334Sandreas.hansson@arm.com pendingDelete.reset(pkt); 36311334Sandreas.hansson@arm.com 36412351Snikos.nikoleris@arm.com // normally we respond to the packet we just received if we need to 36512351Snikos.nikoleris@arm.com PacketPtr rsp_pkt = pkt; 36612351Snikos.nikoleris@arm.com PortID rsp_port_id = slave_port_id; 36712351Snikos.nikoleris@arm.com 36812351Snikos.nikoleris@arm.com // If this is the destination of the cache clean operation the 36912351Snikos.nikoleris@arm.com // crossbar is responsible for responding. This crossbar will 37012351Snikos.nikoleris@arm.com // respond when the cache clean is complete. A cache clean 37112351Snikos.nikoleris@arm.com // is complete either: 37212351Snikos.nikoleris@arm.com // * direcly, if no cache above had a dirty copy of the block 37312351Snikos.nikoleris@arm.com // as indicated by the satisfied flag of the packet, or 37412351Snikos.nikoleris@arm.com // * when the crossbar has seen both the cache clean request 37512351Snikos.nikoleris@arm.com // (CleanSharedReq, CleanInvalidReq) and the corresponding 37612351Snikos.nikoleris@arm.com // write (WriteClean) which updates the block in the memory 37712351Snikos.nikoleris@arm.com // below. 37812351Snikos.nikoleris@arm.com if (success && 37912351Snikos.nikoleris@arm.com ((pkt->isClean() && pkt->satisfied()) || 38012351Snikos.nikoleris@arm.com pkt->cmd == MemCmd::WriteClean) && 38112351Snikos.nikoleris@arm.com is_destination) { 38212351Snikos.nikoleris@arm.com PacketPtr deferred_rsp = pkt->isWrite() ? nullptr : pkt; 38312351Snikos.nikoleris@arm.com auto cmo_lookup = outstandingCMO.find(pkt->id); 38412351Snikos.nikoleris@arm.com if (cmo_lookup != outstandingCMO.end()) { 38512351Snikos.nikoleris@arm.com // the cache clean request has already reached this xbar 38612351Snikos.nikoleris@arm.com respond_directly = true; 38712351Snikos.nikoleris@arm.com if (pkt->isWrite()) { 38812351Snikos.nikoleris@arm.com rsp_pkt = cmo_lookup->second; 38912351Snikos.nikoleris@arm.com assert(rsp_pkt); 39012351Snikos.nikoleris@arm.com 39112351Snikos.nikoleris@arm.com // determine the destination 39212351Snikos.nikoleris@arm.com const auto route_lookup = routeTo.find(rsp_pkt->req); 39312351Snikos.nikoleris@arm.com assert(route_lookup != routeTo.end()); 39412351Snikos.nikoleris@arm.com rsp_port_id = route_lookup->second; 39512351Snikos.nikoleris@arm.com assert(rsp_port_id != InvalidPortID); 39612351Snikos.nikoleris@arm.com assert(rsp_port_id < respLayers.size()); 39712351Snikos.nikoleris@arm.com // remove the request from the routing table 39812351Snikos.nikoleris@arm.com routeTo.erase(route_lookup); 39912351Snikos.nikoleris@arm.com } 40012351Snikos.nikoleris@arm.com outstandingCMO.erase(cmo_lookup); 40112351Snikos.nikoleris@arm.com } else { 40212351Snikos.nikoleris@arm.com respond_directly = false; 40312351Snikos.nikoleris@arm.com outstandingCMO.emplace(pkt->id, deferred_rsp); 40412351Snikos.nikoleris@arm.com if (!pkt->isWrite()) { 40512351Snikos.nikoleris@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 40612351Snikos.nikoleris@arm.com routeTo[pkt->req] = slave_port_id; 40712351Snikos.nikoleris@arm.com 40814006Stiago.muck@arm.com panic_if(routeTo.size() > maxRoutingTableSizeCheck, 40914006Stiago.muck@arm.com "%s: Routing table exceeds %d packets\n", 41014006Stiago.muck@arm.com name(), maxRoutingTableSizeCheck); 41112351Snikos.nikoleris@arm.com } 41212351Snikos.nikoleris@arm.com } 41312351Snikos.nikoleris@arm.com } 41412351Snikos.nikoleris@arm.com 41512351Snikos.nikoleris@arm.com 41611334Sandreas.hansson@arm.com if (respond_directly) { 41712351Snikos.nikoleris@arm.com assert(rsp_pkt->needsResponse()); 41811334Sandreas.hansson@arm.com assert(success); 41911334Sandreas.hansson@arm.com 42012351Snikos.nikoleris@arm.com rsp_pkt->makeResponse(); 42111334Sandreas.hansson@arm.com 42211334Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 42311334Sandreas.hansson@arm.com // let the snoop filter inspect the response and update its state 42412351Snikos.nikoleris@arm.com snoopFilter->updateResponse(rsp_pkt, *slavePorts[rsp_port_id]); 42511334Sandreas.hansson@arm.com } 42611334Sandreas.hansson@arm.com 42712351Snikos.nikoleris@arm.com // we send the response after the current packet, even if the 42812351Snikos.nikoleris@arm.com // response is not for this packet (e.g. cache clean operation 42912351Snikos.nikoleris@arm.com // where both the request and the write packet have to cross 43012351Snikos.nikoleris@arm.com // the destination xbar before the response is sent.) 43111334Sandreas.hansson@arm.com Tick response_time = clockEdge() + pkt->headerDelay; 43212351Snikos.nikoleris@arm.com rsp_pkt->headerDelay = 0; 43311334Sandreas.hansson@arm.com 43412351Snikos.nikoleris@arm.com slavePorts[rsp_port_id]->schedTimingResp(rsp_pkt, response_time); 43511334Sandreas.hansson@arm.com } 43611334Sandreas.hansson@arm.com 4379091SN/A return success; 4388975SN/A} 4398975SN/A 4408975SN/Abool 44110405Sandreas.hansson@arm.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 4428975SN/A{ 4438975SN/A // determine the source port based on the id 4449032SN/A MasterPort *src_port = masterPorts[master_port_id]; 4458975SN/A 44610656Sandreas.hansson@arm.com // determine the destination 44710656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 44810656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 44910656Sandreas.hansson@arm.com const PortID slave_port_id = route_lookup->second; 45010572Sandreas.hansson@arm.com assert(slave_port_id != InvalidPortID); 45110572Sandreas.hansson@arm.com assert(slave_port_id < respLayers.size()); 4529713SN/A 45310405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 45410405Sandreas.hansson@arm.com // current port 4559715SN/A if (!respLayers[slave_port_id]->tryTiming(src_port)) { 45611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 45711744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 4588975SN/A return false; 4598975SN/A } 4608975SN/A 46111744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 46211744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 4638975SN/A 4649712SN/A // store size and command as they might be modified when 4659712SN/A // forwarding the packet 4669712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 4679712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 4689712SN/A 46910719SMarco.Balboni@ARM.com // a response sees the response latency 47010719SMarco.Balboni@ARM.com Tick xbar_delay = responseLatency * clockPeriod(); 47110719SMarco.Balboni@ARM.com 47210719SMarco.Balboni@ARM.com // set the packet header and payload delay 47310719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 47410719SMarco.Balboni@ARM.com 47510719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 47610719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 4778975SN/A 47810821Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 47910402SN/A // let the snoop filter inspect the response and update its state 48010402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 48110402SN/A } 48210402SN/A 48310888Sandreas.hansson@arm.com // send the packet through the destination slave port and pay for 48410888Sandreas.hansson@arm.com // any outstanding header delay 48510888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 48610888Sandreas.hansson@arm.com pkt->headerDelay = 0; 48710888Sandreas.hansson@arm.com slavePorts[slave_port_id]->schedTimingResp(pkt, curTick() + latency); 4888975SN/A 48910656Sandreas.hansson@arm.com // remove the request from the routing table 49010656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 49110656Sandreas.hansson@arm.com 4929715SN/A respLayers[slave_port_id]->succeededTiming(packetFinishTime); 4938975SN/A 4949712SN/A // stats updates 4959712SN/A pktCount[slave_port_id][master_port_id]++; 49610405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 4979712SN/A transDist[pkt_cmd]++; 4989712SN/A 4998975SN/A return true; 5008975SN/A} 5018975SN/A 5028975SN/Avoid 50310405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 5048975SN/A{ 50511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 50611744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 5078975SN/A 5089712SN/A // update stats here as we know the forwarding will succeed 50911564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 5109712SN/A transDist[pkt->cmdToIndex()]++; 51110405Sandreas.hansson@arm.com snoops++; 51211564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 5139712SN/A 5148975SN/A // we should only see express snoops from caches 5158975SN/A assert(pkt->isExpressSnoop()); 5168975SN/A 51711127Sandreas.hansson@arm.com // set the packet header and payload delay, for now use forward latency 51811127Sandreas.hansson@arm.com // @todo Assess the choice of latency further 51911127Sandreas.hansson@arm.com calcPacketTiming(pkt, forwardLatency * clockPeriod()); 52011127Sandreas.hansson@arm.com 52111284Sandreas.hansson@arm.com // remember if a cache has already committed to responding so we 52211284Sandreas.hansson@arm.com // can see if it changes during the snooping 52311284Sandreas.hansson@arm.com const bool cache_responding = pkt->cacheResponding(); 5249032SN/A 52511127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 52611127Sandreas.hansson@arm.com 52710402SN/A if (snoopFilter) { 52810402SN/A // let the Snoop Filter work its magic and guide probing 52910402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 53011126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 53111126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 53211126Sandreas.hansson@arm.com // of the snoop filter 53311126Sandreas.hansson@arm.com pkt->headerDelay += sf_res.second * clockPeriod(); 53411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 53511744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 53611744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 53710402SN/A 53810402SN/A // forward to all snoopers 53910402SN/A forwardTiming(pkt, InvalidPortID, sf_res.first); 54010402SN/A } else { 54110402SN/A forwardTiming(pkt, InvalidPortID); 54210402SN/A } 5438975SN/A 54411127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 54511127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 54611127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 54711127Sandreas.hansson@arm.com 54810656Sandreas.hansson@arm.com // if we can expect a response, remember how to route it 54911284Sandreas.hansson@arm.com if (!cache_responding && pkt->cacheResponding()) { 55010656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 55110656Sandreas.hansson@arm.com routeTo[pkt->req] = master_port_id; 55210656Sandreas.hansson@arm.com } 55310656Sandreas.hansson@arm.com 5548975SN/A // a snoop request came from a connected slave device (one of 5558975SN/A // our master ports), and if it is not coming from the slave 5568975SN/A // device responsible for the address range something is 5578975SN/A // wrong, hence there is nothing further to do as the packet 5588975SN/A // would be going back to where it came from 55913856Sodanrc@yahoo.com.br assert(findPort(pkt->getAddrRange()) == master_port_id); 5608975SN/A} 5618975SN/A 5628975SN/Abool 56310405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 5648975SN/A{ 5658975SN/A // determine the source port based on the id 5669032SN/A SlavePort* src_port = slavePorts[slave_port_id]; 5678975SN/A 56810656Sandreas.hansson@arm.com // get the destination 56910656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 57010656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 57110656Sandreas.hansson@arm.com const PortID dest_port_id = route_lookup->second; 57210572Sandreas.hansson@arm.com assert(dest_port_id != InvalidPortID); 5739714SN/A 5749714SN/A // determine if the response is from a snoop request we 5759714SN/A // created as the result of a normal request (in which case it 57610656Sandreas.hansson@arm.com // should be in the outstandingSnoop), or if we merely forwarded 5779714SN/A // someone else's snoop request 57810656Sandreas.hansson@arm.com const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 57910656Sandreas.hansson@arm.com outstandingSnoop.end(); 5809714SN/A 58110405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 58210405Sandreas.hansson@arm.com // current port, note that the check is bypassed if the response 58310405Sandreas.hansson@arm.com // is being passed on as a normal response since this is occupying 58410405Sandreas.hansson@arm.com // the response layer rather than the snoop response layer 5859715SN/A if (forwardAsSnoop) { 58610572Sandreas.hansson@arm.com assert(dest_port_id < snoopLayers.size()); 5879715SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 58811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 58911744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 5909715SN/A return false; 5919715SN/A } 5929716SN/A } else { 5939716SN/A // get the master port that mirrors this slave port internally 5949716SN/A MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 59510572Sandreas.hansson@arm.com assert(dest_port_id < respLayers.size()); 5969716SN/A if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 59711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 59811744Snikos.nikoleris@arm.com snoop_port->name(), pkt->print()); 5999716SN/A return false; 6009716SN/A } 6018975SN/A } 6028975SN/A 60311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 60411744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 6058975SN/A 6069712SN/A // store size and command as they might be modified when 6079712SN/A // forwarding the packet 6089712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 6099712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 6109712SN/A 6118975SN/A // responses are never express snoops 6128975SN/A assert(!pkt->isExpressSnoop()); 6138975SN/A 61410719SMarco.Balboni@ARM.com // a snoop response sees the snoop response latency, and if it is 61510719SMarco.Balboni@ARM.com // forwarded as a normal response, the response latency 61610719SMarco.Balboni@ARM.com Tick xbar_delay = 61710719SMarco.Balboni@ARM.com (forwardAsSnoop ? snoopResponseLatency : responseLatency) * 61810719SMarco.Balboni@ARM.com clockPeriod(); 61910719SMarco.Balboni@ARM.com 62010719SMarco.Balboni@ARM.com // set the packet header and payload delay 62110719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 62210719SMarco.Balboni@ARM.com 62310719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 62410719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 6258975SN/A 6269714SN/A // forward it either as a snoop response or a normal response 6279714SN/A if (forwardAsSnoop) { 6289714SN/A // this is a snoop response to a snoop request we forwarded, 6299714SN/A // e.g. coming from the L1 and going to the L2, and it should 6309714SN/A // be forwarded as a snoop response 63110402SN/A 63210402SN/A if (snoopFilter) { 63310402SN/A // update the probe filter so that it can properly track the line 63410402SN/A snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 63510402SN/A *masterPorts[dest_port_id]); 63610402SN/A } 63710402SN/A 6389712SN/A bool success M5_VAR_USED = 6399712SN/A masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 6409712SN/A pktCount[slave_port_id][dest_port_id]++; 64110405Sandreas.hansson@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 6428975SN/A assert(success); 6439714SN/A 6449715SN/A snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 6453244SN/A } else { 6468975SN/A // we got a snoop response on one of our slave ports, 64710405Sandreas.hansson@arm.com // i.e. from a coherent master connected to the crossbar, and 64810405Sandreas.hansson@arm.com // since we created the snoop request as part of recvTiming, 64910405Sandreas.hansson@arm.com // this should now be a normal response again 65010656Sandreas.hansson@arm.com outstandingSnoop.erase(pkt->req); 6518948SN/A 65210656Sandreas.hansson@arm.com // this is a snoop response from a coherent master, hence it 65310656Sandreas.hansson@arm.com // should never go back to where the snoop response came from, 65410656Sandreas.hansson@arm.com // but instead to where the original request came from 6559712SN/A assert(slave_port_id != dest_port_id); 6568948SN/A 65710402SN/A if (snoopFilter) { 65810402SN/A // update the probe filter so that it can properly track the line 65910402SN/A snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 66010402SN/A *slavePorts[dest_port_id]); 66110402SN/A } 66210402SN/A 66311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s FWD RESP\n", __func__, 66411744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 66510402SN/A 6669714SN/A // as a normal response, it should go back to a master through 66710888Sandreas.hansson@arm.com // one of our slave ports, we also pay for any outstanding 66810888Sandreas.hansson@arm.com // header latency 66910888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 67010888Sandreas.hansson@arm.com pkt->headerDelay = 0; 67110888Sandreas.hansson@arm.com slavePorts[dest_port_id]->schedTimingResp(pkt, curTick() + latency); 6729716SN/A 6739716SN/A respLayers[dest_port_id]->succeededTiming(packetFinishTime); 6743244SN/A } 6753244SN/A 67610656Sandreas.hansson@arm.com // remove the request from the routing table 67710656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 67810656Sandreas.hansson@arm.com 6799712SN/A // stats updates 6809712SN/A transDist[pkt_cmd]++; 68110405Sandreas.hansson@arm.com snoops++; 68211564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 6839712SN/A 6848948SN/A return true; 6858948SN/A} 6868948SN/A 6873210SN/A 6888948SN/Avoid 68910405Sandreas.hansson@arm.comCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 69010888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 6918948SN/A{ 69211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s for %s\n", __func__, pkt->print()); 6939663SN/A 6949524SN/A // snoops should only happen if the system isn't bypassing caches 6959524SN/A assert(!system->bypassCaches()); 6969524SN/A 69710401SN/A unsigned fanout = 0; 69810401SN/A 69910405Sandreas.hansson@arm.com for (const auto& p: dests) { 7008948SN/A // we could have gotten this request from a snooping master 7018948SN/A // (corresponding to our own slave port that is also in 7028948SN/A // snoopPorts) and should not send it back to where it came 7038948SN/A // from 7049031SN/A if (exclude_slave_port_id == InvalidPortID || 7058948SN/A p->getId() != exclude_slave_port_id) { 7068948SN/A // cache is not allowed to refuse snoop 7078975SN/A p->sendTimingSnoopReq(pkt); 70810401SN/A fanout++; 7098948SN/A } 7108948SN/A } 71110401SN/A 71210401SN/A // Stats for fanout of this forward operation 71310401SN/A snoopFanout.sample(fanout); 7142497SN/A} 7152497SN/A 7169092SN/Avoid 71710713Sandreas.hansson@arm.comCoherentXBar::recvReqRetry(PortID master_port_id) 7189092SN/A{ 7199093SN/A // responses and snoop responses never block on forwarding them, 7209093SN/A // so the retry will always be coming from a port to which we 7219093SN/A // tried to forward a request 7229715SN/A reqLayers[master_port_id]->recvRetry(); 7239092SN/A} 7249092SN/A 7259036SN/ATick 72613847Sgabeblack@google.comCoherentXBar::recvAtomicBackdoor(PacketPtr pkt, PortID slave_port_id, 72713847Sgabeblack@google.com MemBackdoorPtr *backdoor) 7282657SN/A{ 72911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 73011744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 7318915SN/A 73210405Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 73310405Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 7349712SN/A 7358979SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 7368979SN/A Tick snoop_response_latency = 0; 7378979SN/A 73812346Snikos.nikoleris@arm.com // is this the destination point for this packet? (e.g. true if 73912346Snikos.nikoleris@arm.com // this xbar is the PoC for a cache maintenance operation to the 74012346Snikos.nikoleris@arm.com // PoC) otherwise the destination is any cache that can satisfy 74112346Snikos.nikoleris@arm.com // the request 74212346Snikos.nikoleris@arm.com const bool is_destination = isDestination(pkt); 74312346Snikos.nikoleris@arm.com 74412345Snikos.nikoleris@arm.com const bool snoop_caches = !system->bypassCaches() && 74512345Snikos.nikoleris@arm.com pkt->cmd != MemCmd::WriteClean; 74612345Snikos.nikoleris@arm.com if (snoop_caches) { 7478979SN/A // forward to all snoopers but the source 74810402SN/A std::pair<MemCmd, Tick> snoop_result; 74910402SN/A if (snoopFilter) { 75010402SN/A // check with the snoop filter where to forward this packet 75110402SN/A auto sf_res = 75210402SN/A snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 75310402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 75411744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 75511744Snikos.nikoleris@arm.com __func__, slavePorts[slave_port_id]->name(), pkt->print(), 75611744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 75711130Sali.jafri@arm.com 75811130Sali.jafri@arm.com // let the snoop filter know about the success of the send 75911130Sali.jafri@arm.com // operation, and do it even before sending it onwards to 76011130Sali.jafri@arm.com // avoid situations where atomic upward snoops sneak in 76111130Sali.jafri@arm.com // between and change the filter state 76211605Snikos.nikoleris@arm.com snoopFilter->finishRequest(false, pkt->getAddr(), pkt->isSecure()); 76311130Sali.jafri@arm.com 76412241Snikos.nikoleris@arm.com if (pkt->isEviction()) { 76512241Snikos.nikoleris@arm.com // for block-evicting packets, i.e. writebacks and 76612241Snikos.nikoleris@arm.com // clean evictions, there is no need to snoop up, as 76712241Snikos.nikoleris@arm.com // all we do is determine if the block is cached or 76812241Snikos.nikoleris@arm.com // not, instead just set it here based on the snoop 76912241Snikos.nikoleris@arm.com // filter result 77012241Snikos.nikoleris@arm.com if (!sf_res.first.empty()) 77112241Snikos.nikoleris@arm.com pkt->setBlockCached(); 77212241Snikos.nikoleris@arm.com } else { 77312241Snikos.nikoleris@arm.com snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 77412241Snikos.nikoleris@arm.com sf_res.first); 77512241Snikos.nikoleris@arm.com } 77610402SN/A } else { 77710402SN/A snoop_result = forwardAtomic(pkt, slave_port_id); 77810402SN/A } 7798979SN/A snoop_response_cmd = snoop_result.first; 78010402SN/A snoop_response_latency += snoop_result.second; 7818979SN/A } 7828915SN/A 78311334Sandreas.hansson@arm.com // set up a sensible default value 78411334Sandreas.hansson@arm.com Tick response_latency = 0; 78511334Sandreas.hansson@arm.com 78611334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 78711130Sali.jafri@arm.com 7888948SN/A // even if we had a snoop response, we must continue and also 7898948SN/A // perform the actual request at the destination 79013856Sodanrc@yahoo.com.br PortID master_port_id = findPort(pkt->getAddrRange()); 79110405Sandreas.hansson@arm.com 79211334Sandreas.hansson@arm.com if (sink_packet) { 79311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 79411744Snikos.nikoleris@arm.com pkt->print()); 79511334Sandreas.hansson@arm.com } else { 79612346Snikos.nikoleris@arm.com if (forwardPacket(pkt)) { 79712346Snikos.nikoleris@arm.com // make sure that the write request (e.g., WriteClean) 79812346Snikos.nikoleris@arm.com // will stop at the memory below if this crossbar is its 79912346Snikos.nikoleris@arm.com // destination 80012346Snikos.nikoleris@arm.com if (pkt->isWrite() && is_destination) { 80112346Snikos.nikoleris@arm.com pkt->clearWriteThrough(); 80212346Snikos.nikoleris@arm.com } 80312346Snikos.nikoleris@arm.com 80411334Sandreas.hansson@arm.com // forward the request to the appropriate destination 80513847Sgabeblack@google.com auto master = masterPorts[master_port_id]; 80613847Sgabeblack@google.com response_latency = backdoor ? 80713847Sgabeblack@google.com master->sendAtomicBackdoor(pkt, *backdoor) : 80813847Sgabeblack@google.com master->sendAtomic(pkt); 80911334Sandreas.hansson@arm.com } else { 81011334Sandreas.hansson@arm.com // if it does not need a response we sink the packet above 81111334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 81211334Sandreas.hansson@arm.com 81311334Sandreas.hansson@arm.com pkt->makeResponse(); 81411334Sandreas.hansson@arm.com } 81511334Sandreas.hansson@arm.com } 81611334Sandreas.hansson@arm.com 81710405Sandreas.hansson@arm.com // stats updates for the request 81810405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 81910405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 82010405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 8218948SN/A 8228948SN/A 82311130Sali.jafri@arm.com // if lower levels have replied, tell the snoop filter 82411130Sali.jafri@arm.com if (!system->bypassCaches() && snoopFilter && pkt->isResponse()) { 82510402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 82610402SN/A } 82710402SN/A 8288948SN/A // if we got a response from a snooper, restore it here 8298948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) { 8308948SN/A // no one else should have responded 8318948SN/A assert(!pkt->isResponse()); 8328948SN/A pkt->cmd = snoop_response_cmd; 8338948SN/A response_latency = snoop_response_latency; 8348948SN/A } 8358948SN/A 83612351Snikos.nikoleris@arm.com // If this is the destination of the cache clean operation the 83712351Snikos.nikoleris@arm.com // crossbar is responsible for responding. This crossbar will 83812351Snikos.nikoleris@arm.com // respond when the cache clean is complete. An atomic cache clean 83912351Snikos.nikoleris@arm.com // is complete when the crossbars receives the cache clean 84012351Snikos.nikoleris@arm.com // request (CleanSharedReq, CleanInvalidReq), as either: 84112351Snikos.nikoleris@arm.com // * no cache above had a dirty copy of the block as indicated by 84212351Snikos.nikoleris@arm.com // the satisfied flag of the packet, or 84312351Snikos.nikoleris@arm.com // * the crossbar has already seen the corresponding write 84412351Snikos.nikoleris@arm.com // (WriteClean) which updates the block in the memory below. 84512351Snikos.nikoleris@arm.com if (pkt->isClean() && isDestination(pkt) && pkt->satisfied()) { 84612351Snikos.nikoleris@arm.com auto it = outstandingCMO.find(pkt->id); 84712351Snikos.nikoleris@arm.com assert(it != outstandingCMO.end()); 84812351Snikos.nikoleris@arm.com // we are responding right away 84912351Snikos.nikoleris@arm.com outstandingCMO.erase(it); 85012351Snikos.nikoleris@arm.com } else if (pkt->cmd == MemCmd::WriteClean && isDestination(pkt)) { 85112351Snikos.nikoleris@arm.com // if this is the destination of the operation, the xbar 85212351Snikos.nikoleris@arm.com // sends the responce to the cache clean operation only 85312351Snikos.nikoleris@arm.com // after having encountered the cache clean request 85412351Snikos.nikoleris@arm.com auto M5_VAR_USED ret = outstandingCMO.emplace(pkt->id, nullptr); 85512351Snikos.nikoleris@arm.com // in atomic mode we know that the WriteClean packet should 85612351Snikos.nikoleris@arm.com // precede the clean request 85712351Snikos.nikoleris@arm.com assert(ret.second); 85812351Snikos.nikoleris@arm.com } 85912351Snikos.nikoleris@arm.com 8609712SN/A // add the response data 86110405Sandreas.hansson@arm.com if (pkt->isResponse()) { 86210405Sandreas.hansson@arm.com pkt_size = pkt->hasData() ? pkt->getSize() : 0; 86310405Sandreas.hansson@arm.com pkt_cmd = pkt->cmdToIndex(); 86410405Sandreas.hansson@arm.com 86510405Sandreas.hansson@arm.com // stats updates 86610405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 86710405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 86810405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 86910405Sandreas.hansson@arm.com } 8709712SN/A 87110694SMarco.Balboni@ARM.com // @todo: Not setting header time 87210694SMarco.Balboni@ARM.com pkt->payloadDelay = response_latency; 8738948SN/A return response_latency; 8748948SN/A} 8758948SN/A 8768948SN/ATick 87710405Sandreas.hansson@arm.comCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 8788948SN/A{ 87911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 88011744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 8818948SN/A 8829712SN/A // add the request snoop data 88311564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 88410405Sandreas.hansson@arm.com snoops++; 88511564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 8869712SN/A 8878948SN/A // forward to all snoopers 88810402SN/A std::pair<MemCmd, Tick> snoop_result; 88910402SN/A Tick snoop_response_latency = 0; 89010402SN/A if (snoopFilter) { 89110402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 89210402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 89311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 89411744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 89511744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 89610402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 89710402SN/A sf_res.first); 89810402SN/A } else { 89910402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID); 90010402SN/A } 9018948SN/A MemCmd snoop_response_cmd = snoop_result.first; 90210402SN/A snoop_response_latency += snoop_result.second; 9038948SN/A 9048948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) 9058948SN/A pkt->cmd = snoop_response_cmd; 9068948SN/A 9079712SN/A // add the response snoop data 90810401SN/A if (pkt->isResponse()) { 90910405Sandreas.hansson@arm.com snoops++; 91010401SN/A } 9119712SN/A 91210694SMarco.Balboni@ARM.com // @todo: Not setting header time 91310694SMarco.Balboni@ARM.com pkt->payloadDelay = snoop_response_latency; 9148948SN/A return snoop_response_latency; 9158948SN/A} 9168948SN/A 9178948SN/Astd::pair<MemCmd, Tick> 91810405Sandreas.hansson@arm.comCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 91910402SN/A PortID source_master_port_id, 92010888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 9218948SN/A{ 9229032SN/A // the packet may be changed on snoops, record the original 9239032SN/A // command to enable us to restore it between snoops so that 9248948SN/A // additional snoops can take place properly 9254626SN/A MemCmd orig_cmd = pkt->cmd; 9264879SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 9274879SN/A Tick snoop_response_latency = 0; 9283662SN/A 9299524SN/A // snoops should only happen if the system isn't bypassing caches 9309524SN/A assert(!system->bypassCaches()); 9319524SN/A 93210401SN/A unsigned fanout = 0; 93310401SN/A 93410405Sandreas.hansson@arm.com for (const auto& p: dests) { 9358915SN/A // we could have gotten this request from a snooping master 9368915SN/A // (corresponding to our own slave port that is also in 9378915SN/A // snoopPorts) and should not send it back to where it came 9388915SN/A // from 93910402SN/A if (exclude_slave_port_id != InvalidPortID && 94010402SN/A p->getId() == exclude_slave_port_id) 94110402SN/A continue; 94210401SN/A 94310402SN/A Tick latency = p->sendAtomicSnoop(pkt); 94410402SN/A fanout++; 94510402SN/A 94610402SN/A // in contrast to a functional access, we have to keep on 94710402SN/A // going as all snoopers must be updated even if we get a 94810402SN/A // response 94910402SN/A if (!pkt->isResponse()) 95010402SN/A continue; 95110402SN/A 95210402SN/A // response from snoop agent 95310402SN/A assert(pkt->cmd != orig_cmd); 95411284Sandreas.hansson@arm.com assert(pkt->cacheResponding()); 95510402SN/A // should only happen once 95610402SN/A assert(snoop_response_cmd == MemCmd::InvalidCmd); 95710402SN/A // save response state 95810402SN/A snoop_response_cmd = pkt->cmd; 95910402SN/A snoop_response_latency = latency; 96010402SN/A 96110402SN/A if (snoopFilter) { 96210402SN/A // Handle responses by the snoopers and differentiate between 96310402SN/A // responses to requests from above and snoops from below 96410402SN/A if (source_master_port_id != InvalidPortID) { 96510402SN/A // Getting a response for a snoop from below 96610402SN/A assert(exclude_slave_port_id == InvalidPortID); 96710402SN/A snoopFilter->updateSnoopForward(pkt, *p, 96810402SN/A *masterPorts[source_master_port_id]); 96910402SN/A } else { 97010402SN/A // Getting a response for a request from above 97110402SN/A assert(source_master_port_id == InvalidPortID); 97210402SN/A snoopFilter->updateSnoopResponse(pkt, *p, 97310402SN/A *slavePorts[exclude_slave_port_id]); 9744626SN/A } 9754626SN/A } 97610402SN/A // restore original packet state for remaining snoopers 97710402SN/A pkt->cmd = orig_cmd; 9784626SN/A } 9794626SN/A 98010401SN/A // Stats for fanout 98110401SN/A snoopFanout.sample(fanout); 98210401SN/A 9838948SN/A // the packet is restored as part of the loop and any potential 9848948SN/A // snoop response is part of the returned pair 9858948SN/A return std::make_pair(snoop_response_cmd, snoop_response_latency); 9862497SN/A} 9872497SN/A 9882497SN/Avoid 98910405Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 9902497SN/A{ 9918663SN/A if (!pkt->isPrint()) { 9928663SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 99311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 99411744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 9958663SN/A } 9968663SN/A 99710821Sandreas.hansson@arm.com if (!system->bypassCaches()) { 9988979SN/A // forward to all snoopers but the source 9999032SN/A forwardFunctional(pkt, slave_port_id); 10008979SN/A } 10014912SN/A 10028948SN/A // there is no need to continue if the snooping has found what we 10038948SN/A // were looking for and the packet is already a response 10048948SN/A if (!pkt->isResponse()) { 100510888Sandreas.hansson@arm.com // since our slave ports are queued ports we need to check them as well 100610888Sandreas.hansson@arm.com for (const auto& p : slavePorts) { 100710888Sandreas.hansson@arm.com // if we find a response that has the data, then the 100810888Sandreas.hansson@arm.com // downstream caches/memories may be out of date, so simply stop 100910888Sandreas.hansson@arm.com // here 101012823Srmk35@cl.cam.ac.uk if (p->trySatisfyFunctional(pkt)) { 101110888Sandreas.hansson@arm.com if (pkt->needsResponse()) 101210888Sandreas.hansson@arm.com pkt->makeResponse(); 101310888Sandreas.hansson@arm.com return; 101410888Sandreas.hansson@arm.com } 101510888Sandreas.hansson@arm.com } 101610888Sandreas.hansson@arm.com 101713856Sodanrc@yahoo.com.br PortID dest_id = findPort(pkt->getAddrRange()); 10188948SN/A 10198948SN/A masterPorts[dest_id]->sendFunctional(pkt); 10208948SN/A } 10218948SN/A} 10228948SN/A 10238948SN/Avoid 102410405Sandreas.hansson@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 10258948SN/A{ 10268948SN/A if (!pkt->isPrint()) { 10278948SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 102811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 102911744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 10308948SN/A } 10318948SN/A 103211188Sandreas.sandberg@arm.com for (const auto& p : slavePorts) { 103312823Srmk35@cl.cam.ac.uk if (p->trySatisfyFunctional(pkt)) { 103411188Sandreas.sandberg@arm.com if (pkt->needsResponse()) 103511188Sandreas.sandberg@arm.com pkt->makeResponse(); 103611188Sandreas.sandberg@arm.com return; 103711188Sandreas.sandberg@arm.com } 103811188Sandreas.sandberg@arm.com } 103911188Sandreas.sandberg@arm.com 10408948SN/A // forward to all snoopers 10419031SN/A forwardFunctional(pkt, InvalidPortID); 10428948SN/A} 10438948SN/A 10448948SN/Avoid 104510405Sandreas.hansson@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 10468948SN/A{ 10479524SN/A // snoops should only happen if the system isn't bypassing caches 10489524SN/A assert(!system->bypassCaches()); 10499524SN/A 105010405Sandreas.hansson@arm.com for (const auto& p: snoopPorts) { 10518915SN/A // we could have gotten this request from a snooping master 10528915SN/A // (corresponding to our own slave port that is also in 10538915SN/A // snoopPorts) and should not send it back to where it came 10548915SN/A // from 10559031SN/A if (exclude_slave_port_id == InvalidPortID || 10568948SN/A p->getId() != exclude_slave_port_id) 10578948SN/A p->sendFunctionalSnoop(pkt); 10588915SN/A 10598948SN/A // if we get a response we are done 10608948SN/A if (pkt->isResponse()) { 10618948SN/A break; 10628915SN/A } 10633650SN/A } 10642497SN/A} 10652497SN/A 106611334Sandreas.hansson@arm.combool 106711334Sandreas.hansson@arm.comCoherentXBar::sinkPacket(const PacketPtr pkt) const 106811334Sandreas.hansson@arm.com{ 106911334Sandreas.hansson@arm.com // we can sink the packet if: 107011334Sandreas.hansson@arm.com // 1) the crossbar is the point of coherency, and a cache is 107111334Sandreas.hansson@arm.com // responding after being snooped 107211334Sandreas.hansson@arm.com // 2) the crossbar is the point of coherency, and the packet is a 107311334Sandreas.hansson@arm.com // coherency packet (not a read or a write) that does not 107411334Sandreas.hansson@arm.com // require a response 107511334Sandreas.hansson@arm.com // 3) this is a clean evict or clean writeback, but the packet is 107611334Sandreas.hansson@arm.com // found in a cache above this crossbar 107711334Sandreas.hansson@arm.com // 4) a cache is responding after being snooped, and the packet 107811334Sandreas.hansson@arm.com // either does not need the block to be writable, or the cache 107911334Sandreas.hansson@arm.com // that has promised to respond (setting the cache responding 108011334Sandreas.hansson@arm.com // flag) is providing writable and thus had a Modified block, 108111334Sandreas.hansson@arm.com // and no further action is needed 108211334Sandreas.hansson@arm.com return (pointOfCoherency && pkt->cacheResponding()) || 108311334Sandreas.hansson@arm.com (pointOfCoherency && !(pkt->isRead() || pkt->isWrite()) && 108411334Sandreas.hansson@arm.com !pkt->needsResponse()) || 108511334Sandreas.hansson@arm.com (pkt->isCleanEviction() && pkt->isBlockCached()) || 108611334Sandreas.hansson@arm.com (pkt->cacheResponding() && 108711334Sandreas.hansson@arm.com (!pkt->needsWritable() || pkt->responderHadWritable())); 108811334Sandreas.hansson@arm.com} 108911334Sandreas.hansson@arm.com 109012346Snikos.nikoleris@arm.combool 109112346Snikos.nikoleris@arm.comCoherentXBar::forwardPacket(const PacketPtr pkt) 109212346Snikos.nikoleris@arm.com{ 109312346Snikos.nikoleris@arm.com // we are forwarding the packet if: 109412351Snikos.nikoleris@arm.com // 1) this is a cache clean request to the PoU/PoC and this 109512351Snikos.nikoleris@arm.com // crossbar is above the PoU/PoC 109612351Snikos.nikoleris@arm.com // 2) this is a read or a write 109712351Snikos.nikoleris@arm.com // 3) this crossbar is above the point of coherency 109812351Snikos.nikoleris@arm.com if (pkt->isClean()) { 109912351Snikos.nikoleris@arm.com return !isDestination(pkt); 110012351Snikos.nikoleris@arm.com } 110112346Snikos.nikoleris@arm.com return pkt->isRead() || pkt->isWrite() || !pointOfCoherency; 110212346Snikos.nikoleris@arm.com} 110312346Snikos.nikoleris@arm.com 110412346Snikos.nikoleris@arm.com 11059712SN/Avoid 110610405Sandreas.hansson@arm.comCoherentXBar::regStats() 11079712SN/A{ 110810405Sandreas.hansson@arm.com // register the stats of the base class and our layers 110910405Sandreas.hansson@arm.com BaseXBar::regStats(); 111010405Sandreas.hansson@arm.com for (auto l: reqLayers) 111110405Sandreas.hansson@arm.com l->regStats(); 111210405Sandreas.hansson@arm.com for (auto l: respLayers) 111310405Sandreas.hansson@arm.com l->regStats(); 111410405Sandreas.hansson@arm.com for (auto l: snoopLayers) 111510405Sandreas.hansson@arm.com l->regStats(); 11169712SN/A 111710405Sandreas.hansson@arm.com snoops 111810405Sandreas.hansson@arm.com .name(name() + ".snoops") 111910401SN/A .desc("Total snoops (count)") 112010401SN/A ; 112110401SN/A 112211564Sdavid.guillen@arm.com snoopTraffic 112311564Sdavid.guillen@arm.com .name(name() + ".snoopTraffic") 112411564Sdavid.guillen@arm.com .desc("Total snoop traffic (bytes)") 112511564Sdavid.guillen@arm.com ; 112611564Sdavid.guillen@arm.com 112710401SN/A snoopFanout 112810401SN/A .init(0, snoopPorts.size(), 1) 112910401SN/A .name(name() + ".snoop_fanout") 113010401SN/A .desc("Request fanout histogram") 113110401SN/A ; 11329712SN/A} 11339712SN/A 113410405Sandreas.hansson@arm.comCoherentXBar * 113510405Sandreas.hansson@arm.comCoherentXBarParams::create() 11362497SN/A{ 113710405Sandreas.hansson@arm.com return new CoherentXBar(this); 11382497SN/A} 1139