coherent_xbar.cc revision 12346
12497SN/A/* 212241Snikos.nikoleris@arm.com * Copyright (c) 2011-2017 ARM Limited 38711SN/A * All rights reserved 48711SN/A * 58711SN/A * The license below extends only to copyright in the software and shall 68711SN/A * not be construed as granting a license to any other intellectual 78711SN/A * property including but not limited to intellectual property relating 88711SN/A * to a hardware implementation of the functionality of the software 98711SN/A * licensed hereunder. You may use the software subject to the license 108711SN/A * terms below provided that you ensure that this notice is replicated 118711SN/A * unmodified and in its entirety in all distributions of the software, 128711SN/A * modified or unmodified, in source code or in binary form. 138711SN/A * 142497SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152497SN/A * All rights reserved. 162497SN/A * 172497SN/A * Redistribution and use in source and binary forms, with or without 182497SN/A * modification, are permitted provided that the following conditions are 192497SN/A * met: redistributions of source code must retain the above copyright 202497SN/A * notice, this list of conditions and the following disclaimer; 212497SN/A * redistributions in binary form must reproduce the above copyright 222497SN/A * notice, this list of conditions and the following disclaimer in the 232497SN/A * documentation and/or other materials provided with the distribution; 242497SN/A * neither the name of the copyright holders nor the names of its 252497SN/A * contributors may be used to endorse or promote products derived from 262497SN/A * this software without specific prior written permission. 272497SN/A * 282497SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292497SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302497SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312497SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322497SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332497SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342497SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352497SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362497SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372497SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 382497SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 392665SN/A * 402665SN/A * Authors: Ali Saidi 418715SN/A * Andreas Hansson 428922SN/A * William Wang 432497SN/A */ 442497SN/A 452497SN/A/** 462982SN/A * @file 4710405Sandreas.hansson@arm.com * Definition of a crossbar object. 482497SN/A */ 492497SN/A 5011793Sbrandon.potter@amd.com#include "mem/coherent_xbar.hh" 5111793Sbrandon.potter@amd.com 5212334Sgabeblack@google.com#include "base/logging.hh" 532548SN/A#include "base/trace.hh" 5410405Sandreas.hansson@arm.com#include "debug/AddrRanges.hh" 5510405Sandreas.hansson@arm.com#include "debug/CoherentXBar.hh" 569524SN/A#include "sim/system.hh" 572497SN/A 5810405Sandreas.hansson@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 5910719SMarco.Balboni@ARM.com : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter), 6011334Sandreas.hansson@arm.com snoopResponseLatency(p->snoop_response_latency), 6112341Snikos.nikoleris@arm.com pointOfCoherency(p->point_of_coherency), 6212341Snikos.nikoleris@arm.com pointOfUnification(p->point_of_unification) 637523SN/A{ 648851SN/A // create the ports based on the size of the master and slave 658948SN/A // vector ports, and the presence of the default port, the ports 668948SN/A // are enumerated starting from zero 678851SN/A for (int i = 0; i < p->port_master_connection_count; ++i) { 689095SN/A std::string portName = csprintf("%s.master[%d]", name(), i); 6910405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 708922SN/A masterPorts.push_back(bp); 719715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 729715SN/A csprintf(".reqLayer%d", i))); 7310713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 7410713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", i))); 758851SN/A } 768851SN/A 778948SN/A // see if we have a default slave device connected and if so add 788948SN/A // our corresponding master port 798915SN/A if (p->port_default_connection_count) { 809031SN/A defaultPortID = masterPorts.size(); 819095SN/A std::string portName = name() + ".default"; 8210405Sandreas.hansson@arm.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 839036SN/A defaultPortID); 848922SN/A masterPorts.push_back(bp); 859715SN/A reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 869715SN/A defaultPortID))); 8710713Sandreas.hansson@arm.com snoopLayers.push_back(new SnoopRespLayer(*bp, *this, 8810713Sandreas.hansson@arm.com csprintf(".snoopLayer%d", 8910713Sandreas.hansson@arm.com defaultPortID))); 908915SN/A } 918915SN/A 928948SN/A // create the slave ports, once again starting at zero 938851SN/A for (int i = 0; i < p->port_slave_connection_count; ++i) { 949095SN/A std::string portName = csprintf("%s.slave[%d]", name(), i); 9510888Sandreas.hansson@arm.com QueuedSlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 968922SN/A slavePorts.push_back(bp); 979715SN/A respLayers.push_back(new RespLayer(*bp, *this, 989715SN/A csprintf(".respLayer%d", i))); 999716SN/A snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 1008851SN/A } 1018851SN/A 1027523SN/A clearPortCache(); 1037523SN/A} 1047523SN/A 10510405Sandreas.hansson@arm.comCoherentXBar::~CoherentXBar() 1069715SN/A{ 10710405Sandreas.hansson@arm.com for (auto l: reqLayers) 10810405Sandreas.hansson@arm.com delete l; 10910405Sandreas.hansson@arm.com for (auto l: respLayers) 11010405Sandreas.hansson@arm.com delete l; 11110405Sandreas.hansson@arm.com for (auto l: snoopLayers) 11210405Sandreas.hansson@arm.com delete l; 11310405Sandreas.hansson@arm.com for (auto p: snoopRespPorts) 11410405Sandreas.hansson@arm.com delete p; 1159715SN/A} 1169715SN/A 1172568SN/Avoid 11810405Sandreas.hansson@arm.comCoherentXBar::init() 1192568SN/A{ 12010405Sandreas.hansson@arm.com BaseXBar::init(); 1219278SN/A 1228948SN/A // iterate over our slave ports and determine which of our 1238948SN/A // neighbouring master ports are snooping and add them as snoopers 12410405Sandreas.hansson@arm.com for (const auto& p: slavePorts) { 1259088SN/A // check if the connected master port is snooping 12610405Sandreas.hansson@arm.com if (p->isSnooping()) { 12710405Sandreas.hansson@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 12810405Sandreas.hansson@arm.com p->getMasterPort().name()); 12910405Sandreas.hansson@arm.com snoopPorts.push_back(p); 1308711SN/A } 1318711SN/A } 1322568SN/A 1339036SN/A if (snoopPorts.empty()) 13410405Sandreas.hansson@arm.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 13511133Sandreas.hansson@arm.com 13611133Sandreas.hansson@arm.com // inform the snoop filter about the slave ports so it can create 13711133Sandreas.hansson@arm.com // its own internal representation 13811133Sandreas.hansson@arm.com if (snoopFilter) 13911133Sandreas.hansson@arm.com snoopFilter->setSlavePorts(slavePorts); 1403244SN/A} 1413244SN/A 1428948SN/Abool 14310405Sandreas.hansson@arm.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1443244SN/A{ 1458975SN/A // determine the source port based on the id 1469032SN/A SlavePort *src_port = slavePorts[slave_port_id]; 1473244SN/A 1489091SN/A // remember if the packet is an express snoop 1499091SN/A bool is_express_snoop = pkt->isExpressSnoop(); 15011284Sandreas.hansson@arm.com bool cache_responding = pkt->cacheResponding(); 15110656Sandreas.hansson@arm.com // for normal requests, going downstream, the express snoop flag 15211284Sandreas.hansson@arm.com // and the cache responding flag should always be the same 15311284Sandreas.hansson@arm.com assert(is_express_snoop == cache_responding); 1549091SN/A 1559612SN/A // determine the destination based on the address 1569712SN/A PortID master_port_id = findPort(pkt->getAddr()); 1579612SN/A 15810405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the current 1599033SN/A // port, and exclude express snoops from the check 1609715SN/A if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 16111744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 16211744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1633244SN/A return false; 1643244SN/A } 1653244SN/A 16611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 16711744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 1685197SN/A 1699712SN/A // store size and command as they might be modified when 1709712SN/A // forwarding the packet 1719712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1729712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 1739712SN/A 17410719SMarco.Balboni@ARM.com // store the old header delay so we can restore it if needed 17510719SMarco.Balboni@ARM.com Tick old_header_delay = pkt->headerDelay; 17610719SMarco.Balboni@ARM.com 17710719SMarco.Balboni@ARM.com // a request sees the frontend and forward latency 17810719SMarco.Balboni@ARM.com Tick xbar_delay = (frontendLatency + forwardLatency) * clockPeriod(); 17910719SMarco.Balboni@ARM.com 18010719SMarco.Balboni@ARM.com // set the packet header and payload delay 18110719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 18210719SMarco.Balboni@ARM.com 18310719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 18410719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 1854912SN/A 18612346Snikos.nikoleris@arm.com // is this the destination point for this packet? (e.g. true if 18712346Snikos.nikoleris@arm.com // this xbar is the PoC for a cache maintenance operation to the 18812346Snikos.nikoleris@arm.com // PoC) otherwise the destination is any cache that can satisfy 18912346Snikos.nikoleris@arm.com // the request 19012346Snikos.nikoleris@arm.com const bool is_destination = isDestination(pkt); 19112346Snikos.nikoleris@arm.com 19212345Snikos.nikoleris@arm.com const bool snoop_caches = !system->bypassCaches() && 19312345Snikos.nikoleris@arm.com pkt->cmd != MemCmd::WriteClean; 19412345Snikos.nikoleris@arm.com if (snoop_caches) { 19511127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 19611127Sandreas.hansson@arm.com 1978979SN/A // the packet is a memory-mapped request and should be 1988979SN/A // broadcasted to our snoopers but the source 19910402SN/A if (snoopFilter) { 20010402SN/A // check with the snoop filter where to forward this packet 20110402SN/A auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 20211126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 20311126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 20411126Sandreas.hansson@arm.com // of the snoop filter 20510719SMarco.Balboni@ARM.com pkt->headerDelay += sf_res.second * clockPeriod(); 20611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 20711744Snikos.nikoleris@arm.com __func__, src_port->name(), pkt->print(), 20811744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 20911196Sali.jafri@arm.com 21011199Sandreas.hansson@arm.com if (pkt->isEviction()) { 21111196Sali.jafri@arm.com // for block-evicting packets, i.e. writebacks and 21211196Sali.jafri@arm.com // clean evictions, there is no need to snoop up, as 21311196Sali.jafri@arm.com // all we do is determine if the block is cached or 21411196Sali.jafri@arm.com // not, instead just set it here based on the snoop 21511196Sali.jafri@arm.com // filter result 21611196Sali.jafri@arm.com if (!sf_res.first.empty()) 21711196Sali.jafri@arm.com pkt->setBlockCached(); 21811196Sali.jafri@arm.com } else { 21911196Sali.jafri@arm.com forwardTiming(pkt, slave_port_id, sf_res.first); 22011196Sali.jafri@arm.com } 22110402SN/A } else { 22210402SN/A forwardTiming(pkt, slave_port_id); 22310402SN/A } 22411127Sandreas.hansson@arm.com 22511127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 22611127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 22711127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 2288979SN/A } 2298948SN/A 23011334Sandreas.hansson@arm.com // set up a sensible starting point 23111334Sandreas.hansson@arm.com bool success = true; 23210883Sali.jafri@arm.com 23311284Sandreas.hansson@arm.com // remember if the packet will generate a snoop response by 23411284Sandreas.hansson@arm.com // checking if a cache set the cacheResponding flag during the 23511284Sandreas.hansson@arm.com // snooping above 23611284Sandreas.hansson@arm.com const bool expect_snoop_resp = !cache_responding && pkt->cacheResponding(); 23711334Sandreas.hansson@arm.com bool expect_response = pkt->needsResponse() && !pkt->cacheResponding(); 2388915SN/A 23911334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 24011334Sandreas.hansson@arm.com 24111334Sandreas.hansson@arm.com // in certain cases the crossbar is responsible for responding 24211334Sandreas.hansson@arm.com bool respond_directly = false; 24311544Snikos.nikoleris@arm.com // store the original address as an address mapper could possibly 24411544Snikos.nikoleris@arm.com // modify the address upon a sendTimingRequest 24511544Snikos.nikoleris@arm.com const Addr addr(pkt->getAddr()); 24611334Sandreas.hansson@arm.com if (sink_packet) { 24711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 24811744Snikos.nikoleris@arm.com pkt->print()); 24911334Sandreas.hansson@arm.com } else { 25011334Sandreas.hansson@arm.com // determine if we are forwarding the packet, or responding to 25111334Sandreas.hansson@arm.com // it 25212346Snikos.nikoleris@arm.com if (forwardPacket(pkt)) { 25311334Sandreas.hansson@arm.com // if we are passing on, rather than sinking, a packet to 25411334Sandreas.hansson@arm.com // which an upstream cache has committed to responding, 25511334Sandreas.hansson@arm.com // the line was needs writable, and the responding only 25611334Sandreas.hansson@arm.com // had an Owned copy, so we need to immidiately let the 25711334Sandreas.hansson@arm.com // downstream caches know, bypass any flow control 25811334Sandreas.hansson@arm.com if (pkt->cacheResponding()) { 25911334Sandreas.hansson@arm.com pkt->setExpressSnoop(); 26011334Sandreas.hansson@arm.com } 26111334Sandreas.hansson@arm.com 26212346Snikos.nikoleris@arm.com // make sure that the write request (e.g., WriteClean) 26312346Snikos.nikoleris@arm.com // will stop at the memory below if this crossbar is its 26412346Snikos.nikoleris@arm.com // destination 26512346Snikos.nikoleris@arm.com if (pkt->isWrite() && is_destination) { 26612346Snikos.nikoleris@arm.com pkt->clearWriteThrough(); 26712346Snikos.nikoleris@arm.com } 26812346Snikos.nikoleris@arm.com 26911334Sandreas.hansson@arm.com // since it is a normal request, attempt to send the packet 27011334Sandreas.hansson@arm.com success = masterPorts[master_port_id]->sendTimingReq(pkt); 27111334Sandreas.hansson@arm.com } else { 27211334Sandreas.hansson@arm.com // no need to forward, turn this packet around and respond 27311334Sandreas.hansson@arm.com // directly 27411334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 27511334Sandreas.hansson@arm.com 27611334Sandreas.hansson@arm.com respond_directly = true; 27711334Sandreas.hansson@arm.com assert(!expect_snoop_resp); 27811334Sandreas.hansson@arm.com expect_response = false; 27911334Sandreas.hansson@arm.com } 28011334Sandreas.hansson@arm.com } 2818948SN/A 28212345Snikos.nikoleris@arm.com if (snoopFilter && snoop_caches) { 28310402SN/A // Let the snoop filter know about the success of the send operation 28411605Snikos.nikoleris@arm.com snoopFilter->finishRequest(!success, addr, pkt->isSecure()); 28510402SN/A } 28610402SN/A 28710656Sandreas.hansson@arm.com // check if we were successful in sending the packet onwards 28810656Sandreas.hansson@arm.com if (!success) { 28911284Sandreas.hansson@arm.com // express snoops should never be forced to retry 29010656Sandreas.hansson@arm.com assert(!is_express_snoop); 29110656Sandreas.hansson@arm.com 29210719SMarco.Balboni@ARM.com // restore the header delay 29310719SMarco.Balboni@ARM.com pkt->headerDelay = old_header_delay; 29410656Sandreas.hansson@arm.com 29511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s RETRY\n", __func__, 29611744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 29710656Sandreas.hansson@arm.com 29810656Sandreas.hansson@arm.com // update the layer state and schedule an idle event 29910656Sandreas.hansson@arm.com reqLayers[master_port_id]->failedTiming(src_port, 30010719SMarco.Balboni@ARM.com clockEdge(Cycles(1))); 3019091SN/A } else { 30210656Sandreas.hansson@arm.com // express snoops currently bypass the crossbar state entirely 30310656Sandreas.hansson@arm.com if (!is_express_snoop) { 30410656Sandreas.hansson@arm.com // if this particular request will generate a snoop 30510656Sandreas.hansson@arm.com // response 30610656Sandreas.hansson@arm.com if (expect_snoop_resp) { 30710656Sandreas.hansson@arm.com // we should never have an exsiting request outstanding 30810656Sandreas.hansson@arm.com assert(outstandingSnoop.find(pkt->req) == 30910656Sandreas.hansson@arm.com outstandingSnoop.end()); 31010656Sandreas.hansson@arm.com outstandingSnoop.insert(pkt->req); 3118948SN/A 31210656Sandreas.hansson@arm.com // basic sanity check on the outstanding snoops 31310656Sandreas.hansson@arm.com panic_if(outstandingSnoop.size() > 512, 31410656Sandreas.hansson@arm.com "Outstanding snoop requests exceeded 512\n"); 31510656Sandreas.hansson@arm.com } 3168948SN/A 31710656Sandreas.hansson@arm.com // remember where to route the normal response to 31810656Sandreas.hansson@arm.com if (expect_response || expect_snoop_resp) { 31910656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 32010656Sandreas.hansson@arm.com routeTo[pkt->req] = slave_port_id; 3219549SN/A 32210656Sandreas.hansson@arm.com panic_if(routeTo.size() > 512, 32310656Sandreas.hansson@arm.com "Routing table exceeds 512 packets\n"); 32410656Sandreas.hansson@arm.com } 3258948SN/A 32610405Sandreas.hansson@arm.com // update the layer state and schedule an idle event 3279715SN/A reqLayers[master_port_id]->succeededTiming(packetFinishTime); 3289091SN/A } 3298975SN/A 33010656Sandreas.hansson@arm.com // stats updates only consider packets that were successfully sent 3319712SN/A pktCount[slave_port_id][master_port_id]++; 33210405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 3339712SN/A transDist[pkt_cmd]++; 33410656Sandreas.hansson@arm.com 33511564Sdavid.guillen@arm.com if (is_express_snoop) { 33610656Sandreas.hansson@arm.com snoops++; 33711564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 33811564Sdavid.guillen@arm.com } 3399712SN/A } 3409712SN/A 34111334Sandreas.hansson@arm.com if (sink_packet) 34211334Sandreas.hansson@arm.com // queue the packet for deletion 34311334Sandreas.hansson@arm.com pendingDelete.reset(pkt); 34411334Sandreas.hansson@arm.com 34511334Sandreas.hansson@arm.com if (respond_directly) { 34611334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 34711334Sandreas.hansson@arm.com assert(success); 34811334Sandreas.hansson@arm.com 34911334Sandreas.hansson@arm.com pkt->makeResponse(); 35011334Sandreas.hansson@arm.com 35111334Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 35211334Sandreas.hansson@arm.com // let the snoop filter inspect the response and update its state 35311334Sandreas.hansson@arm.com snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 35411334Sandreas.hansson@arm.com } 35511334Sandreas.hansson@arm.com 35611334Sandreas.hansson@arm.com Tick response_time = clockEdge() + pkt->headerDelay; 35711334Sandreas.hansson@arm.com pkt->headerDelay = 0; 35811334Sandreas.hansson@arm.com 35911334Sandreas.hansson@arm.com slavePorts[slave_port_id]->schedTimingResp(pkt, response_time); 36011334Sandreas.hansson@arm.com } 36111334Sandreas.hansson@arm.com 3629091SN/A return success; 3638975SN/A} 3648975SN/A 3658975SN/Abool 36610405Sandreas.hansson@arm.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 3678975SN/A{ 3688975SN/A // determine the source port based on the id 3699032SN/A MasterPort *src_port = masterPorts[master_port_id]; 3708975SN/A 37110656Sandreas.hansson@arm.com // determine the destination 37210656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 37310656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 37410656Sandreas.hansson@arm.com const PortID slave_port_id = route_lookup->second; 37510572Sandreas.hansson@arm.com assert(slave_port_id != InvalidPortID); 37610572Sandreas.hansson@arm.com assert(slave_port_id < respLayers.size()); 3779713SN/A 37810405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 37910405Sandreas.hansson@arm.com // current port 3809715SN/A if (!respLayers[slave_port_id]->tryTiming(src_port)) { 38111744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 38211744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 3838975SN/A return false; 3848975SN/A } 3858975SN/A 38611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 38711744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 3888975SN/A 3899712SN/A // store size and command as they might be modified when 3909712SN/A // forwarding the packet 3919712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 3929712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 3939712SN/A 39410719SMarco.Balboni@ARM.com // a response sees the response latency 39510719SMarco.Balboni@ARM.com Tick xbar_delay = responseLatency * clockPeriod(); 39610719SMarco.Balboni@ARM.com 39710719SMarco.Balboni@ARM.com // set the packet header and payload delay 39810719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 39910719SMarco.Balboni@ARM.com 40010719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 40110719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 4028975SN/A 40310821Sandreas.hansson@arm.com if (snoopFilter && !system->bypassCaches()) { 40410402SN/A // let the snoop filter inspect the response and update its state 40510402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 40610402SN/A } 40710402SN/A 40810888Sandreas.hansson@arm.com // send the packet through the destination slave port and pay for 40910888Sandreas.hansson@arm.com // any outstanding header delay 41010888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 41110888Sandreas.hansson@arm.com pkt->headerDelay = 0; 41210888Sandreas.hansson@arm.com slavePorts[slave_port_id]->schedTimingResp(pkt, curTick() + latency); 4138975SN/A 41410656Sandreas.hansson@arm.com // remove the request from the routing table 41510656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 41610656Sandreas.hansson@arm.com 4179715SN/A respLayers[slave_port_id]->succeededTiming(packetFinishTime); 4188975SN/A 4199712SN/A // stats updates 4209712SN/A pktCount[slave_port_id][master_port_id]++; 42110405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 4229712SN/A transDist[pkt_cmd]++; 4239712SN/A 4248975SN/A return true; 4258975SN/A} 4268975SN/A 4278975SN/Avoid 42810405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 4298975SN/A{ 43011744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 43111744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 4328975SN/A 4339712SN/A // update stats here as we know the forwarding will succeed 43411564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 4359712SN/A transDist[pkt->cmdToIndex()]++; 43610405Sandreas.hansson@arm.com snoops++; 43711564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 4389712SN/A 4398975SN/A // we should only see express snoops from caches 4408975SN/A assert(pkt->isExpressSnoop()); 4418975SN/A 44211127Sandreas.hansson@arm.com // set the packet header and payload delay, for now use forward latency 44311127Sandreas.hansson@arm.com // @todo Assess the choice of latency further 44411127Sandreas.hansson@arm.com calcPacketTiming(pkt, forwardLatency * clockPeriod()); 44511127Sandreas.hansson@arm.com 44611284Sandreas.hansson@arm.com // remember if a cache has already committed to responding so we 44711284Sandreas.hansson@arm.com // can see if it changes during the snooping 44811284Sandreas.hansson@arm.com const bool cache_responding = pkt->cacheResponding(); 4499032SN/A 45011127Sandreas.hansson@arm.com assert(pkt->snoopDelay == 0); 45111127Sandreas.hansson@arm.com 45210402SN/A if (snoopFilter) { 45310402SN/A // let the Snoop Filter work its magic and guide probing 45410402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 45511126Sandreas.hansson@arm.com // the time required by a packet to be delivered through 45611126Sandreas.hansson@arm.com // the xbar has to be charged also with to lookup latency 45711126Sandreas.hansson@arm.com // of the snoop filter 45811126Sandreas.hansson@arm.com pkt->headerDelay += sf_res.second * clockPeriod(); 45911744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 46011744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 46111744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 46210402SN/A 46310402SN/A // forward to all snoopers 46410402SN/A forwardTiming(pkt, InvalidPortID, sf_res.first); 46510402SN/A } else { 46610402SN/A forwardTiming(pkt, InvalidPortID); 46710402SN/A } 4688975SN/A 46911127Sandreas.hansson@arm.com // add the snoop delay to our header delay, and then reset it 47011127Sandreas.hansson@arm.com pkt->headerDelay += pkt->snoopDelay; 47111127Sandreas.hansson@arm.com pkt->snoopDelay = 0; 47211127Sandreas.hansson@arm.com 47310656Sandreas.hansson@arm.com // if we can expect a response, remember how to route it 47411284Sandreas.hansson@arm.com if (!cache_responding && pkt->cacheResponding()) { 47510656Sandreas.hansson@arm.com assert(routeTo.find(pkt->req) == routeTo.end()); 47610656Sandreas.hansson@arm.com routeTo[pkt->req] = master_port_id; 47710656Sandreas.hansson@arm.com } 47810656Sandreas.hansson@arm.com 4798975SN/A // a snoop request came from a connected slave device (one of 4808975SN/A // our master ports), and if it is not coming from the slave 4818975SN/A // device responsible for the address range something is 4828975SN/A // wrong, hence there is nothing further to do as the packet 4838975SN/A // would be going back to where it came from 4849032SN/A assert(master_port_id == findPort(pkt->getAddr())); 4858975SN/A} 4868975SN/A 4878975SN/Abool 48810405Sandreas.hansson@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 4898975SN/A{ 4908975SN/A // determine the source port based on the id 4919032SN/A SlavePort* src_port = slavePorts[slave_port_id]; 4928975SN/A 49310656Sandreas.hansson@arm.com // get the destination 49410656Sandreas.hansson@arm.com const auto route_lookup = routeTo.find(pkt->req); 49510656Sandreas.hansson@arm.com assert(route_lookup != routeTo.end()); 49610656Sandreas.hansson@arm.com const PortID dest_port_id = route_lookup->second; 49710572Sandreas.hansson@arm.com assert(dest_port_id != InvalidPortID); 4989714SN/A 4999714SN/A // determine if the response is from a snoop request we 5009714SN/A // created as the result of a normal request (in which case it 50110656Sandreas.hansson@arm.com // should be in the outstandingSnoop), or if we merely forwarded 5029714SN/A // someone else's snoop request 50310656Sandreas.hansson@arm.com const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 50410656Sandreas.hansson@arm.com outstandingSnoop.end(); 5059714SN/A 50610405Sandreas.hansson@arm.com // test if the crossbar should be considered occupied for the 50710405Sandreas.hansson@arm.com // current port, note that the check is bypassed if the response 50810405Sandreas.hansson@arm.com // is being passed on as a normal response since this is occupying 50910405Sandreas.hansson@arm.com // the response layer rather than the snoop response layer 5109715SN/A if (forwardAsSnoop) { 51110572Sandreas.hansson@arm.com assert(dest_port_id < snoopLayers.size()); 5129715SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 51311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 51411744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 5159715SN/A return false; 5169715SN/A } 5179716SN/A } else { 5189716SN/A // get the master port that mirrors this slave port internally 5199716SN/A MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 52010572Sandreas.hansson@arm.com assert(dest_port_id < respLayers.size()); 5219716SN/A if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 52211744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s BUSY\n", __func__, 52311744Snikos.nikoleris@arm.com snoop_port->name(), pkt->print()); 5249716SN/A return false; 5259716SN/A } 5268975SN/A } 5278975SN/A 52811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 52911744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 5308975SN/A 5319712SN/A // store size and command as they might be modified when 5329712SN/A // forwarding the packet 5339712SN/A unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 5349712SN/A unsigned int pkt_cmd = pkt->cmdToIndex(); 5359712SN/A 5368975SN/A // responses are never express snoops 5378975SN/A assert(!pkt->isExpressSnoop()); 5388975SN/A 53910719SMarco.Balboni@ARM.com // a snoop response sees the snoop response latency, and if it is 54010719SMarco.Balboni@ARM.com // forwarded as a normal response, the response latency 54110719SMarco.Balboni@ARM.com Tick xbar_delay = 54210719SMarco.Balboni@ARM.com (forwardAsSnoop ? snoopResponseLatency : responseLatency) * 54310719SMarco.Balboni@ARM.com clockPeriod(); 54410719SMarco.Balboni@ARM.com 54510719SMarco.Balboni@ARM.com // set the packet header and payload delay 54610719SMarco.Balboni@ARM.com calcPacketTiming(pkt, xbar_delay); 54710719SMarco.Balboni@ARM.com 54810719SMarco.Balboni@ARM.com // determine how long to be crossbar layer is busy 54910719SMarco.Balboni@ARM.com Tick packetFinishTime = clockEdge(Cycles(1)) + pkt->payloadDelay; 5508975SN/A 5519714SN/A // forward it either as a snoop response or a normal response 5529714SN/A if (forwardAsSnoop) { 5539714SN/A // this is a snoop response to a snoop request we forwarded, 5549714SN/A // e.g. coming from the L1 and going to the L2, and it should 5559714SN/A // be forwarded as a snoop response 55610402SN/A 55710402SN/A if (snoopFilter) { 55810402SN/A // update the probe filter so that it can properly track the line 55910402SN/A snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 56010402SN/A *masterPorts[dest_port_id]); 56110402SN/A } 56210402SN/A 5639712SN/A bool success M5_VAR_USED = 5649712SN/A masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 5659712SN/A pktCount[slave_port_id][dest_port_id]++; 56610405Sandreas.hansson@arm.com pktSize[slave_port_id][dest_port_id] += pkt_size; 5678975SN/A assert(success); 5689714SN/A 5699715SN/A snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 5703244SN/A } else { 5718975SN/A // we got a snoop response on one of our slave ports, 57210405Sandreas.hansson@arm.com // i.e. from a coherent master connected to the crossbar, and 57310405Sandreas.hansson@arm.com // since we created the snoop request as part of recvTiming, 57410405Sandreas.hansson@arm.com // this should now be a normal response again 57510656Sandreas.hansson@arm.com outstandingSnoop.erase(pkt->req); 5768948SN/A 57710656Sandreas.hansson@arm.com // this is a snoop response from a coherent master, hence it 57810656Sandreas.hansson@arm.com // should never go back to where the snoop response came from, 57910656Sandreas.hansson@arm.com // but instead to where the original request came from 5809712SN/A assert(slave_port_id != dest_port_id); 5818948SN/A 58210402SN/A if (snoopFilter) { 58310402SN/A // update the probe filter so that it can properly track the line 58410402SN/A snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 58510402SN/A *slavePorts[dest_port_id]); 58610402SN/A } 58710402SN/A 58811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s FWD RESP\n", __func__, 58911744Snikos.nikoleris@arm.com src_port->name(), pkt->print()); 59010402SN/A 5919714SN/A // as a normal response, it should go back to a master through 59210888Sandreas.hansson@arm.com // one of our slave ports, we also pay for any outstanding 59310888Sandreas.hansson@arm.com // header latency 59410888Sandreas.hansson@arm.com Tick latency = pkt->headerDelay; 59510888Sandreas.hansson@arm.com pkt->headerDelay = 0; 59610888Sandreas.hansson@arm.com slavePorts[dest_port_id]->schedTimingResp(pkt, curTick() + latency); 5979716SN/A 5989716SN/A respLayers[dest_port_id]->succeededTiming(packetFinishTime); 5993244SN/A } 6003244SN/A 60110656Sandreas.hansson@arm.com // remove the request from the routing table 60210656Sandreas.hansson@arm.com routeTo.erase(route_lookup); 60310656Sandreas.hansson@arm.com 6049712SN/A // stats updates 6059712SN/A transDist[pkt_cmd]++; 60610405Sandreas.hansson@arm.com snoops++; 60711564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 6089712SN/A 6098948SN/A return true; 6108948SN/A} 6118948SN/A 6123210SN/A 6138948SN/Avoid 61410405Sandreas.hansson@arm.comCoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 61510888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 6168948SN/A{ 61711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s for %s\n", __func__, pkt->print()); 6189663SN/A 6199524SN/A // snoops should only happen if the system isn't bypassing caches 6209524SN/A assert(!system->bypassCaches()); 6219524SN/A 62210401SN/A unsigned fanout = 0; 62310401SN/A 62410405Sandreas.hansson@arm.com for (const auto& p: dests) { 6258948SN/A // we could have gotten this request from a snooping master 6268948SN/A // (corresponding to our own slave port that is also in 6278948SN/A // snoopPorts) and should not send it back to where it came 6288948SN/A // from 6299031SN/A if (exclude_slave_port_id == InvalidPortID || 6308948SN/A p->getId() != exclude_slave_port_id) { 6318948SN/A // cache is not allowed to refuse snoop 6328975SN/A p->sendTimingSnoopReq(pkt); 63310401SN/A fanout++; 6348948SN/A } 6358948SN/A } 63610401SN/A 63710401SN/A // Stats for fanout of this forward operation 63810401SN/A snoopFanout.sample(fanout); 6392497SN/A} 6402497SN/A 6419092SN/Avoid 64210713Sandreas.hansson@arm.comCoherentXBar::recvReqRetry(PortID master_port_id) 6439092SN/A{ 6449093SN/A // responses and snoop responses never block on forwarding them, 6459093SN/A // so the retry will always be coming from a port to which we 6469093SN/A // tried to forward a request 6479715SN/A reqLayers[master_port_id]->recvRetry(); 6489092SN/A} 6499092SN/A 6509036SN/ATick 65110405Sandreas.hansson@arm.comCoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 6522657SN/A{ 65311744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 65411744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 6558915SN/A 65610405Sandreas.hansson@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 65710405Sandreas.hansson@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 6589712SN/A 6598979SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 6608979SN/A Tick snoop_response_latency = 0; 6618979SN/A 66212346Snikos.nikoleris@arm.com // is this the destination point for this packet? (e.g. true if 66312346Snikos.nikoleris@arm.com // this xbar is the PoC for a cache maintenance operation to the 66412346Snikos.nikoleris@arm.com // PoC) otherwise the destination is any cache that can satisfy 66512346Snikos.nikoleris@arm.com // the request 66612346Snikos.nikoleris@arm.com const bool is_destination = isDestination(pkt); 66712346Snikos.nikoleris@arm.com 66812345Snikos.nikoleris@arm.com const bool snoop_caches = !system->bypassCaches() && 66912345Snikos.nikoleris@arm.com pkt->cmd != MemCmd::WriteClean; 67012345Snikos.nikoleris@arm.com if (snoop_caches) { 6718979SN/A // forward to all snoopers but the source 67210402SN/A std::pair<MemCmd, Tick> snoop_result; 67310402SN/A if (snoopFilter) { 67410402SN/A // check with the snoop filter where to forward this packet 67510402SN/A auto sf_res = 67610402SN/A snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 67710402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 67811744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 67911744Snikos.nikoleris@arm.com __func__, slavePorts[slave_port_id]->name(), pkt->print(), 68011744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 68111130Sali.jafri@arm.com 68211130Sali.jafri@arm.com // let the snoop filter know about the success of the send 68311130Sali.jafri@arm.com // operation, and do it even before sending it onwards to 68411130Sali.jafri@arm.com // avoid situations where atomic upward snoops sneak in 68511130Sali.jafri@arm.com // between and change the filter state 68611605Snikos.nikoleris@arm.com snoopFilter->finishRequest(false, pkt->getAddr(), pkt->isSecure()); 68711130Sali.jafri@arm.com 68812241Snikos.nikoleris@arm.com if (pkt->isEviction()) { 68912241Snikos.nikoleris@arm.com // for block-evicting packets, i.e. writebacks and 69012241Snikos.nikoleris@arm.com // clean evictions, there is no need to snoop up, as 69112241Snikos.nikoleris@arm.com // all we do is determine if the block is cached or 69212241Snikos.nikoleris@arm.com // not, instead just set it here based on the snoop 69312241Snikos.nikoleris@arm.com // filter result 69412241Snikos.nikoleris@arm.com if (!sf_res.first.empty()) 69512241Snikos.nikoleris@arm.com pkt->setBlockCached(); 69612241Snikos.nikoleris@arm.com } else { 69712241Snikos.nikoleris@arm.com snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 69812241Snikos.nikoleris@arm.com sf_res.first); 69912241Snikos.nikoleris@arm.com } 70010402SN/A } else { 70110402SN/A snoop_result = forwardAtomic(pkt, slave_port_id); 70210402SN/A } 7038979SN/A snoop_response_cmd = snoop_result.first; 70410402SN/A snoop_response_latency += snoop_result.second; 7058979SN/A } 7068915SN/A 70711334Sandreas.hansson@arm.com // set up a sensible default value 70811334Sandreas.hansson@arm.com Tick response_latency = 0; 70911334Sandreas.hansson@arm.com 71011334Sandreas.hansson@arm.com const bool sink_packet = sinkPacket(pkt); 71111130Sali.jafri@arm.com 7128948SN/A // even if we had a snoop response, we must continue and also 7138948SN/A // perform the actual request at the destination 71410405Sandreas.hansson@arm.com PortID master_port_id = findPort(pkt->getAddr()); 71510405Sandreas.hansson@arm.com 71611334Sandreas.hansson@arm.com if (sink_packet) { 71711744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: Not forwarding %s\n", __func__, 71811744Snikos.nikoleris@arm.com pkt->print()); 71911334Sandreas.hansson@arm.com } else { 72012346Snikos.nikoleris@arm.com if (forwardPacket(pkt)) { 72112346Snikos.nikoleris@arm.com // make sure that the write request (e.g., WriteClean) 72212346Snikos.nikoleris@arm.com // will stop at the memory below if this crossbar is its 72312346Snikos.nikoleris@arm.com // destination 72412346Snikos.nikoleris@arm.com if (pkt->isWrite() && is_destination) { 72512346Snikos.nikoleris@arm.com pkt->clearWriteThrough(); 72612346Snikos.nikoleris@arm.com } 72712346Snikos.nikoleris@arm.com 72811334Sandreas.hansson@arm.com // forward the request to the appropriate destination 72911334Sandreas.hansson@arm.com response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 73011334Sandreas.hansson@arm.com } else { 73111334Sandreas.hansson@arm.com // if it does not need a response we sink the packet above 73211334Sandreas.hansson@arm.com assert(pkt->needsResponse()); 73311334Sandreas.hansson@arm.com 73411334Sandreas.hansson@arm.com pkt->makeResponse(); 73511334Sandreas.hansson@arm.com } 73611334Sandreas.hansson@arm.com } 73711334Sandreas.hansson@arm.com 73810405Sandreas.hansson@arm.com // stats updates for the request 73910405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 74010405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 74110405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 7428948SN/A 7438948SN/A 74411130Sali.jafri@arm.com // if lower levels have replied, tell the snoop filter 74511130Sali.jafri@arm.com if (!system->bypassCaches() && snoopFilter && pkt->isResponse()) { 74610402SN/A snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 74710402SN/A } 74810402SN/A 7498948SN/A // if we got a response from a snooper, restore it here 7508948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) { 7518948SN/A // no one else should have responded 7528948SN/A assert(!pkt->isResponse()); 7538948SN/A pkt->cmd = snoop_response_cmd; 7548948SN/A response_latency = snoop_response_latency; 7558948SN/A } 7568948SN/A 7579712SN/A // add the response data 75810405Sandreas.hansson@arm.com if (pkt->isResponse()) { 75910405Sandreas.hansson@arm.com pkt_size = pkt->hasData() ? pkt->getSize() : 0; 76010405Sandreas.hansson@arm.com pkt_cmd = pkt->cmdToIndex(); 76110405Sandreas.hansson@arm.com 76210405Sandreas.hansson@arm.com // stats updates 76310405Sandreas.hansson@arm.com pktCount[slave_port_id][master_port_id]++; 76410405Sandreas.hansson@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 76510405Sandreas.hansson@arm.com transDist[pkt_cmd]++; 76610405Sandreas.hansson@arm.com } 7679712SN/A 76810694SMarco.Balboni@ARM.com // @todo: Not setting header time 76910694SMarco.Balboni@ARM.com pkt->payloadDelay = response_latency; 7708948SN/A return response_latency; 7718948SN/A} 7728948SN/A 7738948SN/ATick 77410405Sandreas.hansson@arm.comCoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 7758948SN/A{ 77611744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 77711744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 7788948SN/A 7799712SN/A // add the request snoop data 78011564Sdavid.guillen@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 78110405Sandreas.hansson@arm.com snoops++; 78211564Sdavid.guillen@arm.com snoopTraffic += pkt_size; 7839712SN/A 7848948SN/A // forward to all snoopers 78510402SN/A std::pair<MemCmd, Tick> snoop_result; 78610402SN/A Tick snoop_response_latency = 0; 78710402SN/A if (snoopFilter) { 78810402SN/A auto sf_res = snoopFilter->lookupSnoop(pkt); 78910402SN/A snoop_response_latency += sf_res.second * clockPeriod(); 79011744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s SF size: %i lat: %i\n", 79111744Snikos.nikoleris@arm.com __func__, masterPorts[master_port_id]->name(), pkt->print(), 79211744Snikos.nikoleris@arm.com sf_res.first.size(), sf_res.second); 79310402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 79410402SN/A sf_res.first); 79510402SN/A } else { 79610402SN/A snoop_result = forwardAtomic(pkt, InvalidPortID); 79710402SN/A } 7988948SN/A MemCmd snoop_response_cmd = snoop_result.first; 79910402SN/A snoop_response_latency += snoop_result.second; 8008948SN/A 8018948SN/A if (snoop_response_cmd != MemCmd::InvalidCmd) 8028948SN/A pkt->cmd = snoop_response_cmd; 8038948SN/A 8049712SN/A // add the response snoop data 80510401SN/A if (pkt->isResponse()) { 80610405Sandreas.hansson@arm.com snoops++; 80710401SN/A } 8089712SN/A 80910694SMarco.Balboni@ARM.com // @todo: Not setting header time 81010694SMarco.Balboni@ARM.com pkt->payloadDelay = snoop_response_latency; 8118948SN/A return snoop_response_latency; 8128948SN/A} 8138948SN/A 8148948SN/Astd::pair<MemCmd, Tick> 81510405Sandreas.hansson@arm.comCoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 81610402SN/A PortID source_master_port_id, 81710888Sandreas.hansson@arm.com const std::vector<QueuedSlavePort*>& dests) 8188948SN/A{ 8199032SN/A // the packet may be changed on snoops, record the original 8209032SN/A // command to enable us to restore it between snoops so that 8218948SN/A // additional snoops can take place properly 8224626SN/A MemCmd orig_cmd = pkt->cmd; 8234879SN/A MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 8244879SN/A Tick snoop_response_latency = 0; 8253662SN/A 8269524SN/A // snoops should only happen if the system isn't bypassing caches 8279524SN/A assert(!system->bypassCaches()); 8289524SN/A 82910401SN/A unsigned fanout = 0; 83010401SN/A 83110405Sandreas.hansson@arm.com for (const auto& p: dests) { 8328915SN/A // we could have gotten this request from a snooping master 8338915SN/A // (corresponding to our own slave port that is also in 8348915SN/A // snoopPorts) and should not send it back to where it came 8358915SN/A // from 83610402SN/A if (exclude_slave_port_id != InvalidPortID && 83710402SN/A p->getId() == exclude_slave_port_id) 83810402SN/A continue; 83910401SN/A 84010402SN/A Tick latency = p->sendAtomicSnoop(pkt); 84110402SN/A fanout++; 84210402SN/A 84310402SN/A // in contrast to a functional access, we have to keep on 84410402SN/A // going as all snoopers must be updated even if we get a 84510402SN/A // response 84610402SN/A if (!pkt->isResponse()) 84710402SN/A continue; 84810402SN/A 84910402SN/A // response from snoop agent 85010402SN/A assert(pkt->cmd != orig_cmd); 85111284Sandreas.hansson@arm.com assert(pkt->cacheResponding()); 85210402SN/A // should only happen once 85310402SN/A assert(snoop_response_cmd == MemCmd::InvalidCmd); 85410402SN/A // save response state 85510402SN/A snoop_response_cmd = pkt->cmd; 85610402SN/A snoop_response_latency = latency; 85710402SN/A 85810402SN/A if (snoopFilter) { 85910402SN/A // Handle responses by the snoopers and differentiate between 86010402SN/A // responses to requests from above and snoops from below 86110402SN/A if (source_master_port_id != InvalidPortID) { 86210402SN/A // Getting a response for a snoop from below 86310402SN/A assert(exclude_slave_port_id == InvalidPortID); 86410402SN/A snoopFilter->updateSnoopForward(pkt, *p, 86510402SN/A *masterPorts[source_master_port_id]); 86610402SN/A } else { 86710402SN/A // Getting a response for a request from above 86810402SN/A assert(source_master_port_id == InvalidPortID); 86910402SN/A snoopFilter->updateSnoopResponse(pkt, *p, 87010402SN/A *slavePorts[exclude_slave_port_id]); 8714626SN/A } 8724626SN/A } 87310402SN/A // restore original packet state for remaining snoopers 87410402SN/A pkt->cmd = orig_cmd; 8754626SN/A } 8764626SN/A 87710401SN/A // Stats for fanout 87810401SN/A snoopFanout.sample(fanout); 87910401SN/A 8808948SN/A // the packet is restored as part of the loop and any potential 8818948SN/A // snoop response is part of the returned pair 8828948SN/A return std::make_pair(snoop_response_cmd, snoop_response_latency); 8832497SN/A} 8842497SN/A 8852497SN/Avoid 88610405Sandreas.hansson@arm.comCoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 8872497SN/A{ 8888663SN/A if (!pkt->isPrint()) { 8898663SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 89011744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 89111744Snikos.nikoleris@arm.com slavePorts[slave_port_id]->name(), pkt->print()); 8928663SN/A } 8938663SN/A 89410821Sandreas.hansson@arm.com if (!system->bypassCaches()) { 8958979SN/A // forward to all snoopers but the source 8969032SN/A forwardFunctional(pkt, slave_port_id); 8978979SN/A } 8984912SN/A 8998948SN/A // there is no need to continue if the snooping has found what we 9008948SN/A // were looking for and the packet is already a response 9018948SN/A if (!pkt->isResponse()) { 90210888Sandreas.hansson@arm.com // since our slave ports are queued ports we need to check them as well 90310888Sandreas.hansson@arm.com for (const auto& p : slavePorts) { 90410888Sandreas.hansson@arm.com // if we find a response that has the data, then the 90510888Sandreas.hansson@arm.com // downstream caches/memories may be out of date, so simply stop 90610888Sandreas.hansson@arm.com // here 90710888Sandreas.hansson@arm.com if (p->checkFunctional(pkt)) { 90810888Sandreas.hansson@arm.com if (pkt->needsResponse()) 90910888Sandreas.hansson@arm.com pkt->makeResponse(); 91010888Sandreas.hansson@arm.com return; 91110888Sandreas.hansson@arm.com } 91210888Sandreas.hansson@arm.com } 91310888Sandreas.hansson@arm.com 9149031SN/A PortID dest_id = findPort(pkt->getAddr()); 9158948SN/A 9168948SN/A masterPorts[dest_id]->sendFunctional(pkt); 9178948SN/A } 9188948SN/A} 9198948SN/A 9208948SN/Avoid 92110405Sandreas.hansson@arm.comCoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 9228948SN/A{ 9238948SN/A if (!pkt->isPrint()) { 9248948SN/A // don't do DPRINTFs on PrintReq as it clutters up the output 92511744Snikos.nikoleris@arm.com DPRINTF(CoherentXBar, "%s: src %s packet %s\n", __func__, 92611744Snikos.nikoleris@arm.com masterPorts[master_port_id]->name(), pkt->print()); 9278948SN/A } 9288948SN/A 92911188Sandreas.sandberg@arm.com for (const auto& p : slavePorts) { 93011188Sandreas.sandberg@arm.com if (p->checkFunctional(pkt)) { 93111188Sandreas.sandberg@arm.com if (pkt->needsResponse()) 93211188Sandreas.sandberg@arm.com pkt->makeResponse(); 93311188Sandreas.sandberg@arm.com return; 93411188Sandreas.sandberg@arm.com } 93511188Sandreas.sandberg@arm.com } 93611188Sandreas.sandberg@arm.com 9378948SN/A // forward to all snoopers 9389031SN/A forwardFunctional(pkt, InvalidPortID); 9398948SN/A} 9408948SN/A 9418948SN/Avoid 94210405Sandreas.hansson@arm.comCoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 9438948SN/A{ 9449524SN/A // snoops should only happen if the system isn't bypassing caches 9459524SN/A assert(!system->bypassCaches()); 9469524SN/A 94710405Sandreas.hansson@arm.com for (const auto& p: snoopPorts) { 9488915SN/A // we could have gotten this request from a snooping master 9498915SN/A // (corresponding to our own slave port that is also in 9508915SN/A // snoopPorts) and should not send it back to where it came 9518915SN/A // from 9529031SN/A if (exclude_slave_port_id == InvalidPortID || 9538948SN/A p->getId() != exclude_slave_port_id) 9548948SN/A p->sendFunctionalSnoop(pkt); 9558915SN/A 9568948SN/A // if we get a response we are done 9578948SN/A if (pkt->isResponse()) { 9588948SN/A break; 9598915SN/A } 9603650SN/A } 9612497SN/A} 9622497SN/A 96311334Sandreas.hansson@arm.combool 96411334Sandreas.hansson@arm.comCoherentXBar::sinkPacket(const PacketPtr pkt) const 96511334Sandreas.hansson@arm.com{ 96611334Sandreas.hansson@arm.com // we can sink the packet if: 96711334Sandreas.hansson@arm.com // 1) the crossbar is the point of coherency, and a cache is 96811334Sandreas.hansson@arm.com // responding after being snooped 96911334Sandreas.hansson@arm.com // 2) the crossbar is the point of coherency, and the packet is a 97011334Sandreas.hansson@arm.com // coherency packet (not a read or a write) that does not 97111334Sandreas.hansson@arm.com // require a response 97211334Sandreas.hansson@arm.com // 3) this is a clean evict or clean writeback, but the packet is 97311334Sandreas.hansson@arm.com // found in a cache above this crossbar 97411334Sandreas.hansson@arm.com // 4) a cache is responding after being snooped, and the packet 97511334Sandreas.hansson@arm.com // either does not need the block to be writable, or the cache 97611334Sandreas.hansson@arm.com // that has promised to respond (setting the cache responding 97711334Sandreas.hansson@arm.com // flag) is providing writable and thus had a Modified block, 97811334Sandreas.hansson@arm.com // and no further action is needed 97911334Sandreas.hansson@arm.com return (pointOfCoherency && pkt->cacheResponding()) || 98011334Sandreas.hansson@arm.com (pointOfCoherency && !(pkt->isRead() || pkt->isWrite()) && 98111334Sandreas.hansson@arm.com !pkt->needsResponse()) || 98211334Sandreas.hansson@arm.com (pkt->isCleanEviction() && pkt->isBlockCached()) || 98311334Sandreas.hansson@arm.com (pkt->cacheResponding() && 98411334Sandreas.hansson@arm.com (!pkt->needsWritable() || pkt->responderHadWritable())); 98511334Sandreas.hansson@arm.com} 98611334Sandreas.hansson@arm.com 98712346Snikos.nikoleris@arm.combool 98812346Snikos.nikoleris@arm.comCoherentXBar::forwardPacket(const PacketPtr pkt) 98912346Snikos.nikoleris@arm.com{ 99012346Snikos.nikoleris@arm.com // we are forwarding the packet if: 99112346Snikos.nikoleris@arm.com // 1) this is a read or a write 99212346Snikos.nikoleris@arm.com // 2) this crossbar is above the point of coherency 99312346Snikos.nikoleris@arm.com return pkt->isRead() || pkt->isWrite() || !pointOfCoherency; 99412346Snikos.nikoleris@arm.com} 99512346Snikos.nikoleris@arm.com 99612346Snikos.nikoleris@arm.com 9979712SN/Avoid 99810405Sandreas.hansson@arm.comCoherentXBar::regStats() 9999712SN/A{ 100010405Sandreas.hansson@arm.com // register the stats of the base class and our layers 100110405Sandreas.hansson@arm.com BaseXBar::regStats(); 100210405Sandreas.hansson@arm.com for (auto l: reqLayers) 100310405Sandreas.hansson@arm.com l->regStats(); 100410405Sandreas.hansson@arm.com for (auto l: respLayers) 100510405Sandreas.hansson@arm.com l->regStats(); 100610405Sandreas.hansson@arm.com for (auto l: snoopLayers) 100710405Sandreas.hansson@arm.com l->regStats(); 10089712SN/A 100910405Sandreas.hansson@arm.com snoops 101010405Sandreas.hansson@arm.com .name(name() + ".snoops") 101110401SN/A .desc("Total snoops (count)") 101210401SN/A ; 101310401SN/A 101411564Sdavid.guillen@arm.com snoopTraffic 101511564Sdavid.guillen@arm.com .name(name() + ".snoopTraffic") 101611564Sdavid.guillen@arm.com .desc("Total snoop traffic (bytes)") 101711564Sdavid.guillen@arm.com ; 101811564Sdavid.guillen@arm.com 101910401SN/A snoopFanout 102010401SN/A .init(0, snoopPorts.size(), 1) 102110401SN/A .name(name() + ".snoop_fanout") 102210401SN/A .desc("Request fanout histogram") 102310401SN/A ; 10249712SN/A} 10259712SN/A 102610405Sandreas.hansson@arm.comCoherentXBar * 102710405Sandreas.hansson@arm.comCoherentXBarParams::create() 10282497SN/A{ 102910405Sandreas.hansson@arm.com return new CoherentXBar(this); 10302497SN/A} 1031