coherent_xbar.cc revision 10405
11689SN/A/* 212106SRekai.GonzalezAlberquilla@arm.com * Copyright (c) 2011-2014 ARM Limited 310715SRekai.GonzalezAlberquilla@arm.com * All rights reserved 410715SRekai.GonzalezAlberquilla@arm.com * 510715SRekai.GonzalezAlberquilla@arm.com * The license below extends only to copyright in the software and shall 610715SRekai.GonzalezAlberquilla@arm.com * not be construed as granting a license to any other intellectual 710715SRekai.GonzalezAlberquilla@arm.com * property including but not limited to intellectual property relating 810715SRekai.GonzalezAlberquilla@arm.com * to a hardware implementation of the functionality of the software 910715SRekai.GonzalezAlberquilla@arm.com * licensed hereunder. You may use the software subject to the license 1010715SRekai.GonzalezAlberquilla@arm.com * terms below provided that you ensure that this notice is replicated 1110715SRekai.GonzalezAlberquilla@arm.com * unmodified and in its entirety in all distributions of the software, 1210715SRekai.GonzalezAlberquilla@arm.com * modified or unmodified, in source code or in binary form. 1310715SRekai.GonzalezAlberquilla@arm.com * 141689SN/A * Copyright (c) 2006 The Regents of The University of Michigan 159919Ssteve.reinhardt@amd.com * All rights reserved. 161689SN/A * 171689SN/A * Redistribution and use in source and binary forms, with or without 181689SN/A * modification, are permitted provided that the following conditions are 191689SN/A * met: redistributions of source code must retain the above copyright 201689SN/A * notice, this list of conditions and the following disclaimer; 211689SN/A * redistributions in binary form must reproduce the above copyright 221689SN/A * notice, this list of conditions and the following disclaimer in the 231689SN/A * documentation and/or other materials provided with the distribution; 241689SN/A * neither the name of the copyright holders nor the names of its 251689SN/A * contributors may be used to endorse or promote products derived from 261689SN/A * this software without specific prior written permission. 271689SN/A * 281689SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 291689SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 301689SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 311689SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 321689SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 331689SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 341689SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 351689SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 361689SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 371689SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 381689SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 391689SN/A * 402665Ssaidi@eecs.umich.edu * Authors: Ali Saidi 412665Ssaidi@eecs.umich.edu * Andreas Hansson 429919Ssteve.reinhardt@amd.com * William Wang 431689SN/A */ 441689SN/A 452292SN/A/** 462292SN/A * @file 471060SN/A * Definition of a crossbar object. 481060SN/A */ 491461SN/A 501060SN/A#include "base/misc.hh" 511060SN/A#include "base/trace.hh" 526658Snate@binkert.org#include "debug/AddrRanges.hh" 536658Snate@binkert.org#include "debug/CoherentXBar.hh" 541717SN/A#include "mem/coherent_xbar.hh" 559919Ssteve.reinhardt@amd.com#include "sim/system.hh" 569919Ssteve.reinhardt@amd.com 5712109SRekai.GonzalezAlberquilla@arm.comCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 581060SN/A : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 599919Ssteve.reinhardt@amd.com{ 609919Ssteve.reinhardt@amd.com // create the ports based on the size of the master and slave 619919Ssteve.reinhardt@amd.com // vector ports, and the presence of the default port, the ports 629919Ssteve.reinhardt@amd.com // are enumerated starting from zero 639919Ssteve.reinhardt@amd.com for (int i = 0; i < p->port_master_connection_count; ++i) { 649919Ssteve.reinhardt@amd.com std::string portName = csprintf("%s.master[%d]", name(), i); 659919Ssteve.reinhardt@amd.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 661060SN/A masterPorts.push_back(bp); 671060SN/A reqLayers.push_back(new ReqLayer(*bp, *this, 689919Ssteve.reinhardt@amd.com csprintf(".reqLayer%d", i))); 6912106SRekai.GonzalezAlberquilla@arm.com snoopLayers.push_back(new SnoopLayer(*bp, *this, 709919Ssteve.reinhardt@amd.com csprintf(".snoopLayer%d", i))); 7112106SRekai.GonzalezAlberquilla@arm.com } 7212109SRekai.GonzalezAlberquilla@arm.com 7312109SRekai.GonzalezAlberquilla@arm.com // see if we have a default slave device connected and if so add 7412109SRekai.GonzalezAlberquilla@arm.com // our corresponding master port 7512109SRekai.GonzalezAlberquilla@arm.com if (p->port_default_connection_count) { 769919Ssteve.reinhardt@amd.com defaultPortID = masterPorts.size(); 779919Ssteve.reinhardt@amd.com std::string portName = name() + ".default"; 789919Ssteve.reinhardt@amd.com MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 799919Ssteve.reinhardt@amd.com defaultPortID); 809919Ssteve.reinhardt@amd.com masterPorts.push_back(bp); 819919Ssteve.reinhardt@amd.com reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 829919Ssteve.reinhardt@amd.com defaultPortID))); 839919Ssteve.reinhardt@amd.com snoopLayers.push_back(new SnoopLayer(*bp, *this, 849919Ssteve.reinhardt@amd.com csprintf(".snoopLayer%d", 859919Ssteve.reinhardt@amd.com defaultPortID))); 869919Ssteve.reinhardt@amd.com } 879919Ssteve.reinhardt@amd.com 889919Ssteve.reinhardt@amd.com // create the slave ports, once again starting at zero 899919Ssteve.reinhardt@amd.com for (int i = 0; i < p->port_slave_connection_count; ++i) { 9012106SRekai.GonzalezAlberquilla@arm.com std::string portName = csprintf("%s.slave[%d]", name(), i); 919919Ssteve.reinhardt@amd.com SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 921060SN/A slavePorts.push_back(bp); 939919Ssteve.reinhardt@amd.com respLayers.push_back(new RespLayer(*bp, *this, 949919Ssteve.reinhardt@amd.com csprintf(".respLayer%d", i))); 959919Ssteve.reinhardt@amd.com snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 969919Ssteve.reinhardt@amd.com } 979919Ssteve.reinhardt@amd.com 981060SN/A if (snoopFilter) 999919Ssteve.reinhardt@amd.com snoopFilter->setSlavePorts(slavePorts); 1009919Ssteve.reinhardt@amd.com 1019919Ssteve.reinhardt@amd.com clearPortCache(); 1021060SN/A} 1039919Ssteve.reinhardt@amd.com 1041060SN/ACoherentXBar::~CoherentXBar() 1051060SN/A{ 1061060SN/A for (auto l: reqLayers) 1071060SN/A delete l; 1081060SN/A for (auto l: respLayers) 1091060SN/A delete l; 1101060SN/A for (auto l: snoopLayers) 11112105Snathanael.premillieu@arm.com delete l; 1121060SN/A for (auto p: snoopRespPorts) 1139919Ssteve.reinhardt@amd.com delete p; 1149919Ssteve.reinhardt@amd.com} 1159919Ssteve.reinhardt@amd.com 1169919Ssteve.reinhardt@amd.comvoid 1179919Ssteve.reinhardt@amd.comCoherentXBar::init() 1189919Ssteve.reinhardt@amd.com{ 1199919Ssteve.reinhardt@amd.com // the base class is responsible for determining the block size 12012106SRekai.GonzalezAlberquilla@arm.com BaseXBar::init(); 1219919Ssteve.reinhardt@amd.com 1229919Ssteve.reinhardt@amd.com // iterate over our slave ports and determine which of our 1239919Ssteve.reinhardt@amd.com // neighbouring master ports are snooping and add them as snoopers 1249919Ssteve.reinhardt@amd.com for (const auto& p: slavePorts) { 1259919Ssteve.reinhardt@amd.com // check if the connected master port is snooping 1269919Ssteve.reinhardt@amd.com if (p->isSnooping()) { 12712106SRekai.GonzalezAlberquilla@arm.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 1289919Ssteve.reinhardt@amd.com p->getMasterPort().name()); 12912106SRekai.GonzalezAlberquilla@arm.com snoopPorts.push_back(p); 13012106SRekai.GonzalezAlberquilla@arm.com } 1319919Ssteve.reinhardt@amd.com } 1329919Ssteve.reinhardt@amd.com 1339919Ssteve.reinhardt@amd.com if (snoopPorts.empty()) 1349919Ssteve.reinhardt@amd.com warn("CoherentXBar %s has no snooping ports attached!\n", name()); 1359919Ssteve.reinhardt@amd.com} 1369919Ssteve.reinhardt@amd.com 1379919Ssteve.reinhardt@amd.combool 1389919Ssteve.reinhardt@amd.comCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 13912106SRekai.GonzalezAlberquilla@arm.com{ 1409919Ssteve.reinhardt@amd.com // determine the source port based on the id 14112106SRekai.GonzalezAlberquilla@arm.com SlavePort *src_port = slavePorts[slave_port_id]; 14212106SRekai.GonzalezAlberquilla@arm.com 1439919Ssteve.reinhardt@amd.com // remember if the packet is an express snoop 1449919Ssteve.reinhardt@amd.com bool is_express_snoop = pkt->isExpressSnoop(); 1459919Ssteve.reinhardt@amd.com 1469919Ssteve.reinhardt@amd.com // determine the destination based on the address 14712109SRekai.GonzalezAlberquilla@arm.com PortID master_port_id = findPort(pkt->getAddr()); 14812109SRekai.GonzalezAlberquilla@arm.com 14912109SRekai.GonzalezAlberquilla@arm.com // test if the crossbar should be considered occupied for the current 15012109SRekai.GonzalezAlberquilla@arm.com // port, and exclude express snoops from the check 15112109SRekai.GonzalezAlberquilla@arm.com if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 15212109SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 15312109SRekai.GonzalezAlberquilla@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 15412109SRekai.GonzalezAlberquilla@arm.com return false; 15512109SRekai.GonzalezAlberquilla@arm.com } 15612109SRekai.GonzalezAlberquilla@arm.com 15712109SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 15812109SRekai.GonzalezAlberquilla@arm.com src_port->name(), pkt->cmdString(), is_express_snoop, 15912109SRekai.GonzalezAlberquilla@arm.com pkt->getAddr()); 16012109SRekai.GonzalezAlberquilla@arm.com 1619919Ssteve.reinhardt@amd.com // store size and command as they might be modified when 1629919Ssteve.reinhardt@amd.com // forwarding the packet 1639919Ssteve.reinhardt@amd.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1649919Ssteve.reinhardt@amd.com unsigned int pkt_cmd = pkt->cmdToIndex(); 1659919Ssteve.reinhardt@amd.com 1669919Ssteve.reinhardt@amd.com // set the source port for routing of the response 16712104Snathanael.premillieu@arm.com pkt->setSrc(slave_port_id); 1689919Ssteve.reinhardt@amd.com 16912104Snathanael.premillieu@arm.com calcPacketTiming(pkt); 1709919Ssteve.reinhardt@amd.com Tick packetFinishTime = pkt->lastWordDelay + curTick(); 1719919Ssteve.reinhardt@amd.com 1729919Ssteve.reinhardt@amd.com // uncacheable requests need never be snooped 1739919Ssteve.reinhardt@amd.com if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 17412109SRekai.GonzalezAlberquilla@arm.com // the packet is a memory-mapped request and should be 17512109SRekai.GonzalezAlberquilla@arm.com // broadcasted to our snoopers but the source 1769919Ssteve.reinhardt@amd.com if (snoopFilter) { 1779919Ssteve.reinhardt@amd.com // check with the snoop filter where to forward this packet 1789919Ssteve.reinhardt@amd.com auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 1799919Ssteve.reinhardt@amd.com packetFinishTime += sf_res.second * clockPeriod(); 1809919Ssteve.reinhardt@amd.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 1819919Ssteve.reinhardt@amd.com " SF size: %i lat: %i\n", src_port->name(), 1829919Ssteve.reinhardt@amd.com pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 18312105Snathanael.premillieu@arm.com sf_res.second); 18412105Snathanael.premillieu@arm.com forwardTiming(pkt, slave_port_id, sf_res.first); 18512105Snathanael.premillieu@arm.com } else { 18612109SRekai.GonzalezAlberquilla@arm.com forwardTiming(pkt, slave_port_id); 18712109SRekai.GonzalezAlberquilla@arm.com } 18812109SRekai.GonzalezAlberquilla@arm.com } 18912109SRekai.GonzalezAlberquilla@arm.com 19012109SRekai.GonzalezAlberquilla@arm.com // remember if we add an outstanding req so we can undo it if 19112109SRekai.GonzalezAlberquilla@arm.com // necessary, if the packet needs a response, we should add it 19212109SRekai.GonzalezAlberquilla@arm.com // as outstanding and express snoops never fail so there is 19312109SRekai.GonzalezAlberquilla@arm.com // not need to worry about them 19412109SRekai.GonzalezAlberquilla@arm.com bool add_outstanding = !is_express_snoop && pkt->needsResponse(); 1959919Ssteve.reinhardt@amd.com 19612105Snathanael.premillieu@arm.com // keep track that we have an outstanding request packet 19712105Snathanael.premillieu@arm.com // matching this request, this is used by the coherency 1989919Ssteve.reinhardt@amd.com // mechanism in determining what to do with snoop responses 1999919Ssteve.reinhardt@amd.com // (in recvTimingSnoop) 2009919Ssteve.reinhardt@amd.com if (add_outstanding) { 2011060SN/A // we should never have an exsiting request outstanding 2029919Ssteve.reinhardt@amd.com assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 2039919Ssteve.reinhardt@amd.com outstandingReq.insert(pkt->req); 2049919Ssteve.reinhardt@amd.com } 2052348SN/A 20610537Sandreas.hansson@arm.com // Note: Cannot create a copy of the full packet, here. 2071060SN/A MemCmd orig_cmd(pkt->cmd); 2081061SN/A 2099919Ssteve.reinhardt@amd.com // since it is a normal request, attempt to send the packet 2101061SN/A bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 2112348SN/A 2129919Ssteve.reinhardt@amd.com if (snoopFilter && !pkt->req->isUncacheable() 2132292SN/A && !system->bypassCaches()) { 2142292SN/A // The packet may already be overwritten by the sendTimingReq function. 21512109SRekai.GonzalezAlberquilla@arm.com // The snoop filter needs to see the original request *and* the return 21612109SRekai.GonzalezAlberquilla@arm.com // status of the send operation, so we need to recreate the original 2172292SN/A // request. Atomic mode does not have the issue, as there the send 2189919Ssteve.reinhardt@amd.com // operation and the response happen instantaneously and don't need two 2199919Ssteve.reinhardt@amd.com // phase tracking. 22012104Snathanael.premillieu@arm.com MemCmd tmp_cmd(pkt->cmd); 22112106SRekai.GonzalezAlberquilla@arm.com pkt->cmd = orig_cmd; 22212106SRekai.GonzalezAlberquilla@arm.com // Let the snoop filter know about the success of the send operation 2239919Ssteve.reinhardt@amd.com snoopFilter->updateRequest(pkt, *src_port, !success); 2249919Ssteve.reinhardt@amd.com pkt->cmd = tmp_cmd; 2259919Ssteve.reinhardt@amd.com } 22612106SRekai.GonzalezAlberquilla@arm.com 22712106SRekai.GonzalezAlberquilla@arm.com // if this is an express snoop, we are done at this point 22812106SRekai.GonzalezAlberquilla@arm.com if (is_express_snoop) { 22912106SRekai.GonzalezAlberquilla@arm.com assert(success); 23012106SRekai.GonzalezAlberquilla@arm.com snoops++; 23112106SRekai.GonzalezAlberquilla@arm.com } else { 23212106SRekai.GonzalezAlberquilla@arm.com // for normal requests, check if successful 23312109SRekai.GonzalezAlberquilla@arm.com if (!success) { 23412109SRekai.GonzalezAlberquilla@arm.com // inhibited packets should never be forced to retry 23512109SRekai.GonzalezAlberquilla@arm.com assert(!pkt->memInhibitAsserted()); 23612109SRekai.GonzalezAlberquilla@arm.com 23712109SRekai.GonzalezAlberquilla@arm.com // if it was added as outstanding and the send failed, then 23812109SRekai.GonzalezAlberquilla@arm.com // erase it again 23912106SRekai.GonzalezAlberquilla@arm.com if (add_outstanding) 24012106SRekai.GonzalezAlberquilla@arm.com outstandingReq.erase(pkt->req); 24112106SRekai.GonzalezAlberquilla@arm.com 24212106SRekai.GonzalezAlberquilla@arm.com // undo the calculation so we can check for 0 again 24312106SRekai.GonzalezAlberquilla@arm.com pkt->firstWordDelay = pkt->lastWordDelay = 0; 24412106SRekai.GonzalezAlberquilla@arm.com 24512106SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 24612106SRekai.GonzalezAlberquilla@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 24712106SRekai.GonzalezAlberquilla@arm.com 24812106SRekai.GonzalezAlberquilla@arm.com // update the layer state and schedule an idle event 2491060SN/A reqLayers[master_port_id]->failedTiming(src_port, 25012106SRekai.GonzalezAlberquilla@arm.com clockEdge(headerCycles)); 25112106SRekai.GonzalezAlberquilla@arm.com } else { 25212106SRekai.GonzalezAlberquilla@arm.com // update the layer state and schedule an idle event 25312106SRekai.GonzalezAlberquilla@arm.com reqLayers[master_port_id]->succeededTiming(packetFinishTime); 2549919Ssteve.reinhardt@amd.com } 2551060SN/A } 2561060SN/A 2579919Ssteve.reinhardt@amd.com // stats updates only consider packets that were successfully sent 25812104Snathanael.premillieu@arm.com if (success) { 2599919Ssteve.reinhardt@amd.com pktCount[slave_port_id][master_port_id]++; 26012104Snathanael.premillieu@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 2619919Ssteve.reinhardt@amd.com transDist[pkt_cmd]++; 2629919Ssteve.reinhardt@amd.com } 26312106SRekai.GonzalezAlberquilla@arm.com 26412106SRekai.GonzalezAlberquilla@arm.com return success; 26512106SRekai.GonzalezAlberquilla@arm.com} 26612106SRekai.GonzalezAlberquilla@arm.com 26712106SRekai.GonzalezAlberquilla@arm.combool 2681060SN/ACoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 26912106SRekai.GonzalezAlberquilla@arm.com{ 27012106SRekai.GonzalezAlberquilla@arm.com // determine the source port based on the id 2711060SN/A MasterPort *src_port = masterPorts[master_port_id]; 27212109SRekai.GonzalezAlberquilla@arm.com 27312109SRekai.GonzalezAlberquilla@arm.com // determine the destination based on what is stored in the packet 27412109SRekai.GonzalezAlberquilla@arm.com PortID slave_port_id = pkt->getDest(); 27512109SRekai.GonzalezAlberquilla@arm.com 27612109SRekai.GonzalezAlberquilla@arm.com // test if the crossbar should be considered occupied for the 27712109SRekai.GonzalezAlberquilla@arm.com // current port 27812109SRekai.GonzalezAlberquilla@arm.com if (!respLayers[slave_port_id]->tryTiming(src_port)) { 27912109SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 28012106SRekai.GonzalezAlberquilla@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 28112106SRekai.GonzalezAlberquilla@arm.com return false; 2821060SN/A } 28312106SRekai.GonzalezAlberquilla@arm.com 28412106SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 28512106SRekai.GonzalezAlberquilla@arm.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 28612106SRekai.GonzalezAlberquilla@arm.com 2879920Syasuko.eckert@amd.com // store size and command as they might be modified when 28812106SRekai.GonzalezAlberquilla@arm.com // forwarding the packet 28912106SRekai.GonzalezAlberquilla@arm.com unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 29012106SRekai.GonzalezAlberquilla@arm.com unsigned int pkt_cmd = pkt->cmdToIndex(); 29112106SRekai.GonzalezAlberquilla@arm.com 2929919Ssteve.reinhardt@amd.com calcPacketTiming(pkt); 2931060SN/A Tick packetFinishTime = pkt->lastWordDelay + curTick(); 2949919Ssteve.reinhardt@amd.com 2959919Ssteve.reinhardt@amd.com // the packet is a normal response to a request that we should 2969919Ssteve.reinhardt@amd.com // have seen passing through the crossbar 29712104Snathanael.premillieu@arm.com assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 2989919Ssteve.reinhardt@amd.com 29912104Snathanael.premillieu@arm.com if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 3009919Ssteve.reinhardt@amd.com // let the snoop filter inspect the response and update its state 3019919Ssteve.reinhardt@amd.com snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 30212106SRekai.GonzalezAlberquilla@arm.com } 30312106SRekai.GonzalezAlberquilla@arm.com 30412106SRekai.GonzalezAlberquilla@arm.com // remove it as outstanding 30512106SRekai.GonzalezAlberquilla@arm.com outstandingReq.erase(pkt->req); 30612106SRekai.GonzalezAlberquilla@arm.com 30712106SRekai.GonzalezAlberquilla@arm.com // send the packet through the destination slave port 3081060SN/A bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 30912106SRekai.GonzalezAlberquilla@arm.com 31012106SRekai.GonzalezAlberquilla@arm.com // currently it is illegal to block responses... can lead to 31112106SRekai.GonzalezAlberquilla@arm.com // deadlock 3121060SN/A assert(success); 31312109SRekai.GonzalezAlberquilla@arm.com 31412109SRekai.GonzalezAlberquilla@arm.com respLayers[slave_port_id]->succeededTiming(packetFinishTime); 31512109SRekai.GonzalezAlberquilla@arm.com 31612109SRekai.GonzalezAlberquilla@arm.com // stats updates 31712109SRekai.GonzalezAlberquilla@arm.com pktCount[slave_port_id][master_port_id]++; 31812109SRekai.GonzalezAlberquilla@arm.com pktSize[slave_port_id][master_port_id] += pkt_size; 31912109SRekai.GonzalezAlberquilla@arm.com transDist[pkt_cmd]++; 32012109SRekai.GonzalezAlberquilla@arm.com 32112109SRekai.GonzalezAlberquilla@arm.com return true; 32212109SRekai.GonzalezAlberquilla@arm.com} 32312106SRekai.GonzalezAlberquilla@arm.com 32412106SRekai.GonzalezAlberquilla@arm.comvoid 32512106SRekai.GonzalezAlberquilla@arm.comCoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 3261060SN/A{ 32712106SRekai.GonzalezAlberquilla@arm.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 32812106SRekai.GonzalezAlberquilla@arm.com masterPorts[master_port_id]->name(), pkt->cmdString(), 32912106SRekai.GonzalezAlberquilla@arm.com pkt->getAddr()); 33012106SRekai.GonzalezAlberquilla@arm.com 33112106SRekai.GonzalezAlberquilla@arm.com // update stats here as we know the forwarding will succeed 33212106SRekai.GonzalezAlberquilla@arm.com transDist[pkt->cmdToIndex()]++; 33312106SRekai.GonzalezAlberquilla@arm.com snoops++; 33412106SRekai.GonzalezAlberquilla@arm.com 33512106SRekai.GonzalezAlberquilla@arm.com // we should only see express snoops from caches 33612106SRekai.GonzalezAlberquilla@arm.com assert(pkt->isExpressSnoop()); 33712106SRekai.GonzalezAlberquilla@arm.com 33812106SRekai.GonzalezAlberquilla@arm.com // set the source port for routing of the response 3399920Syasuko.eckert@amd.com pkt->setSrc(master_port_id); 3409920Syasuko.eckert@amd.com 3419920Syasuko.eckert@amd.com if (snoopFilter) { 3429919Ssteve.reinhardt@amd.com // let the Snoop Filter work its magic and guide probing 3439919Ssteve.reinhardt@amd.com auto sf_res = snoopFilter->lookupSnoop(pkt); 3449919Ssteve.reinhardt@amd.com // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 3459919Ssteve.reinhardt@amd.com DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 3461060SN/A " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 3479919Ssteve.reinhardt@amd.com pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 3481060SN/A sf_res.second); 34912109SRekai.GonzalezAlberquilla@arm.com 35012109SRekai.GonzalezAlberquilla@arm.com // forward to all snoopers 35112109SRekai.GonzalezAlberquilla@arm.com forwardTiming(pkt, InvalidPortID, sf_res.first); 35212109SRekai.GonzalezAlberquilla@arm.com } else { 3539919Ssteve.reinhardt@amd.com forwardTiming(pkt, InvalidPortID); 35410715SRekai.GonzalezAlberquilla@arm.com } 35512109SRekai.GonzalezAlberquilla@arm.com 35612109SRekai.GonzalezAlberquilla@arm.com // a snoop request came from a connected slave device (one of 35712109SRekai.GonzalezAlberquilla@arm.com // our master ports), and if it is not coming from the slave 35812109SRekai.GonzalezAlberquilla@arm.com // device responsible for the address range something is 35912109SRekai.GonzalezAlberquilla@arm.com // wrong, hence there is nothing further to do as the packet 36012109SRekai.GonzalezAlberquilla@arm.com // would be going back to where it came from 36112109SRekai.GonzalezAlberquilla@arm.com assert(master_port_id == findPort(pkt->getAddr())); 36212109SRekai.GonzalezAlberquilla@arm.com} 36312109SRekai.GonzalezAlberquilla@arm.com 36412109SRekai.GonzalezAlberquilla@arm.combool 36510715SRekai.GonzalezAlberquilla@arm.comCoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 36610715SRekai.GonzalezAlberquilla@arm.com{ 36710715SRekai.GonzalezAlberquilla@arm.com // determine the source port based on the id 36812109SRekai.GonzalezAlberquilla@arm.com SlavePort* src_port = slavePorts[slave_port_id]; 36912109SRekai.GonzalezAlberquilla@arm.com 37010715SRekai.GonzalezAlberquilla@arm.com // get the destination from the packet 37110715SRekai.GonzalezAlberquilla@arm.com PortID dest_port_id = pkt->getDest(); 37210715SRekai.GonzalezAlberquilla@arm.com 37312109SRekai.GonzalezAlberquilla@arm.com // determine if the response is from a snoop request we 37412109SRekai.GonzalezAlberquilla@arm.com // created as the result of a normal request (in which case it 37510935Snilay@cs.wisc.edu // should be in the outstandingReq), or if we merely forwarded 37610715SRekai.GonzalezAlberquilla@arm.com // someone else's snoop request 37712109SRekai.GonzalezAlberquilla@arm.com bool forwardAsSnoop = outstandingReq.find(pkt->req) == 37812109SRekai.GonzalezAlberquilla@arm.com outstandingReq.end(); 37912109SRekai.GonzalezAlberquilla@arm.com 38012109SRekai.GonzalezAlberquilla@arm.com // test if the crossbar should be considered occupied for the 38112109SRekai.GonzalezAlberquilla@arm.com // current port, note that the check is bypassed if the response 38210715SRekai.GonzalezAlberquilla@arm.com // is being passed on as a normal response since this is occupying 3831060SN/A // the response layer rather than the snoop response layer 3841060SN/A if (forwardAsSnoop) { 3852292SN/A if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 386 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 387 src_port->name(), pkt->cmdString(), pkt->getAddr()); 388 return false; 389 } 390 } else { 391 // get the master port that mirrors this slave port internally 392 MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 393 if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 394 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 395 snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 396 return false; 397 } 398 } 399 400 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 401 src_port->name(), pkt->cmdString(), pkt->getAddr()); 402 403 // store size and command as they might be modified when 404 // forwarding the packet 405 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 406 unsigned int pkt_cmd = pkt->cmdToIndex(); 407 408 // responses are never express snoops 409 assert(!pkt->isExpressSnoop()); 410 411 calcPacketTiming(pkt); 412 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 413 414 // forward it either as a snoop response or a normal response 415 if (forwardAsSnoop) { 416 // this is a snoop response to a snoop request we forwarded, 417 // e.g. coming from the L1 and going to the L2, and it should 418 // be forwarded as a snoop response 419 420 if (snoopFilter) { 421 // update the probe filter so that it can properly track the line 422 snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 423 *masterPorts[dest_port_id]); 424 } 425 426 bool success M5_VAR_USED = 427 masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 428 pktCount[slave_port_id][dest_port_id]++; 429 pktSize[slave_port_id][dest_port_id] += pkt_size; 430 assert(success); 431 432 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 433 } else { 434 // we got a snoop response on one of our slave ports, 435 // i.e. from a coherent master connected to the crossbar, and 436 // since we created the snoop request as part of recvTiming, 437 // this should now be a normal response again 438 outstandingReq.erase(pkt->req); 439 440 // this is a snoop response from a coherent master, with a 441 // destination field set on its way through the crossbar as 442 // request, hence it should never go back to where the snoop 443 // response came from, but instead to where the original 444 // request came from 445 assert(slave_port_id != dest_port_id); 446 447 if (snoopFilter) { 448 // update the probe filter so that it can properly track the line 449 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 450 *slavePorts[dest_port_id]); 451 } 452 453 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 454 " FWD RESP\n", src_port->name(), pkt->cmdString(), 455 pkt->getAddr()); 456 457 // as a normal response, it should go back to a master through 458 // one of our slave ports, at this point we are ignoring the 459 // fact that the response layer could be busy and do not touch 460 // its state 461 bool success M5_VAR_USED = 462 slavePorts[dest_port_id]->sendTimingResp(pkt); 463 464 // @todo Put the response in an internal FIFO and pass it on 465 // to the response layer from there 466 467 // currently it is illegal to block responses... can lead 468 // to deadlock 469 assert(success); 470 471 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 472 } 473 474 // stats updates 475 transDist[pkt_cmd]++; 476 snoops++; 477 478 return true; 479} 480 481 482void 483CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 484 const std::vector<SlavePort*>& dests) 485{ 486 DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 487 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 488 489 // snoops should only happen if the system isn't bypassing caches 490 assert(!system->bypassCaches()); 491 492 unsigned fanout = 0; 493 494 for (const auto& p: dests) { 495 // we could have gotten this request from a snooping master 496 // (corresponding to our own slave port that is also in 497 // snoopPorts) and should not send it back to where it came 498 // from 499 if (exclude_slave_port_id == InvalidPortID || 500 p->getId() != exclude_slave_port_id) { 501 // cache is not allowed to refuse snoop 502 p->sendTimingSnoopReq(pkt); 503 fanout++; 504 } 505 } 506 507 // Stats for fanout of this forward operation 508 snoopFanout.sample(fanout); 509} 510 511void 512CoherentXBar::recvRetry(PortID master_port_id) 513{ 514 // responses and snoop responses never block on forwarding them, 515 // so the retry will always be coming from a port to which we 516 // tried to forward a request 517 reqLayers[master_port_id]->recvRetry(); 518} 519 520Tick 521CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 522{ 523 DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 524 slavePorts[slave_port_id]->name(), pkt->getAddr(), 525 pkt->cmdString()); 526 527 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 528 unsigned int pkt_cmd = pkt->cmdToIndex(); 529 530 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 531 Tick snoop_response_latency = 0; 532 533 // uncacheable requests need never be snooped 534 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 535 // forward to all snoopers but the source 536 std::pair<MemCmd, Tick> snoop_result; 537 if (snoopFilter) { 538 // check with the snoop filter where to forward this packet 539 auto sf_res = 540 snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 541 snoop_response_latency += sf_res.second * clockPeriod(); 542 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 543 " SF size: %i lat: %i\n", __func__, 544 slavePorts[slave_port_id]->name(), pkt->cmdString(), 545 pkt->getAddr(), sf_res.first.size(), sf_res.second); 546 snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 547 sf_res.first); 548 } else { 549 snoop_result = forwardAtomic(pkt, slave_port_id); 550 } 551 snoop_response_cmd = snoop_result.first; 552 snoop_response_latency += snoop_result.second; 553 } 554 555 // even if we had a snoop response, we must continue and also 556 // perform the actual request at the destination 557 PortID master_port_id = findPort(pkt->getAddr()); 558 559 // stats updates for the request 560 pktCount[slave_port_id][master_port_id]++; 561 pktSize[slave_port_id][master_port_id] += pkt_size; 562 transDist[pkt_cmd]++; 563 564 // forward the request to the appropriate destination 565 Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 566 567 // Lower levels have replied, tell the snoop filter 568 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 569 pkt->isResponse()) { 570 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 571 } 572 573 // if we got a response from a snooper, restore it here 574 if (snoop_response_cmd != MemCmd::InvalidCmd) { 575 // no one else should have responded 576 assert(!pkt->isResponse()); 577 pkt->cmd = snoop_response_cmd; 578 response_latency = snoop_response_latency; 579 } 580 581 // add the response data 582 if (pkt->isResponse()) { 583 pkt_size = pkt->hasData() ? pkt->getSize() : 0; 584 pkt_cmd = pkt->cmdToIndex(); 585 586 // stats updates 587 pktCount[slave_port_id][master_port_id]++; 588 pktSize[slave_port_id][master_port_id] += pkt_size; 589 transDist[pkt_cmd]++; 590 } 591 592 // @todo: Not setting first-word time 593 pkt->lastWordDelay = response_latency; 594 return response_latency; 595} 596 597Tick 598CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 599{ 600 DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 601 masterPorts[master_port_id]->name(), pkt->getAddr(), 602 pkt->cmdString()); 603 604 // add the request snoop data 605 snoops++; 606 607 // forward to all snoopers 608 std::pair<MemCmd, Tick> snoop_result; 609 Tick snoop_response_latency = 0; 610 if (snoopFilter) { 611 auto sf_res = snoopFilter->lookupSnoop(pkt); 612 snoop_response_latency += sf_res.second * clockPeriod(); 613 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 614 __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 615 pkt->getAddr(), sf_res.first.size(), sf_res.second); 616 snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 617 sf_res.first); 618 } else { 619 snoop_result = forwardAtomic(pkt, InvalidPortID); 620 } 621 MemCmd snoop_response_cmd = snoop_result.first; 622 snoop_response_latency += snoop_result.second; 623 624 if (snoop_response_cmd != MemCmd::InvalidCmd) 625 pkt->cmd = snoop_response_cmd; 626 627 // add the response snoop data 628 if (pkt->isResponse()) { 629 snoops++; 630 } 631 632 // @todo: Not setting first-word time 633 pkt->lastWordDelay = snoop_response_latency; 634 return snoop_response_latency; 635} 636 637std::pair<MemCmd, Tick> 638CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 639 PortID source_master_port_id, 640 const std::vector<SlavePort*>& dests) 641{ 642 // the packet may be changed on snoops, record the original 643 // command to enable us to restore it between snoops so that 644 // additional snoops can take place properly 645 MemCmd orig_cmd = pkt->cmd; 646 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 647 Tick snoop_response_latency = 0; 648 649 // snoops should only happen if the system isn't bypassing caches 650 assert(!system->bypassCaches()); 651 652 unsigned fanout = 0; 653 654 for (const auto& p: dests) { 655 // we could have gotten this request from a snooping master 656 // (corresponding to our own slave port that is also in 657 // snoopPorts) and should not send it back to where it came 658 // from 659 if (exclude_slave_port_id != InvalidPortID && 660 p->getId() == exclude_slave_port_id) 661 continue; 662 663 Tick latency = p->sendAtomicSnoop(pkt); 664 fanout++; 665 666 // in contrast to a functional access, we have to keep on 667 // going as all snoopers must be updated even if we get a 668 // response 669 if (!pkt->isResponse()) 670 continue; 671 672 // response from snoop agent 673 assert(pkt->cmd != orig_cmd); 674 assert(pkt->memInhibitAsserted()); 675 // should only happen once 676 assert(snoop_response_cmd == MemCmd::InvalidCmd); 677 // save response state 678 snoop_response_cmd = pkt->cmd; 679 snoop_response_latency = latency; 680 681 if (snoopFilter) { 682 // Handle responses by the snoopers and differentiate between 683 // responses to requests from above and snoops from below 684 if (source_master_port_id != InvalidPortID) { 685 // Getting a response for a snoop from below 686 assert(exclude_slave_port_id == InvalidPortID); 687 snoopFilter->updateSnoopForward(pkt, *p, 688 *masterPorts[source_master_port_id]); 689 } else { 690 // Getting a response for a request from above 691 assert(source_master_port_id == InvalidPortID); 692 snoopFilter->updateSnoopResponse(pkt, *p, 693 *slavePorts[exclude_slave_port_id]); 694 } 695 } 696 // restore original packet state for remaining snoopers 697 pkt->cmd = orig_cmd; 698 } 699 700 // Stats for fanout 701 snoopFanout.sample(fanout); 702 703 // the packet is restored as part of the loop and any potential 704 // snoop response is part of the returned pair 705 return std::make_pair(snoop_response_cmd, snoop_response_latency); 706} 707 708void 709CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 710{ 711 if (!pkt->isPrint()) { 712 // don't do DPRINTFs on PrintReq as it clutters up the output 713 DPRINTF(CoherentXBar, 714 "recvFunctional: packet src %s addr 0x%x cmd %s\n", 715 slavePorts[slave_port_id]->name(), pkt->getAddr(), 716 pkt->cmdString()); 717 } 718 719 // uncacheable requests need never be snooped 720 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 721 // forward to all snoopers but the source 722 forwardFunctional(pkt, slave_port_id); 723 } 724 725 // there is no need to continue if the snooping has found what we 726 // were looking for and the packet is already a response 727 if (!pkt->isResponse()) { 728 PortID dest_id = findPort(pkt->getAddr()); 729 730 masterPorts[dest_id]->sendFunctional(pkt); 731 } 732} 733 734void 735CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 736{ 737 if (!pkt->isPrint()) { 738 // don't do DPRINTFs on PrintReq as it clutters up the output 739 DPRINTF(CoherentXBar, 740 "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 741 masterPorts[master_port_id]->name(), pkt->getAddr(), 742 pkt->cmdString()); 743 } 744 745 // forward to all snoopers 746 forwardFunctional(pkt, InvalidPortID); 747} 748 749void 750CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 751{ 752 // snoops should only happen if the system isn't bypassing caches 753 assert(!system->bypassCaches()); 754 755 for (const auto& p: snoopPorts) { 756 // we could have gotten this request from a snooping master 757 // (corresponding to our own slave port that is also in 758 // snoopPorts) and should not send it back to where it came 759 // from 760 if (exclude_slave_port_id == InvalidPortID || 761 p->getId() != exclude_slave_port_id) 762 p->sendFunctionalSnoop(pkt); 763 764 // if we get a response we are done 765 if (pkt->isResponse()) { 766 break; 767 } 768 } 769} 770 771unsigned int 772CoherentXBar::drain(DrainManager *dm) 773{ 774 // sum up the individual layers 775 unsigned int total = 0; 776 for (auto l: reqLayers) 777 total += l->drain(dm); 778 for (auto l: respLayers) 779 total += l->drain(dm); 780 for (auto l: snoopLayers) 781 total += l->drain(dm); 782 return total; 783} 784 785void 786CoherentXBar::regStats() 787{ 788 // register the stats of the base class and our layers 789 BaseXBar::regStats(); 790 for (auto l: reqLayers) 791 l->regStats(); 792 for (auto l: respLayers) 793 l->regStats(); 794 for (auto l: snoopLayers) 795 l->regStats(); 796 797 snoops 798 .name(name() + ".snoops") 799 .desc("Total snoops (count)") 800 ; 801 802 snoopFanout 803 .init(0, snoopPorts.size(), 1) 804 .name(name() + ".snoop_fanout") 805 .desc("Request fanout histogram") 806 ; 807} 808 809CoherentXBar * 810CoherentXBarParams::create() 811{ 812 return new CoherentXBar(this); 813} 814