coherent_xbar.cc revision 10572
12568SN/A/* 22568SN/A * Copyright (c) 2011-2014 ARM Limited 37636Ssteve.reinhardt@amd.com * All rights reserved 42568SN/A * 52568SN/A * The license below extends only to copyright in the software and shall 62568SN/A * not be construed as granting a license to any other intellectual 72568SN/A * property including but not limited to intellectual property relating 82568SN/A * to a hardware implementation of the functionality of the software 92568SN/A * licensed hereunder. You may use the software subject to the license 102568SN/A * terms below provided that you ensure that this notice is replicated 112568SN/A * unmodified and in its entirety in all distributions of the software, 122568SN/A * modified or unmodified, in source code or in binary form. 132568SN/A * 142568SN/A * Copyright (c) 2006 The Regents of The University of Michigan 152568SN/A * All rights reserved. 162568SN/A * 172568SN/A * Redistribution and use in source and binary forms, with or without 182568SN/A * modification, are permitted provided that the following conditions are 192568SN/A * met: redistributions of source code must retain the above copyright 202568SN/A * notice, this list of conditions and the following disclaimer; 212568SN/A * redistributions in binary form must reproduce the above copyright 222568SN/A * notice, this list of conditions and the following disclaimer in the 232568SN/A * documentation and/or other materials provided with the distribution; 242568SN/A * neither the name of the copyright holders nor the names of its 252568SN/A * contributors may be used to endorse or promote products derived from 262568SN/A * this software without specific prior written permission. 272568SN/A * 282665Ssaidi@eecs.umich.edu * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 292665Ssaidi@eecs.umich.edu * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 302665Ssaidi@eecs.umich.edu * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312568SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 322568SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 332568SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 342568SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 352568SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 362568SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 372568SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 383260Ssaidi@eecs.umich.edu * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 398229Snate@binkert.org * 403260Ssaidi@eecs.umich.edu * Authors: Ali Saidi 418229Snate@binkert.org * Andreas Hansson 425314Sstever@gmail.com * William Wang 432590SN/A */ 443348Sbinkertn@umich.edu 452568SN/A/** 462568SN/A * @file 475735Snate@binkert.org * Definition of a crossbar object. 485735Snate@binkert.org */ 494022Sstever@eecs.umich.edu 504022Sstever@eecs.umich.edu#include "base/misc.hh" 514022Sstever@eecs.umich.edu#include "base/trace.hh" 524022Sstever@eecs.umich.edu#include "debug/AddrRanges.hh" 534022Sstever@eecs.umich.edu#include "debug/CoherentXBar.hh" 544022Sstever@eecs.umich.edu#include "mem/coherent_xbar.hh" 554022Sstever@eecs.umich.edu#include "sim/system.hh" 562641Sstever@eecs.umich.edu 574022Sstever@eecs.umich.eduCoherentXBar::CoherentXBar(const CoherentXBarParams *p) 584022Sstever@eecs.umich.edu : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 592641Sstever@eecs.umich.edu{ 604022Sstever@eecs.umich.edu // create the ports based on the size of the master and slave 614022Sstever@eecs.umich.edu // vector ports, and the presence of the default port, the ports 624022Sstever@eecs.umich.edu // are enumerated starting from zero 634022Sstever@eecs.umich.edu for (int i = 0; i < p->port_master_connection_count; ++i) { 644473Sstever@eecs.umich.edu std::string portName = csprintf("%s.master[%d]", name(), i); 654473Sstever@eecs.umich.edu MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 665319Sstever@gmail.com masterPorts.push_back(bp); 675319Sstever@gmail.com reqLayers.push_back(new ReqLayer(*bp, *this, 685319Sstever@gmail.com csprintf(".reqLayer%d", i))); 694022Sstever@eecs.umich.edu snoopLayers.push_back(new SnoopLayer(*bp, *this, 704626Sstever@eecs.umich.edu csprintf(".snoopLayer%d", i))); 714022Sstever@eecs.umich.edu } 724022Sstever@eecs.umich.edu 734626Sstever@eecs.umich.edu // see if we have a default slave device connected and if so add 744022Sstever@eecs.umich.edu // our corresponding master port 754628Sstever@eecs.umich.edu if (p->port_default_connection_count) { 764628Sstever@eecs.umich.edu defaultPortID = masterPorts.size(); 774022Sstever@eecs.umich.edu std::string portName = name() + ".default"; 784022Sstever@eecs.umich.edu MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 794022Sstever@eecs.umich.edu defaultPortID); 804022Sstever@eecs.umich.edu masterPorts.push_back(bp); 814022Sstever@eecs.umich.edu reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 824022Sstever@eecs.umich.edu defaultPortID))); 834022Sstever@eecs.umich.edu snoopLayers.push_back(new SnoopLayer(*bp, *this, 844022Sstever@eecs.umich.edu csprintf(".snoopLayer%d", 854022Sstever@eecs.umich.edu defaultPortID))); 864022Sstever@eecs.umich.edu } 874022Sstever@eecs.umich.edu 884022Sstever@eecs.umich.edu // create the slave ports, once again starting at zero 894022Sstever@eecs.umich.edu for (int i = 0; i < p->port_slave_connection_count; ++i) { 904626Sstever@eecs.umich.edu std::string portName = csprintf("%s.slave[%d]", name(), i); 914626Sstever@eecs.umich.edu SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 924022Sstever@eecs.umich.edu slavePorts.push_back(bp); 934022Sstever@eecs.umich.edu respLayers.push_back(new RespLayer(*bp, *this, 945319Sstever@gmail.com csprintf(".respLayer%d", i))); 954022Sstever@eecs.umich.edu snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 964022Sstever@eecs.umich.edu } 977465Ssteve.reinhardt@amd.com 984628Sstever@eecs.umich.edu if (snoopFilter) 997465Ssteve.reinhardt@amd.com snoopFilter->setSlavePorts(slavePorts); 1007465Ssteve.reinhardt@amd.com 1017465Ssteve.reinhardt@amd.com clearPortCache(); 1027465Ssteve.reinhardt@amd.com} 1034628Sstever@eecs.umich.edu 1047465Ssteve.reinhardt@amd.comCoherentXBar::~CoherentXBar() 1057465Ssteve.reinhardt@amd.com{ 1067465Ssteve.reinhardt@amd.com for (auto l: reqLayers) 1077465Ssteve.reinhardt@amd.com delete l; 1087465Ssteve.reinhardt@amd.com for (auto l: respLayers) 1097465Ssteve.reinhardt@amd.com delete l; 1107465Ssteve.reinhardt@amd.com for (auto l: snoopLayers) 1115319Sstever@gmail.com delete l; 1127465Ssteve.reinhardt@amd.com for (auto p: snoopRespPorts) 1134022Sstever@eecs.umich.edu delete p; 1144626Sstever@eecs.umich.edu} 1154022Sstever@eecs.umich.edu 1164022Sstever@eecs.umich.eduvoid 1175319Sstever@gmail.comCoherentXBar::init() 1184040Ssaidi@eecs.umich.edu{ 1195507Sstever@gmail.com // the base class is responsible for determining the block size 1205507Sstever@gmail.com BaseXBar::init(); 1216076Sgblack@eecs.umich.edu 1225507Sstever@gmail.com // iterate over our slave ports and determine which of our 1234626Sstever@eecs.umich.edu // neighbouring master ports are snooping and add them as snoopers 1246076Sgblack@eecs.umich.edu for (const auto& p: slavePorts) { 1254626Sstever@eecs.umich.edu // check if the connected master port is snooping 1264626Sstever@eecs.umich.edu if (p->isSnooping()) { 1277669Ssteve.reinhardt@amd.com DPRINTF(AddrRanges, "Adding snooping master %s\n", 1287669Ssteve.reinhardt@amd.com p->getMasterPort().name()); 1297669Ssteve.reinhardt@amd.com snoopPorts.push_back(p); 1307669Ssteve.reinhardt@amd.com } 1314626Sstever@eecs.umich.edu } 1326076Sgblack@eecs.umich.edu 1334626Sstever@eecs.umich.edu if (snoopPorts.empty()) 1344040Ssaidi@eecs.umich.edu warn("CoherentXBar %s has no snooping ports attached!\n", name()); 1354626Sstever@eecs.umich.edu} 1364040Ssaidi@eecs.umich.edu 1374040Ssaidi@eecs.umich.edubool 1384626Sstever@eecs.umich.eduCoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 1394870Sstever@eecs.umich.edu{ 1405650Sgblack@eecs.umich.edu // determine the source port based on the id 1415650Sgblack@eecs.umich.edu SlavePort *src_port = slavePorts[slave_port_id]; 1426063Sgblack@eecs.umich.edu 1435650Sgblack@eecs.umich.edu // remember if the packet is an express snoop 1446063Sgblack@eecs.umich.edu bool is_express_snoop = pkt->isExpressSnoop(); 1454870Sstever@eecs.umich.edu 1464986Ssaidi@eecs.umich.edu // determine the destination based on the address 1474870Sstever@eecs.umich.edu PortID master_port_id = findPort(pkt->getAddr()); 1484986Ssaidi@eecs.umich.edu 1494870Sstever@eecs.umich.edu // test if the crossbar should be considered occupied for the current 1505314Sstever@gmail.com // port, and exclude express snoops from the check 1518436SBrad.Beckmann@amd.com if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 1528436SBrad.Beckmann@amd.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 1538436SBrad.Beckmann@amd.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 1548436SBrad.Beckmann@amd.com return false; 1555314Sstever@gmail.com } 1568184Ssomayeh@cs.wisc.edu 1578184Ssomayeh@cs.wisc.edu DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 1588436SBrad.Beckmann@amd.com src_port->name(), pkt->cmdString(), is_express_snoop, 1594022Sstever@eecs.umich.edu pkt->getAddr()); 1602592SN/A 1613607Srdreslin@umich.edu // store size and command as they might be modified when 1625314Sstever@gmail.com // forwarding the packet 1632641Sstever@eecs.umich.edu unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 1644626Sstever@eecs.umich.edu unsigned int pkt_cmd = pkt->cmdToIndex(); 1654626Sstever@eecs.umich.edu 1664626Sstever@eecs.umich.edu // set the source port for routing of the response 1674626Sstever@eecs.umich.edu pkt->setSrc(slave_port_id); 1683260Ssaidi@eecs.umich.edu 1694626Sstever@eecs.umich.edu calcPacketTiming(pkt); 1704626Sstever@eecs.umich.edu Tick packetFinishTime = pkt->lastWordDelay + curTick(); 1714626Sstever@eecs.umich.edu 1724626Sstever@eecs.umich.edu // uncacheable requests need never be snooped 1733260Ssaidi@eecs.umich.edu if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 1745314Sstever@gmail.com // the packet is a memory-mapped request and should be 1755314Sstever@gmail.com // broadcasted to our snoopers but the source 1765314Sstever@gmail.com if (snoopFilter) { 1775314Sstever@gmail.com // check with the snoop filter where to forward this packet 1785314Sstever@gmail.com auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 1795314Sstever@gmail.com packetFinishTime += sf_res.second * clockPeriod(); 1805314Sstever@gmail.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 1815314Sstever@gmail.com " SF size: %i lat: %i\n", src_port->name(), 1825314Sstever@gmail.com pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 1835314Sstever@gmail.com sf_res.second); 1845314Sstever@gmail.com forwardTiming(pkt, slave_port_id, sf_res.first); 1854626Sstever@eecs.umich.edu } else { 1864626Sstever@eecs.umich.edu forwardTiming(pkt, slave_port_id); 1874626Sstever@eecs.umich.edu } 1883260Ssaidi@eecs.umich.edu } 1894626Sstever@eecs.umich.edu 1904626Sstever@eecs.umich.edu // remember if we add an outstanding req so we can undo it if 1914626Sstever@eecs.umich.edu // necessary, if the packet needs a response, we should add it 1925735Snate@binkert.org // as outstanding and express snoops never fail so there is 1934626Sstever@eecs.umich.edu // not need to worry about them 1943260Ssaidi@eecs.umich.edu bool add_outstanding = !is_express_snoop && pkt->needsResponse(); 1954489Sstever@eecs.umich.edu 1964489Sstever@eecs.umich.edu // keep track that we have an outstanding request packet 1974489Sstever@eecs.umich.edu // matching this request, this is used by the coherency 1984489Sstever@eecs.umich.edu // mechanism in determining what to do with snoop responses 1994489Sstever@eecs.umich.edu // (in recvTimingSnoop) 2004489Sstever@eecs.umich.edu if (add_outstanding) { 2014626Sstever@eecs.umich.edu // we should never have an exsiting request outstanding 2024626Sstever@eecs.umich.edu assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 2033260Ssaidi@eecs.umich.edu outstandingReq.insert(pkt->req); 2044626Sstever@eecs.umich.edu } 2054626Sstever@eecs.umich.edu 2065735Snate@binkert.org // Note: Cannot create a copy of the full packet, here. 2075735Snate@binkert.org MemCmd orig_cmd(pkt->cmd); 2085735Snate@binkert.org 2095735Snate@binkert.org // since it is a normal request, attempt to send the packet 2105735Snate@binkert.org bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 2115735Snate@binkert.org 2123260Ssaidi@eecs.umich.edu if (snoopFilter && !pkt->req->isUncacheable() 2135314Sstever@gmail.com && !system->bypassCaches()) { 2144626Sstever@eecs.umich.edu // The packet may already be overwritten by the sendTimingReq function. 2155314Sstever@gmail.com // The snoop filter needs to see the original request *and* the return 2165314Sstever@gmail.com // status of the send operation, so we need to recreate the original 2175314Sstever@gmail.com // request. Atomic mode does not have the issue, as there the send 2185314Sstever@gmail.com // operation and the response happen instantaneously and don't need two 2192641Sstever@eecs.umich.edu // phase tracking. 2203260Ssaidi@eecs.umich.edu MemCmd tmp_cmd(pkt->cmd); 2215314Sstever@gmail.com pkt->cmd = orig_cmd; 2225735Snate@binkert.org // Let the snoop filter know about the success of the send operation 2233260Ssaidi@eecs.umich.edu snoopFilter->updateRequest(pkt, *src_port, !success); 2245314Sstever@gmail.com pkt->cmd = tmp_cmd; 2255314Sstever@gmail.com } 2263260Ssaidi@eecs.umich.edu 2273260Ssaidi@eecs.umich.edu // if this is an express snoop, we are done at this point 2285735Snate@binkert.org if (is_express_snoop) { 2295735Snate@binkert.org assert(success); 2305314Sstever@gmail.com snoops++; 2315314Sstever@gmail.com } else { 2325314Sstever@gmail.com // for normal requests, check if successful 2335314Sstever@gmail.com if (!success) { 2345314Sstever@gmail.com // inhibited packets should never be forced to retry 2355314Sstever@gmail.com assert(!pkt->memInhibitAsserted()); 2365314Sstever@gmail.com 2375314Sstever@gmail.com // if it was added as outstanding and the send failed, then 2385314Sstever@gmail.com // erase it again 2395314Sstever@gmail.com if (add_outstanding) 2405314Sstever@gmail.com outstandingReq.erase(pkt->req); 2415314Sstever@gmail.com 2425735Snate@binkert.org // undo the calculation so we can check for 0 again 2435314Sstever@gmail.com pkt->firstWordDelay = pkt->lastWordDelay = 0; 2445314Sstever@gmail.com 2455314Sstever@gmail.com DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 2465314Sstever@gmail.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 2475314Sstever@gmail.com 2485735Snate@binkert.org // update the layer state and schedule an idle event 2495314Sstever@gmail.com reqLayers[master_port_id]->failedTiming(src_port, 2505314Sstever@gmail.com clockEdge(headerCycles)); 2515735Snate@binkert.org } else { 2525314Sstever@gmail.com // update the layer state and schedule an idle event 2535314Sstever@gmail.com reqLayers[master_port_id]->succeededTiming(packetFinishTime); 2545314Sstever@gmail.com } 2555314Sstever@gmail.com } 2565314Sstever@gmail.com 2575314Sstever@gmail.com // stats updates only consider packets that were successfully sent 2585314Sstever@gmail.com if (success) { 2595314Sstever@gmail.com pktCount[slave_port_id][master_port_id]++; 2605314Sstever@gmail.com pktSize[slave_port_id][master_port_id] += pkt_size; 2615314Sstever@gmail.com transDist[pkt_cmd]++; 2625314Sstever@gmail.com } 2635314Sstever@gmail.com 2645314Sstever@gmail.com return success; 2655314Sstever@gmail.com} 2665314Sstever@gmail.com 2675314Sstever@gmail.combool 2685314Sstever@gmail.comCoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 2695314Sstever@gmail.com{ 2705314Sstever@gmail.com // determine the source port based on the id 2715314Sstever@gmail.com MasterPort *src_port = masterPorts[master_port_id]; 2725314Sstever@gmail.com 2735314Sstever@gmail.com // determine the destination based on what is stored in the packet 2745314Sstever@gmail.com PortID slave_port_id = pkt->getDest(); 2755314Sstever@gmail.com assert(slave_port_id != InvalidPortID); 2765314Sstever@gmail.com assert(slave_port_id < respLayers.size()); 2775314Sstever@gmail.com 2785314Sstever@gmail.com // test if the crossbar should be considered occupied for the 2795314Sstever@gmail.com // current port 2805314Sstever@gmail.com if (!respLayers[slave_port_id]->tryTiming(src_port)) { 2815314Sstever@gmail.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 2825314Sstever@gmail.com src_port->name(), pkt->cmdString(), pkt->getAddr()); 2835314Sstever@gmail.com return false; 2845314Sstever@gmail.com } 2855314Sstever@gmail.com 2865314Sstever@gmail.com DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 287 src_port->name(), pkt->cmdString(), pkt->getAddr()); 288 289 // store size and command as they might be modified when 290 // forwarding the packet 291 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 292 unsigned int pkt_cmd = pkt->cmdToIndex(); 293 294 calcPacketTiming(pkt); 295 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 296 297 // the packet is a normal response to a request that we should 298 // have seen passing through the crossbar 299 assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 300 301 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 302 // let the snoop filter inspect the response and update its state 303 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 304 } 305 306 // remove it as outstanding 307 outstandingReq.erase(pkt->req); 308 309 // send the packet through the destination slave port 310 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 311 312 // currently it is illegal to block responses... can lead to 313 // deadlock 314 assert(success); 315 316 respLayers[slave_port_id]->succeededTiming(packetFinishTime); 317 318 // stats updates 319 pktCount[slave_port_id][master_port_id]++; 320 pktSize[slave_port_id][master_port_id] += pkt_size; 321 transDist[pkt_cmd]++; 322 323 return true; 324} 325 326void 327CoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 328{ 329 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 330 masterPorts[master_port_id]->name(), pkt->cmdString(), 331 pkt->getAddr()); 332 333 // update stats here as we know the forwarding will succeed 334 transDist[pkt->cmdToIndex()]++; 335 snoops++; 336 337 // we should only see express snoops from caches 338 assert(pkt->isExpressSnoop()); 339 340 // set the source port for routing of the response 341 pkt->setSrc(master_port_id); 342 343 if (snoopFilter) { 344 // let the Snoop Filter work its magic and guide probing 345 auto sf_res = snoopFilter->lookupSnoop(pkt); 346 // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 347 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 348 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 349 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 350 sf_res.second); 351 352 // forward to all snoopers 353 forwardTiming(pkt, InvalidPortID, sf_res.first); 354 } else { 355 forwardTiming(pkt, InvalidPortID); 356 } 357 358 // a snoop request came from a connected slave device (one of 359 // our master ports), and if it is not coming from the slave 360 // device responsible for the address range something is 361 // wrong, hence there is nothing further to do as the packet 362 // would be going back to where it came from 363 assert(master_port_id == findPort(pkt->getAddr())); 364} 365 366bool 367CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 368{ 369 // determine the source port based on the id 370 SlavePort* src_port = slavePorts[slave_port_id]; 371 372 // get the destination from the packet 373 PortID dest_port_id = pkt->getDest(); 374 assert(dest_port_id != InvalidPortID); 375 376 // determine if the response is from a snoop request we 377 // created as the result of a normal request (in which case it 378 // should be in the outstandingReq), or if we merely forwarded 379 // someone else's snoop request 380 bool forwardAsSnoop = outstandingReq.find(pkt->req) == 381 outstandingReq.end(); 382 383 // test if the crossbar should be considered occupied for the 384 // current port, note that the check is bypassed if the response 385 // is being passed on as a normal response since this is occupying 386 // the response layer rather than the snoop response layer 387 if (forwardAsSnoop) { 388 assert(dest_port_id < snoopLayers.size()); 389 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 390 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 391 src_port->name(), pkt->cmdString(), pkt->getAddr()); 392 return false; 393 } 394 } else { 395 // get the master port that mirrors this slave port internally 396 MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 397 assert(dest_port_id < respLayers.size()); 398 if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 399 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 400 snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 401 return false; 402 } 403 } 404 405 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 406 src_port->name(), pkt->cmdString(), pkt->getAddr()); 407 408 // store size and command as they might be modified when 409 // forwarding the packet 410 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 411 unsigned int pkt_cmd = pkt->cmdToIndex(); 412 413 // responses are never express snoops 414 assert(!pkt->isExpressSnoop()); 415 416 calcPacketTiming(pkt); 417 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 418 419 // forward it either as a snoop response or a normal response 420 if (forwardAsSnoop) { 421 // this is a snoop response to a snoop request we forwarded, 422 // e.g. coming from the L1 and going to the L2, and it should 423 // be forwarded as a snoop response 424 425 if (snoopFilter) { 426 // update the probe filter so that it can properly track the line 427 snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 428 *masterPorts[dest_port_id]); 429 } 430 431 bool success M5_VAR_USED = 432 masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 433 pktCount[slave_port_id][dest_port_id]++; 434 pktSize[slave_port_id][dest_port_id] += pkt_size; 435 assert(success); 436 437 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 438 } else { 439 // we got a snoop response on one of our slave ports, 440 // i.e. from a coherent master connected to the crossbar, and 441 // since we created the snoop request as part of recvTiming, 442 // this should now be a normal response again 443 outstandingReq.erase(pkt->req); 444 445 // this is a snoop response from a coherent master, with a 446 // destination field set on its way through the crossbar as 447 // request, hence it should never go back to where the snoop 448 // response came from, but instead to where the original 449 // request came from 450 assert(slave_port_id != dest_port_id); 451 452 if (snoopFilter) { 453 // update the probe filter so that it can properly track the line 454 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 455 *slavePorts[dest_port_id]); 456 } 457 458 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 459 " FWD RESP\n", src_port->name(), pkt->cmdString(), 460 pkt->getAddr()); 461 462 // as a normal response, it should go back to a master through 463 // one of our slave ports, at this point we are ignoring the 464 // fact that the response layer could be busy and do not touch 465 // its state 466 bool success M5_VAR_USED = 467 slavePorts[dest_port_id]->sendTimingResp(pkt); 468 469 // @todo Put the response in an internal FIFO and pass it on 470 // to the response layer from there 471 472 // currently it is illegal to block responses... can lead 473 // to deadlock 474 assert(success); 475 476 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 477 } 478 479 // stats updates 480 transDist[pkt_cmd]++; 481 snoops++; 482 483 return true; 484} 485 486 487void 488CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 489 const std::vector<SlavePort*>& dests) 490{ 491 DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 492 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 493 494 // snoops should only happen if the system isn't bypassing caches 495 assert(!system->bypassCaches()); 496 497 unsigned fanout = 0; 498 499 for (const auto& p: dests) { 500 // we could have gotten this request from a snooping master 501 // (corresponding to our own slave port that is also in 502 // snoopPorts) and should not send it back to where it came 503 // from 504 if (exclude_slave_port_id == InvalidPortID || 505 p->getId() != exclude_slave_port_id) { 506 // cache is not allowed to refuse snoop 507 p->sendTimingSnoopReq(pkt); 508 fanout++; 509 } 510 } 511 512 // Stats for fanout of this forward operation 513 snoopFanout.sample(fanout); 514} 515 516void 517CoherentXBar::recvRetry(PortID master_port_id) 518{ 519 // responses and snoop responses never block on forwarding them, 520 // so the retry will always be coming from a port to which we 521 // tried to forward a request 522 reqLayers[master_port_id]->recvRetry(); 523} 524 525Tick 526CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 527{ 528 DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 529 slavePorts[slave_port_id]->name(), pkt->getAddr(), 530 pkt->cmdString()); 531 532 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 533 unsigned int pkt_cmd = pkt->cmdToIndex(); 534 535 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 536 Tick snoop_response_latency = 0; 537 538 // uncacheable requests need never be snooped 539 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 540 // forward to all snoopers but the source 541 std::pair<MemCmd, Tick> snoop_result; 542 if (snoopFilter) { 543 // check with the snoop filter where to forward this packet 544 auto sf_res = 545 snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 546 snoop_response_latency += sf_res.second * clockPeriod(); 547 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 548 " SF size: %i lat: %i\n", __func__, 549 slavePorts[slave_port_id]->name(), pkt->cmdString(), 550 pkt->getAddr(), sf_res.first.size(), sf_res.second); 551 snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 552 sf_res.first); 553 } else { 554 snoop_result = forwardAtomic(pkt, slave_port_id); 555 } 556 snoop_response_cmd = snoop_result.first; 557 snoop_response_latency += snoop_result.second; 558 } 559 560 // even if we had a snoop response, we must continue and also 561 // perform the actual request at the destination 562 PortID master_port_id = findPort(pkt->getAddr()); 563 564 // stats updates for the request 565 pktCount[slave_port_id][master_port_id]++; 566 pktSize[slave_port_id][master_port_id] += pkt_size; 567 transDist[pkt_cmd]++; 568 569 // forward the request to the appropriate destination 570 Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 571 572 // Lower levels have replied, tell the snoop filter 573 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 574 pkt->isResponse()) { 575 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 576 } 577 578 // if we got a response from a snooper, restore it here 579 if (snoop_response_cmd != MemCmd::InvalidCmd) { 580 // no one else should have responded 581 assert(!pkt->isResponse()); 582 pkt->cmd = snoop_response_cmd; 583 response_latency = snoop_response_latency; 584 } 585 586 // add the response data 587 if (pkt->isResponse()) { 588 pkt_size = pkt->hasData() ? pkt->getSize() : 0; 589 pkt_cmd = pkt->cmdToIndex(); 590 591 // stats updates 592 pktCount[slave_port_id][master_port_id]++; 593 pktSize[slave_port_id][master_port_id] += pkt_size; 594 transDist[pkt_cmd]++; 595 } 596 597 // @todo: Not setting first-word time 598 pkt->lastWordDelay = response_latency; 599 return response_latency; 600} 601 602Tick 603CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 604{ 605 DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 606 masterPorts[master_port_id]->name(), pkt->getAddr(), 607 pkt->cmdString()); 608 609 // add the request snoop data 610 snoops++; 611 612 // forward to all snoopers 613 std::pair<MemCmd, Tick> snoop_result; 614 Tick snoop_response_latency = 0; 615 if (snoopFilter) { 616 auto sf_res = snoopFilter->lookupSnoop(pkt); 617 snoop_response_latency += sf_res.second * clockPeriod(); 618 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 619 __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 620 pkt->getAddr(), sf_res.first.size(), sf_res.second); 621 snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 622 sf_res.first); 623 } else { 624 snoop_result = forwardAtomic(pkt, InvalidPortID); 625 } 626 MemCmd snoop_response_cmd = snoop_result.first; 627 snoop_response_latency += snoop_result.second; 628 629 if (snoop_response_cmd != MemCmd::InvalidCmd) 630 pkt->cmd = snoop_response_cmd; 631 632 // add the response snoop data 633 if (pkt->isResponse()) { 634 snoops++; 635 } 636 637 // @todo: Not setting first-word time 638 pkt->lastWordDelay = snoop_response_latency; 639 return snoop_response_latency; 640} 641 642std::pair<MemCmd, Tick> 643CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 644 PortID source_master_port_id, 645 const std::vector<SlavePort*>& dests) 646{ 647 // the packet may be changed on snoops, record the original 648 // command to enable us to restore it between snoops so that 649 // additional snoops can take place properly 650 MemCmd orig_cmd = pkt->cmd; 651 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 652 Tick snoop_response_latency = 0; 653 654 // snoops should only happen if the system isn't bypassing caches 655 assert(!system->bypassCaches()); 656 657 unsigned fanout = 0; 658 659 for (const auto& p: dests) { 660 // we could have gotten this request from a snooping master 661 // (corresponding to our own slave port that is also in 662 // snoopPorts) and should not send it back to where it came 663 // from 664 if (exclude_slave_port_id != InvalidPortID && 665 p->getId() == exclude_slave_port_id) 666 continue; 667 668 Tick latency = p->sendAtomicSnoop(pkt); 669 fanout++; 670 671 // in contrast to a functional access, we have to keep on 672 // going as all snoopers must be updated even if we get a 673 // response 674 if (!pkt->isResponse()) 675 continue; 676 677 // response from snoop agent 678 assert(pkt->cmd != orig_cmd); 679 assert(pkt->memInhibitAsserted()); 680 // should only happen once 681 assert(snoop_response_cmd == MemCmd::InvalidCmd); 682 // save response state 683 snoop_response_cmd = pkt->cmd; 684 snoop_response_latency = latency; 685 686 if (snoopFilter) { 687 // Handle responses by the snoopers and differentiate between 688 // responses to requests from above and snoops from below 689 if (source_master_port_id != InvalidPortID) { 690 // Getting a response for a snoop from below 691 assert(exclude_slave_port_id == InvalidPortID); 692 snoopFilter->updateSnoopForward(pkt, *p, 693 *masterPorts[source_master_port_id]); 694 } else { 695 // Getting a response for a request from above 696 assert(source_master_port_id == InvalidPortID); 697 snoopFilter->updateSnoopResponse(pkt, *p, 698 *slavePorts[exclude_slave_port_id]); 699 } 700 } 701 // restore original packet state for remaining snoopers 702 pkt->cmd = orig_cmd; 703 } 704 705 // Stats for fanout 706 snoopFanout.sample(fanout); 707 708 // the packet is restored as part of the loop and any potential 709 // snoop response is part of the returned pair 710 return std::make_pair(snoop_response_cmd, snoop_response_latency); 711} 712 713void 714CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 715{ 716 if (!pkt->isPrint()) { 717 // don't do DPRINTFs on PrintReq as it clutters up the output 718 DPRINTF(CoherentXBar, 719 "recvFunctional: packet src %s addr 0x%x cmd %s\n", 720 slavePorts[slave_port_id]->name(), pkt->getAddr(), 721 pkt->cmdString()); 722 } 723 724 // uncacheable requests need never be snooped 725 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 726 // forward to all snoopers but the source 727 forwardFunctional(pkt, slave_port_id); 728 } 729 730 // there is no need to continue if the snooping has found what we 731 // were looking for and the packet is already a response 732 if (!pkt->isResponse()) { 733 PortID dest_id = findPort(pkt->getAddr()); 734 735 masterPorts[dest_id]->sendFunctional(pkt); 736 } 737} 738 739void 740CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 741{ 742 if (!pkt->isPrint()) { 743 // don't do DPRINTFs on PrintReq as it clutters up the output 744 DPRINTF(CoherentXBar, 745 "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 746 masterPorts[master_port_id]->name(), pkt->getAddr(), 747 pkt->cmdString()); 748 } 749 750 // forward to all snoopers 751 forwardFunctional(pkt, InvalidPortID); 752} 753 754void 755CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 756{ 757 // snoops should only happen if the system isn't bypassing caches 758 assert(!system->bypassCaches()); 759 760 for (const auto& p: snoopPorts) { 761 // we could have gotten this request from a snooping master 762 // (corresponding to our own slave port that is also in 763 // snoopPorts) and should not send it back to where it came 764 // from 765 if (exclude_slave_port_id == InvalidPortID || 766 p->getId() != exclude_slave_port_id) 767 p->sendFunctionalSnoop(pkt); 768 769 // if we get a response we are done 770 if (pkt->isResponse()) { 771 break; 772 } 773 } 774} 775 776unsigned int 777CoherentXBar::drain(DrainManager *dm) 778{ 779 // sum up the individual layers 780 unsigned int total = 0; 781 for (auto l: reqLayers) 782 total += l->drain(dm); 783 for (auto l: respLayers) 784 total += l->drain(dm); 785 for (auto l: snoopLayers) 786 total += l->drain(dm); 787 return total; 788} 789 790void 791CoherentXBar::regStats() 792{ 793 // register the stats of the base class and our layers 794 BaseXBar::regStats(); 795 for (auto l: reqLayers) 796 l->regStats(); 797 for (auto l: respLayers) 798 l->regStats(); 799 for (auto l: snoopLayers) 800 l->regStats(); 801 802 snoops 803 .name(name() + ".snoops") 804 .desc("Total snoops (count)") 805 ; 806 807 snoopFanout 808 .init(0, snoopPorts.size(), 1) 809 .name(name() + ".snoop_fanout") 810 .desc("Request fanout histogram") 811 ; 812} 813 814CoherentXBar * 815CoherentXBarParams::create() 816{ 817 return new CoherentXBar(this); 818} 819