coherent_xbar.cc revision 10572:fc4c90a7d2f5
1/* 2 * Copyright (c) 2011-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2006 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Ali Saidi 41 * Andreas Hansson 42 * William Wang 43 */ 44 45/** 46 * @file 47 * Definition of a crossbar object. 48 */ 49 50#include "base/misc.hh" 51#include "base/trace.hh" 52#include "debug/AddrRanges.hh" 53#include "debug/CoherentXBar.hh" 54#include "mem/coherent_xbar.hh" 55#include "sim/system.hh" 56 57CoherentXBar::CoherentXBar(const CoherentXBarParams *p) 58 : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 59{ 60 // create the ports based on the size of the master and slave 61 // vector ports, and the presence of the default port, the ports 62 // are enumerated starting from zero 63 for (int i = 0; i < p->port_master_connection_count; ++i) { 64 std::string portName = csprintf("%s.master[%d]", name(), i); 65 MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 66 masterPorts.push_back(bp); 67 reqLayers.push_back(new ReqLayer(*bp, *this, 68 csprintf(".reqLayer%d", i))); 69 snoopLayers.push_back(new SnoopLayer(*bp, *this, 70 csprintf(".snoopLayer%d", i))); 71 } 72 73 // see if we have a default slave device connected and if so add 74 // our corresponding master port 75 if (p->port_default_connection_count) { 76 defaultPortID = masterPorts.size(); 77 std::string portName = name() + ".default"; 78 MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 79 defaultPortID); 80 masterPorts.push_back(bp); 81 reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 82 defaultPortID))); 83 snoopLayers.push_back(new SnoopLayer(*bp, *this, 84 csprintf(".snoopLayer%d", 85 defaultPortID))); 86 } 87 88 // create the slave ports, once again starting at zero 89 for (int i = 0; i < p->port_slave_connection_count; ++i) { 90 std::string portName = csprintf("%s.slave[%d]", name(), i); 91 SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 92 slavePorts.push_back(bp); 93 respLayers.push_back(new RespLayer(*bp, *this, 94 csprintf(".respLayer%d", i))); 95 snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 96 } 97 98 if (snoopFilter) 99 snoopFilter->setSlavePorts(slavePorts); 100 101 clearPortCache(); 102} 103 104CoherentXBar::~CoherentXBar() 105{ 106 for (auto l: reqLayers) 107 delete l; 108 for (auto l: respLayers) 109 delete l; 110 for (auto l: snoopLayers) 111 delete l; 112 for (auto p: snoopRespPorts) 113 delete p; 114} 115 116void 117CoherentXBar::init() 118{ 119 // the base class is responsible for determining the block size 120 BaseXBar::init(); 121 122 // iterate over our slave ports and determine which of our 123 // neighbouring master ports are snooping and add them as snoopers 124 for (const auto& p: slavePorts) { 125 // check if the connected master port is snooping 126 if (p->isSnooping()) { 127 DPRINTF(AddrRanges, "Adding snooping master %s\n", 128 p->getMasterPort().name()); 129 snoopPorts.push_back(p); 130 } 131 } 132 133 if (snoopPorts.empty()) 134 warn("CoherentXBar %s has no snooping ports attached!\n", name()); 135} 136 137bool 138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 139{ 140 // determine the source port based on the id 141 SlavePort *src_port = slavePorts[slave_port_id]; 142 143 // remember if the packet is an express snoop 144 bool is_express_snoop = pkt->isExpressSnoop(); 145 146 // determine the destination based on the address 147 PortID master_port_id = findPort(pkt->getAddr()); 148 149 // test if the crossbar should be considered occupied for the current 150 // port, and exclude express snoops from the check 151 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 152 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 153 src_port->name(), pkt->cmdString(), pkt->getAddr()); 154 return false; 155 } 156 157 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 158 src_port->name(), pkt->cmdString(), is_express_snoop, 159 pkt->getAddr()); 160 161 // store size and command as they might be modified when 162 // forwarding the packet 163 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 164 unsigned int pkt_cmd = pkt->cmdToIndex(); 165 166 // set the source port for routing of the response 167 pkt->setSrc(slave_port_id); 168 169 calcPacketTiming(pkt); 170 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 171 172 // uncacheable requests need never be snooped 173 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 174 // the packet is a memory-mapped request and should be 175 // broadcasted to our snoopers but the source 176 if (snoopFilter) { 177 // check with the snoop filter where to forward this packet 178 auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 179 packetFinishTime += sf_res.second * clockPeriod(); 180 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 181 " SF size: %i lat: %i\n", src_port->name(), 182 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 183 sf_res.second); 184 forwardTiming(pkt, slave_port_id, sf_res.first); 185 } else { 186 forwardTiming(pkt, slave_port_id); 187 } 188 } 189 190 // remember if we add an outstanding req so we can undo it if 191 // necessary, if the packet needs a response, we should add it 192 // as outstanding and express snoops never fail so there is 193 // not need to worry about them 194 bool add_outstanding = !is_express_snoop && pkt->needsResponse(); 195 196 // keep track that we have an outstanding request packet 197 // matching this request, this is used by the coherency 198 // mechanism in determining what to do with snoop responses 199 // (in recvTimingSnoop) 200 if (add_outstanding) { 201 // we should never have an exsiting request outstanding 202 assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 203 outstandingReq.insert(pkt->req); 204 } 205 206 // Note: Cannot create a copy of the full packet, here. 207 MemCmd orig_cmd(pkt->cmd); 208 209 // since it is a normal request, attempt to send the packet 210 bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 211 212 if (snoopFilter && !pkt->req->isUncacheable() 213 && !system->bypassCaches()) { 214 // The packet may already be overwritten by the sendTimingReq function. 215 // The snoop filter needs to see the original request *and* the return 216 // status of the send operation, so we need to recreate the original 217 // request. Atomic mode does not have the issue, as there the send 218 // operation and the response happen instantaneously and don't need two 219 // phase tracking. 220 MemCmd tmp_cmd(pkt->cmd); 221 pkt->cmd = orig_cmd; 222 // Let the snoop filter know about the success of the send operation 223 snoopFilter->updateRequest(pkt, *src_port, !success); 224 pkt->cmd = tmp_cmd; 225 } 226 227 // if this is an express snoop, we are done at this point 228 if (is_express_snoop) { 229 assert(success); 230 snoops++; 231 } else { 232 // for normal requests, check if successful 233 if (!success) { 234 // inhibited packets should never be forced to retry 235 assert(!pkt->memInhibitAsserted()); 236 237 // if it was added as outstanding and the send failed, then 238 // erase it again 239 if (add_outstanding) 240 outstandingReq.erase(pkt->req); 241 242 // undo the calculation so we can check for 0 again 243 pkt->firstWordDelay = pkt->lastWordDelay = 0; 244 245 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 246 src_port->name(), pkt->cmdString(), pkt->getAddr()); 247 248 // update the layer state and schedule an idle event 249 reqLayers[master_port_id]->failedTiming(src_port, 250 clockEdge(headerCycles)); 251 } else { 252 // update the layer state and schedule an idle event 253 reqLayers[master_port_id]->succeededTiming(packetFinishTime); 254 } 255 } 256 257 // stats updates only consider packets that were successfully sent 258 if (success) { 259 pktCount[slave_port_id][master_port_id]++; 260 pktSize[slave_port_id][master_port_id] += pkt_size; 261 transDist[pkt_cmd]++; 262 } 263 264 return success; 265} 266 267bool 268CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 269{ 270 // determine the source port based on the id 271 MasterPort *src_port = masterPorts[master_port_id]; 272 273 // determine the destination based on what is stored in the packet 274 PortID slave_port_id = pkt->getDest(); 275 assert(slave_port_id != InvalidPortID); 276 assert(slave_port_id < respLayers.size()); 277 278 // test if the crossbar should be considered occupied for the 279 // current port 280 if (!respLayers[slave_port_id]->tryTiming(src_port)) { 281 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 282 src_port->name(), pkt->cmdString(), pkt->getAddr()); 283 return false; 284 } 285 286 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 287 src_port->name(), pkt->cmdString(), pkt->getAddr()); 288 289 // store size and command as they might be modified when 290 // forwarding the packet 291 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 292 unsigned int pkt_cmd = pkt->cmdToIndex(); 293 294 calcPacketTiming(pkt); 295 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 296 297 // the packet is a normal response to a request that we should 298 // have seen passing through the crossbar 299 assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 300 301 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 302 // let the snoop filter inspect the response and update its state 303 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 304 } 305 306 // remove it as outstanding 307 outstandingReq.erase(pkt->req); 308 309 // send the packet through the destination slave port 310 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 311 312 // currently it is illegal to block responses... can lead to 313 // deadlock 314 assert(success); 315 316 respLayers[slave_port_id]->succeededTiming(packetFinishTime); 317 318 // stats updates 319 pktCount[slave_port_id][master_port_id]++; 320 pktSize[slave_port_id][master_port_id] += pkt_size; 321 transDist[pkt_cmd]++; 322 323 return true; 324} 325 326void 327CoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 328{ 329 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 330 masterPorts[master_port_id]->name(), pkt->cmdString(), 331 pkt->getAddr()); 332 333 // update stats here as we know the forwarding will succeed 334 transDist[pkt->cmdToIndex()]++; 335 snoops++; 336 337 // we should only see express snoops from caches 338 assert(pkt->isExpressSnoop()); 339 340 // set the source port for routing of the response 341 pkt->setSrc(master_port_id); 342 343 if (snoopFilter) { 344 // let the Snoop Filter work its magic and guide probing 345 auto sf_res = snoopFilter->lookupSnoop(pkt); 346 // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 347 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 348 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 349 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 350 sf_res.second); 351 352 // forward to all snoopers 353 forwardTiming(pkt, InvalidPortID, sf_res.first); 354 } else { 355 forwardTiming(pkt, InvalidPortID); 356 } 357 358 // a snoop request came from a connected slave device (one of 359 // our master ports), and if it is not coming from the slave 360 // device responsible for the address range something is 361 // wrong, hence there is nothing further to do as the packet 362 // would be going back to where it came from 363 assert(master_port_id == findPort(pkt->getAddr())); 364} 365 366bool 367CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 368{ 369 // determine the source port based on the id 370 SlavePort* src_port = slavePorts[slave_port_id]; 371 372 // get the destination from the packet 373 PortID dest_port_id = pkt->getDest(); 374 assert(dest_port_id != InvalidPortID); 375 376 // determine if the response is from a snoop request we 377 // created as the result of a normal request (in which case it 378 // should be in the outstandingReq), or if we merely forwarded 379 // someone else's snoop request 380 bool forwardAsSnoop = outstandingReq.find(pkt->req) == 381 outstandingReq.end(); 382 383 // test if the crossbar should be considered occupied for the 384 // current port, note that the check is bypassed if the response 385 // is being passed on as a normal response since this is occupying 386 // the response layer rather than the snoop response layer 387 if (forwardAsSnoop) { 388 assert(dest_port_id < snoopLayers.size()); 389 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 390 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 391 src_port->name(), pkt->cmdString(), pkt->getAddr()); 392 return false; 393 } 394 } else { 395 // get the master port that mirrors this slave port internally 396 MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 397 assert(dest_port_id < respLayers.size()); 398 if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 399 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 400 snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 401 return false; 402 } 403 } 404 405 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 406 src_port->name(), pkt->cmdString(), pkt->getAddr()); 407 408 // store size and command as they might be modified when 409 // forwarding the packet 410 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 411 unsigned int pkt_cmd = pkt->cmdToIndex(); 412 413 // responses are never express snoops 414 assert(!pkt->isExpressSnoop()); 415 416 calcPacketTiming(pkt); 417 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 418 419 // forward it either as a snoop response or a normal response 420 if (forwardAsSnoop) { 421 // this is a snoop response to a snoop request we forwarded, 422 // e.g. coming from the L1 and going to the L2, and it should 423 // be forwarded as a snoop response 424 425 if (snoopFilter) { 426 // update the probe filter so that it can properly track the line 427 snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 428 *masterPorts[dest_port_id]); 429 } 430 431 bool success M5_VAR_USED = 432 masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 433 pktCount[slave_port_id][dest_port_id]++; 434 pktSize[slave_port_id][dest_port_id] += pkt_size; 435 assert(success); 436 437 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 438 } else { 439 // we got a snoop response on one of our slave ports, 440 // i.e. from a coherent master connected to the crossbar, and 441 // since we created the snoop request as part of recvTiming, 442 // this should now be a normal response again 443 outstandingReq.erase(pkt->req); 444 445 // this is a snoop response from a coherent master, with a 446 // destination field set on its way through the crossbar as 447 // request, hence it should never go back to where the snoop 448 // response came from, but instead to where the original 449 // request came from 450 assert(slave_port_id != dest_port_id); 451 452 if (snoopFilter) { 453 // update the probe filter so that it can properly track the line 454 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 455 *slavePorts[dest_port_id]); 456 } 457 458 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 459 " FWD RESP\n", src_port->name(), pkt->cmdString(), 460 pkt->getAddr()); 461 462 // as a normal response, it should go back to a master through 463 // one of our slave ports, at this point we are ignoring the 464 // fact that the response layer could be busy and do not touch 465 // its state 466 bool success M5_VAR_USED = 467 slavePorts[dest_port_id]->sendTimingResp(pkt); 468 469 // @todo Put the response in an internal FIFO and pass it on 470 // to the response layer from there 471 472 // currently it is illegal to block responses... can lead 473 // to deadlock 474 assert(success); 475 476 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 477 } 478 479 // stats updates 480 transDist[pkt_cmd]++; 481 snoops++; 482 483 return true; 484} 485 486 487void 488CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 489 const std::vector<SlavePort*>& dests) 490{ 491 DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 492 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 493 494 // snoops should only happen if the system isn't bypassing caches 495 assert(!system->bypassCaches()); 496 497 unsigned fanout = 0; 498 499 for (const auto& p: dests) { 500 // we could have gotten this request from a snooping master 501 // (corresponding to our own slave port that is also in 502 // snoopPorts) and should not send it back to where it came 503 // from 504 if (exclude_slave_port_id == InvalidPortID || 505 p->getId() != exclude_slave_port_id) { 506 // cache is not allowed to refuse snoop 507 p->sendTimingSnoopReq(pkt); 508 fanout++; 509 } 510 } 511 512 // Stats for fanout of this forward operation 513 snoopFanout.sample(fanout); 514} 515 516void 517CoherentXBar::recvRetry(PortID master_port_id) 518{ 519 // responses and snoop responses never block on forwarding them, 520 // so the retry will always be coming from a port to which we 521 // tried to forward a request 522 reqLayers[master_port_id]->recvRetry(); 523} 524 525Tick 526CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 527{ 528 DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 529 slavePorts[slave_port_id]->name(), pkt->getAddr(), 530 pkt->cmdString()); 531 532 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 533 unsigned int pkt_cmd = pkt->cmdToIndex(); 534 535 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 536 Tick snoop_response_latency = 0; 537 538 // uncacheable requests need never be snooped 539 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 540 // forward to all snoopers but the source 541 std::pair<MemCmd, Tick> snoop_result; 542 if (snoopFilter) { 543 // check with the snoop filter where to forward this packet 544 auto sf_res = 545 snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 546 snoop_response_latency += sf_res.second * clockPeriod(); 547 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 548 " SF size: %i lat: %i\n", __func__, 549 slavePorts[slave_port_id]->name(), pkt->cmdString(), 550 pkt->getAddr(), sf_res.first.size(), sf_res.second); 551 snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 552 sf_res.first); 553 } else { 554 snoop_result = forwardAtomic(pkt, slave_port_id); 555 } 556 snoop_response_cmd = snoop_result.first; 557 snoop_response_latency += snoop_result.second; 558 } 559 560 // even if we had a snoop response, we must continue and also 561 // perform the actual request at the destination 562 PortID master_port_id = findPort(pkt->getAddr()); 563 564 // stats updates for the request 565 pktCount[slave_port_id][master_port_id]++; 566 pktSize[slave_port_id][master_port_id] += pkt_size; 567 transDist[pkt_cmd]++; 568 569 // forward the request to the appropriate destination 570 Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 571 572 // Lower levels have replied, tell the snoop filter 573 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 574 pkt->isResponse()) { 575 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 576 } 577 578 // if we got a response from a snooper, restore it here 579 if (snoop_response_cmd != MemCmd::InvalidCmd) { 580 // no one else should have responded 581 assert(!pkt->isResponse()); 582 pkt->cmd = snoop_response_cmd; 583 response_latency = snoop_response_latency; 584 } 585 586 // add the response data 587 if (pkt->isResponse()) { 588 pkt_size = pkt->hasData() ? pkt->getSize() : 0; 589 pkt_cmd = pkt->cmdToIndex(); 590 591 // stats updates 592 pktCount[slave_port_id][master_port_id]++; 593 pktSize[slave_port_id][master_port_id] += pkt_size; 594 transDist[pkt_cmd]++; 595 } 596 597 // @todo: Not setting first-word time 598 pkt->lastWordDelay = response_latency; 599 return response_latency; 600} 601 602Tick 603CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 604{ 605 DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 606 masterPorts[master_port_id]->name(), pkt->getAddr(), 607 pkt->cmdString()); 608 609 // add the request snoop data 610 snoops++; 611 612 // forward to all snoopers 613 std::pair<MemCmd, Tick> snoop_result; 614 Tick snoop_response_latency = 0; 615 if (snoopFilter) { 616 auto sf_res = snoopFilter->lookupSnoop(pkt); 617 snoop_response_latency += sf_res.second * clockPeriod(); 618 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 619 __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 620 pkt->getAddr(), sf_res.first.size(), sf_res.second); 621 snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 622 sf_res.first); 623 } else { 624 snoop_result = forwardAtomic(pkt, InvalidPortID); 625 } 626 MemCmd snoop_response_cmd = snoop_result.first; 627 snoop_response_latency += snoop_result.second; 628 629 if (snoop_response_cmd != MemCmd::InvalidCmd) 630 pkt->cmd = snoop_response_cmd; 631 632 // add the response snoop data 633 if (pkt->isResponse()) { 634 snoops++; 635 } 636 637 // @todo: Not setting first-word time 638 pkt->lastWordDelay = snoop_response_latency; 639 return snoop_response_latency; 640} 641 642std::pair<MemCmd, Tick> 643CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 644 PortID source_master_port_id, 645 const std::vector<SlavePort*>& dests) 646{ 647 // the packet may be changed on snoops, record the original 648 // command to enable us to restore it between snoops so that 649 // additional snoops can take place properly 650 MemCmd orig_cmd = pkt->cmd; 651 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 652 Tick snoop_response_latency = 0; 653 654 // snoops should only happen if the system isn't bypassing caches 655 assert(!system->bypassCaches()); 656 657 unsigned fanout = 0; 658 659 for (const auto& p: dests) { 660 // we could have gotten this request from a snooping master 661 // (corresponding to our own slave port that is also in 662 // snoopPorts) and should not send it back to where it came 663 // from 664 if (exclude_slave_port_id != InvalidPortID && 665 p->getId() == exclude_slave_port_id) 666 continue; 667 668 Tick latency = p->sendAtomicSnoop(pkt); 669 fanout++; 670 671 // in contrast to a functional access, we have to keep on 672 // going as all snoopers must be updated even if we get a 673 // response 674 if (!pkt->isResponse()) 675 continue; 676 677 // response from snoop agent 678 assert(pkt->cmd != orig_cmd); 679 assert(pkt->memInhibitAsserted()); 680 // should only happen once 681 assert(snoop_response_cmd == MemCmd::InvalidCmd); 682 // save response state 683 snoop_response_cmd = pkt->cmd; 684 snoop_response_latency = latency; 685 686 if (snoopFilter) { 687 // Handle responses by the snoopers and differentiate between 688 // responses to requests from above and snoops from below 689 if (source_master_port_id != InvalidPortID) { 690 // Getting a response for a snoop from below 691 assert(exclude_slave_port_id == InvalidPortID); 692 snoopFilter->updateSnoopForward(pkt, *p, 693 *masterPorts[source_master_port_id]); 694 } else { 695 // Getting a response for a request from above 696 assert(source_master_port_id == InvalidPortID); 697 snoopFilter->updateSnoopResponse(pkt, *p, 698 *slavePorts[exclude_slave_port_id]); 699 } 700 } 701 // restore original packet state for remaining snoopers 702 pkt->cmd = orig_cmd; 703 } 704 705 // Stats for fanout 706 snoopFanout.sample(fanout); 707 708 // the packet is restored as part of the loop and any potential 709 // snoop response is part of the returned pair 710 return std::make_pair(snoop_response_cmd, snoop_response_latency); 711} 712 713void 714CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 715{ 716 if (!pkt->isPrint()) { 717 // don't do DPRINTFs on PrintReq as it clutters up the output 718 DPRINTF(CoherentXBar, 719 "recvFunctional: packet src %s addr 0x%x cmd %s\n", 720 slavePorts[slave_port_id]->name(), pkt->getAddr(), 721 pkt->cmdString()); 722 } 723 724 // uncacheable requests need never be snooped 725 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 726 // forward to all snoopers but the source 727 forwardFunctional(pkt, slave_port_id); 728 } 729 730 // there is no need to continue if the snooping has found what we 731 // were looking for and the packet is already a response 732 if (!pkt->isResponse()) { 733 PortID dest_id = findPort(pkt->getAddr()); 734 735 masterPorts[dest_id]->sendFunctional(pkt); 736 } 737} 738 739void 740CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 741{ 742 if (!pkt->isPrint()) { 743 // don't do DPRINTFs on PrintReq as it clutters up the output 744 DPRINTF(CoherentXBar, 745 "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 746 masterPorts[master_port_id]->name(), pkt->getAddr(), 747 pkt->cmdString()); 748 } 749 750 // forward to all snoopers 751 forwardFunctional(pkt, InvalidPortID); 752} 753 754void 755CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 756{ 757 // snoops should only happen if the system isn't bypassing caches 758 assert(!system->bypassCaches()); 759 760 for (const auto& p: snoopPorts) { 761 // we could have gotten this request from a snooping master 762 // (corresponding to our own slave port that is also in 763 // snoopPorts) and should not send it back to where it came 764 // from 765 if (exclude_slave_port_id == InvalidPortID || 766 p->getId() != exclude_slave_port_id) 767 p->sendFunctionalSnoop(pkt); 768 769 // if we get a response we are done 770 if (pkt->isResponse()) { 771 break; 772 } 773 } 774} 775 776unsigned int 777CoherentXBar::drain(DrainManager *dm) 778{ 779 // sum up the individual layers 780 unsigned int total = 0; 781 for (auto l: reqLayers) 782 total += l->drain(dm); 783 for (auto l: respLayers) 784 total += l->drain(dm); 785 for (auto l: snoopLayers) 786 total += l->drain(dm); 787 return total; 788} 789 790void 791CoherentXBar::regStats() 792{ 793 // register the stats of the base class and our layers 794 BaseXBar::regStats(); 795 for (auto l: reqLayers) 796 l->regStats(); 797 for (auto l: respLayers) 798 l->regStats(); 799 for (auto l: snoopLayers) 800 l->regStats(); 801 802 snoops 803 .name(name() + ".snoops") 804 .desc("Total snoops (count)") 805 ; 806 807 snoopFanout 808 .init(0, snoopPorts.size(), 1) 809 .name(name() + ".snoop_fanout") 810 .desc("Request fanout histogram") 811 ; 812} 813 814CoherentXBar * 815CoherentXBarParams::create() 816{ 817 return new CoherentXBar(this); 818} 819