coherent_xbar.cc revision 10405
1/* 2 * Copyright (c) 2011-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2006 The Regents of The University of Michigan 15 * All rights reserved. 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions are 19 * met: redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer; 21 * redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution; 24 * neither the name of the copyright holders nor the names of its 25 * contributors may be used to endorse or promote products derived from 26 * this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 * 40 * Authors: Ali Saidi 41 * Andreas Hansson 42 * William Wang 43 */ 44 45/** 46 * @file 47 * Definition of a crossbar object. 48 */ 49 50#include "base/misc.hh" 51#include "base/trace.hh" 52#include "debug/AddrRanges.hh" 53#include "debug/CoherentXBar.hh" 54#include "mem/coherent_xbar.hh" 55#include "sim/system.hh" 56 57CoherentXBar::CoherentXBar(const CoherentXBarParams *p) 58 : BaseXBar(p), system(p->system), snoopFilter(p->snoop_filter) 59{ 60 // create the ports based on the size of the master and slave 61 // vector ports, and the presence of the default port, the ports 62 // are enumerated starting from zero 63 for (int i = 0; i < p->port_master_connection_count; ++i) { 64 std::string portName = csprintf("%s.master[%d]", name(), i); 65 MasterPort* bp = new CoherentXBarMasterPort(portName, *this, i); 66 masterPorts.push_back(bp); 67 reqLayers.push_back(new ReqLayer(*bp, *this, 68 csprintf(".reqLayer%d", i))); 69 snoopLayers.push_back(new SnoopLayer(*bp, *this, 70 csprintf(".snoopLayer%d", i))); 71 } 72 73 // see if we have a default slave device connected and if so add 74 // our corresponding master port 75 if (p->port_default_connection_count) { 76 defaultPortID = masterPorts.size(); 77 std::string portName = name() + ".default"; 78 MasterPort* bp = new CoherentXBarMasterPort(portName, *this, 79 defaultPortID); 80 masterPorts.push_back(bp); 81 reqLayers.push_back(new ReqLayer(*bp, *this, csprintf(".reqLayer%d", 82 defaultPortID))); 83 snoopLayers.push_back(new SnoopLayer(*bp, *this, 84 csprintf(".snoopLayer%d", 85 defaultPortID))); 86 } 87 88 // create the slave ports, once again starting at zero 89 for (int i = 0; i < p->port_slave_connection_count; ++i) { 90 std::string portName = csprintf("%s.slave[%d]", name(), i); 91 SlavePort* bp = new CoherentXBarSlavePort(portName, *this, i); 92 slavePorts.push_back(bp); 93 respLayers.push_back(new RespLayer(*bp, *this, 94 csprintf(".respLayer%d", i))); 95 snoopRespPorts.push_back(new SnoopRespPort(*bp, *this)); 96 } 97 98 if (snoopFilter) 99 snoopFilter->setSlavePorts(slavePorts); 100 101 clearPortCache(); 102} 103 104CoherentXBar::~CoherentXBar() 105{ 106 for (auto l: reqLayers) 107 delete l; 108 for (auto l: respLayers) 109 delete l; 110 for (auto l: snoopLayers) 111 delete l; 112 for (auto p: snoopRespPorts) 113 delete p; 114} 115 116void 117CoherentXBar::init() 118{ 119 // the base class is responsible for determining the block size 120 BaseXBar::init(); 121 122 // iterate over our slave ports and determine which of our 123 // neighbouring master ports are snooping and add them as snoopers 124 for (const auto& p: slavePorts) { 125 // check if the connected master port is snooping 126 if (p->isSnooping()) { 127 DPRINTF(AddrRanges, "Adding snooping master %s\n", 128 p->getMasterPort().name()); 129 snoopPorts.push_back(p); 130 } 131 } 132 133 if (snoopPorts.empty()) 134 warn("CoherentXBar %s has no snooping ports attached!\n", name()); 135} 136 137bool 138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 139{ 140 // determine the source port based on the id 141 SlavePort *src_port = slavePorts[slave_port_id]; 142 143 // remember if the packet is an express snoop 144 bool is_express_snoop = pkt->isExpressSnoop(); 145 146 // determine the destination based on the address 147 PortID master_port_id = findPort(pkt->getAddr()); 148 149 // test if the crossbar should be considered occupied for the current 150 // port, and exclude express snoops from the check 151 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 152 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", 153 src_port->name(), pkt->cmdString(), pkt->getAddr()); 154 return false; 155 } 156 157 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s expr %d 0x%x\n", 158 src_port->name(), pkt->cmdString(), is_express_snoop, 159 pkt->getAddr()); 160 161 // store size and command as they might be modified when 162 // forwarding the packet 163 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 164 unsigned int pkt_cmd = pkt->cmdToIndex(); 165 166 // set the source port for routing of the response 167 pkt->setSrc(slave_port_id); 168 169 calcPacketTiming(pkt); 170 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 171 172 // uncacheable requests need never be snooped 173 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 174 // the packet is a memory-mapped request and should be 175 // broadcasted to our snoopers but the source 176 if (snoopFilter) { 177 // check with the snoop filter where to forward this packet 178 auto sf_res = snoopFilter->lookupRequest(pkt, *src_port); 179 packetFinishTime += sf_res.second * clockPeriod(); 180 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x"\ 181 " SF size: %i lat: %i\n", src_port->name(), 182 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 183 sf_res.second); 184 forwardTiming(pkt, slave_port_id, sf_res.first); 185 } else { 186 forwardTiming(pkt, slave_port_id); 187 } 188 } 189 190 // remember if we add an outstanding req so we can undo it if 191 // necessary, if the packet needs a response, we should add it 192 // as outstanding and express snoops never fail so there is 193 // not need to worry about them 194 bool add_outstanding = !is_express_snoop && pkt->needsResponse(); 195 196 // keep track that we have an outstanding request packet 197 // matching this request, this is used by the coherency 198 // mechanism in determining what to do with snoop responses 199 // (in recvTimingSnoop) 200 if (add_outstanding) { 201 // we should never have an exsiting request outstanding 202 assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 203 outstandingReq.insert(pkt->req); 204 } 205 206 // Note: Cannot create a copy of the full packet, here. 207 MemCmd orig_cmd(pkt->cmd); 208 209 // since it is a normal request, attempt to send the packet 210 bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 211 212 if (snoopFilter && !pkt->req->isUncacheable() 213 && !system->bypassCaches()) { 214 // The packet may already be overwritten by the sendTimingReq function. 215 // The snoop filter needs to see the original request *and* the return 216 // status of the send operation, so we need to recreate the original 217 // request. Atomic mode does not have the issue, as there the send 218 // operation and the response happen instantaneously and don't need two 219 // phase tracking. 220 MemCmd tmp_cmd(pkt->cmd); 221 pkt->cmd = orig_cmd; 222 // Let the snoop filter know about the success of the send operation 223 snoopFilter->updateRequest(pkt, *src_port, !success); 224 pkt->cmd = tmp_cmd; 225 } 226 227 // if this is an express snoop, we are done at this point 228 if (is_express_snoop) { 229 assert(success); 230 snoops++; 231 } else { 232 // for normal requests, check if successful 233 if (!success) { 234 // inhibited packets should never be forced to retry 235 assert(!pkt->memInhibitAsserted()); 236 237 // if it was added as outstanding and the send failed, then 238 // erase it again 239 if (add_outstanding) 240 outstandingReq.erase(pkt->req); 241 242 // undo the calculation so we can check for 0 again 243 pkt->firstWordDelay = pkt->lastWordDelay = 0; 244 245 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 246 src_port->name(), pkt->cmdString(), pkt->getAddr()); 247 248 // update the layer state and schedule an idle event 249 reqLayers[master_port_id]->failedTiming(src_port, 250 clockEdge(headerCycles)); 251 } else { 252 // update the layer state and schedule an idle event 253 reqLayers[master_port_id]->succeededTiming(packetFinishTime); 254 } 255 } 256 257 // stats updates only consider packets that were successfully sent 258 if (success) { 259 pktCount[slave_port_id][master_port_id]++; 260 pktSize[slave_port_id][master_port_id] += pkt_size; 261 transDist[pkt_cmd]++; 262 } 263 264 return success; 265} 266 267bool 268CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 269{ 270 // determine the source port based on the id 271 MasterPort *src_port = masterPorts[master_port_id]; 272 273 // determine the destination based on what is stored in the packet 274 PortID slave_port_id = pkt->getDest(); 275 276 // test if the crossbar should be considered occupied for the 277 // current port 278 if (!respLayers[slave_port_id]->tryTiming(src_port)) { 279 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 280 src_port->name(), pkt->cmdString(), pkt->getAddr()); 281 return false; 282 } 283 284 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x\n", 285 src_port->name(), pkt->cmdString(), pkt->getAddr()); 286 287 // store size and command as they might be modified when 288 // forwarding the packet 289 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 290 unsigned int pkt_cmd = pkt->cmdToIndex(); 291 292 calcPacketTiming(pkt); 293 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 294 295 // the packet is a normal response to a request that we should 296 // have seen passing through the crossbar 297 assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 298 299 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 300 // let the snoop filter inspect the response and update its state 301 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 302 } 303 304 // remove it as outstanding 305 outstandingReq.erase(pkt->req); 306 307 // send the packet through the destination slave port 308 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 309 310 // currently it is illegal to block responses... can lead to 311 // deadlock 312 assert(success); 313 314 respLayers[slave_port_id]->succeededTiming(packetFinishTime); 315 316 // stats updates 317 pktCount[slave_port_id][master_port_id]++; 318 pktSize[slave_port_id][master_port_id] += pkt_size; 319 transDist[pkt_cmd]++; 320 321 return true; 322} 323 324void 325CoherentXBar::recvTimingSnoopReq(PacketPtr pkt, PortID master_port_id) 326{ 327 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x\n", 328 masterPorts[master_port_id]->name(), pkt->cmdString(), 329 pkt->getAddr()); 330 331 // update stats here as we know the forwarding will succeed 332 transDist[pkt->cmdToIndex()]++; 333 snoops++; 334 335 // we should only see express snoops from caches 336 assert(pkt->isExpressSnoop()); 337 338 // set the source port for routing of the response 339 pkt->setSrc(master_port_id); 340 341 if (snoopFilter) { 342 // let the Snoop Filter work its magic and guide probing 343 auto sf_res = snoopFilter->lookupSnoop(pkt); 344 // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 345 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 346 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 347 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 348 sf_res.second); 349 350 // forward to all snoopers 351 forwardTiming(pkt, InvalidPortID, sf_res.first); 352 } else { 353 forwardTiming(pkt, InvalidPortID); 354 } 355 356 // a snoop request came from a connected slave device (one of 357 // our master ports), and if it is not coming from the slave 358 // device responsible for the address range something is 359 // wrong, hence there is nothing further to do as the packet 360 // would be going back to where it came from 361 assert(master_port_id == findPort(pkt->getAddr())); 362} 363 364bool 365CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 366{ 367 // determine the source port based on the id 368 SlavePort* src_port = slavePorts[slave_port_id]; 369 370 // get the destination from the packet 371 PortID dest_port_id = pkt->getDest(); 372 373 // determine if the response is from a snoop request we 374 // created as the result of a normal request (in which case it 375 // should be in the outstandingReq), or if we merely forwarded 376 // someone else's snoop request 377 bool forwardAsSnoop = outstandingReq.find(pkt->req) == 378 outstandingReq.end(); 379 380 // test if the crossbar should be considered occupied for the 381 // current port, note that the check is bypassed if the response 382 // is being passed on as a normal response since this is occupying 383 // the response layer rather than the snoop response layer 384 if (forwardAsSnoop) { 385 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { 386 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 387 src_port->name(), pkt->cmdString(), pkt->getAddr()); 388 return false; 389 } 390 } else { 391 // get the master port that mirrors this slave port internally 392 MasterPort* snoop_port = snoopRespPorts[slave_port_id]; 393 if (!respLayers[dest_port_id]->tryTiming(snoop_port)) { 394 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x BUSY\n", 395 snoop_port->name(), pkt->cmdString(), pkt->getAddr()); 396 return false; 397 } 398 } 399 400 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x\n", 401 src_port->name(), pkt->cmdString(), pkt->getAddr()); 402 403 // store size and command as they might be modified when 404 // forwarding the packet 405 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 406 unsigned int pkt_cmd = pkt->cmdToIndex(); 407 408 // responses are never express snoops 409 assert(!pkt->isExpressSnoop()); 410 411 calcPacketTiming(pkt); 412 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 413 414 // forward it either as a snoop response or a normal response 415 if (forwardAsSnoop) { 416 // this is a snoop response to a snoop request we forwarded, 417 // e.g. coming from the L1 and going to the L2, and it should 418 // be forwarded as a snoop response 419 420 if (snoopFilter) { 421 // update the probe filter so that it can properly track the line 422 snoopFilter->updateSnoopForward(pkt, *slavePorts[slave_port_id], 423 *masterPorts[dest_port_id]); 424 } 425 426 bool success M5_VAR_USED = 427 masterPorts[dest_port_id]->sendTimingSnoopResp(pkt); 428 pktCount[slave_port_id][dest_port_id]++; 429 pktSize[slave_port_id][dest_port_id] += pkt_size; 430 assert(success); 431 432 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 433 } else { 434 // we got a snoop response on one of our slave ports, 435 // i.e. from a coherent master connected to the crossbar, and 436 // since we created the snoop request as part of recvTiming, 437 // this should now be a normal response again 438 outstandingReq.erase(pkt->req); 439 440 // this is a snoop response from a coherent master, with a 441 // destination field set on its way through the crossbar as 442 // request, hence it should never go back to where the snoop 443 // response came from, but instead to where the original 444 // request came from 445 assert(slave_port_id != dest_port_id); 446 447 if (snoopFilter) { 448 // update the probe filter so that it can properly track the line 449 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 450 *slavePorts[dest_port_id]); 451 } 452 453 DPRINTF(CoherentXBar, "recvTimingSnoopResp: src %s %s 0x%x"\ 454 " FWD RESP\n", src_port->name(), pkt->cmdString(), 455 pkt->getAddr()); 456 457 // as a normal response, it should go back to a master through 458 // one of our slave ports, at this point we are ignoring the 459 // fact that the response layer could be busy and do not touch 460 // its state 461 bool success M5_VAR_USED = 462 slavePorts[dest_port_id]->sendTimingResp(pkt); 463 464 // @todo Put the response in an internal FIFO and pass it on 465 // to the response layer from there 466 467 // currently it is illegal to block responses... can lead 468 // to deadlock 469 assert(success); 470 471 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 472 } 473 474 // stats updates 475 transDist[pkt_cmd]++; 476 snoops++; 477 478 return true; 479} 480 481 482void 483CoherentXBar::forwardTiming(PacketPtr pkt, PortID exclude_slave_port_id, 484 const std::vector<SlavePort*>& dests) 485{ 486 DPRINTF(CoherentXBar, "%s for %s address %x size %d\n", __func__, 487 pkt->cmdString(), pkt->getAddr(), pkt->getSize()); 488 489 // snoops should only happen if the system isn't bypassing caches 490 assert(!system->bypassCaches()); 491 492 unsigned fanout = 0; 493 494 for (const auto& p: dests) { 495 // we could have gotten this request from a snooping master 496 // (corresponding to our own slave port that is also in 497 // snoopPorts) and should not send it back to where it came 498 // from 499 if (exclude_slave_port_id == InvalidPortID || 500 p->getId() != exclude_slave_port_id) { 501 // cache is not allowed to refuse snoop 502 p->sendTimingSnoopReq(pkt); 503 fanout++; 504 } 505 } 506 507 // Stats for fanout of this forward operation 508 snoopFanout.sample(fanout); 509} 510 511void 512CoherentXBar::recvRetry(PortID master_port_id) 513{ 514 // responses and snoop responses never block on forwarding them, 515 // so the retry will always be coming from a port to which we 516 // tried to forward a request 517 reqLayers[master_port_id]->recvRetry(); 518} 519 520Tick 521CoherentXBar::recvAtomic(PacketPtr pkt, PortID slave_port_id) 522{ 523 DPRINTF(CoherentXBar, "recvAtomic: packet src %s addr 0x%x cmd %s\n", 524 slavePorts[slave_port_id]->name(), pkt->getAddr(), 525 pkt->cmdString()); 526 527 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 528 unsigned int pkt_cmd = pkt->cmdToIndex(); 529 530 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 531 Tick snoop_response_latency = 0; 532 533 // uncacheable requests need never be snooped 534 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 535 // forward to all snoopers but the source 536 std::pair<MemCmd, Tick> snoop_result; 537 if (snoopFilter) { 538 // check with the snoop filter where to forward this packet 539 auto sf_res = 540 snoopFilter->lookupRequest(pkt, *slavePorts[slave_port_id]); 541 snoop_response_latency += sf_res.second * clockPeriod(); 542 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x"\ 543 " SF size: %i lat: %i\n", __func__, 544 slavePorts[slave_port_id]->name(), pkt->cmdString(), 545 pkt->getAddr(), sf_res.first.size(), sf_res.second); 546 snoop_result = forwardAtomic(pkt, slave_port_id, InvalidPortID, 547 sf_res.first); 548 } else { 549 snoop_result = forwardAtomic(pkt, slave_port_id); 550 } 551 snoop_response_cmd = snoop_result.first; 552 snoop_response_latency += snoop_result.second; 553 } 554 555 // even if we had a snoop response, we must continue and also 556 // perform the actual request at the destination 557 PortID master_port_id = findPort(pkt->getAddr()); 558 559 // stats updates for the request 560 pktCount[slave_port_id][master_port_id]++; 561 pktSize[slave_port_id][master_port_id] += pkt_size; 562 transDist[pkt_cmd]++; 563 564 // forward the request to the appropriate destination 565 Tick response_latency = masterPorts[master_port_id]->sendAtomic(pkt); 566 567 // Lower levels have replied, tell the snoop filter 568 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches() && 569 pkt->isResponse()) { 570 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 571 } 572 573 // if we got a response from a snooper, restore it here 574 if (snoop_response_cmd != MemCmd::InvalidCmd) { 575 // no one else should have responded 576 assert(!pkt->isResponse()); 577 pkt->cmd = snoop_response_cmd; 578 response_latency = snoop_response_latency; 579 } 580 581 // add the response data 582 if (pkt->isResponse()) { 583 pkt_size = pkt->hasData() ? pkt->getSize() : 0; 584 pkt_cmd = pkt->cmdToIndex(); 585 586 // stats updates 587 pktCount[slave_port_id][master_port_id]++; 588 pktSize[slave_port_id][master_port_id] += pkt_size; 589 transDist[pkt_cmd]++; 590 } 591 592 // @todo: Not setting first-word time 593 pkt->lastWordDelay = response_latency; 594 return response_latency; 595} 596 597Tick 598CoherentXBar::recvAtomicSnoop(PacketPtr pkt, PortID master_port_id) 599{ 600 DPRINTF(CoherentXBar, "recvAtomicSnoop: packet src %s addr 0x%x cmd %s\n", 601 masterPorts[master_port_id]->name(), pkt->getAddr(), 602 pkt->cmdString()); 603 604 // add the request snoop data 605 snoops++; 606 607 // forward to all snoopers 608 std::pair<MemCmd, Tick> snoop_result; 609 Tick snoop_response_latency = 0; 610 if (snoopFilter) { 611 auto sf_res = snoopFilter->lookupSnoop(pkt); 612 snoop_response_latency += sf_res.second * clockPeriod(); 613 DPRINTF(CoherentXBar, "%s: src %s %s 0x%x SF size: %i lat: %i\n", 614 __func__, masterPorts[master_port_id]->name(), pkt->cmdString(), 615 pkt->getAddr(), sf_res.first.size(), sf_res.second); 616 snoop_result = forwardAtomic(pkt, InvalidPortID, master_port_id, 617 sf_res.first); 618 } else { 619 snoop_result = forwardAtomic(pkt, InvalidPortID); 620 } 621 MemCmd snoop_response_cmd = snoop_result.first; 622 snoop_response_latency += snoop_result.second; 623 624 if (snoop_response_cmd != MemCmd::InvalidCmd) 625 pkt->cmd = snoop_response_cmd; 626 627 // add the response snoop data 628 if (pkt->isResponse()) { 629 snoops++; 630 } 631 632 // @todo: Not setting first-word time 633 pkt->lastWordDelay = snoop_response_latency; 634 return snoop_response_latency; 635} 636 637std::pair<MemCmd, Tick> 638CoherentXBar::forwardAtomic(PacketPtr pkt, PortID exclude_slave_port_id, 639 PortID source_master_port_id, 640 const std::vector<SlavePort*>& dests) 641{ 642 // the packet may be changed on snoops, record the original 643 // command to enable us to restore it between snoops so that 644 // additional snoops can take place properly 645 MemCmd orig_cmd = pkt->cmd; 646 MemCmd snoop_response_cmd = MemCmd::InvalidCmd; 647 Tick snoop_response_latency = 0; 648 649 // snoops should only happen if the system isn't bypassing caches 650 assert(!system->bypassCaches()); 651 652 unsigned fanout = 0; 653 654 for (const auto& p: dests) { 655 // we could have gotten this request from a snooping master 656 // (corresponding to our own slave port that is also in 657 // snoopPorts) and should not send it back to where it came 658 // from 659 if (exclude_slave_port_id != InvalidPortID && 660 p->getId() == exclude_slave_port_id) 661 continue; 662 663 Tick latency = p->sendAtomicSnoop(pkt); 664 fanout++; 665 666 // in contrast to a functional access, we have to keep on 667 // going as all snoopers must be updated even if we get a 668 // response 669 if (!pkt->isResponse()) 670 continue; 671 672 // response from snoop agent 673 assert(pkt->cmd != orig_cmd); 674 assert(pkt->memInhibitAsserted()); 675 // should only happen once 676 assert(snoop_response_cmd == MemCmd::InvalidCmd); 677 // save response state 678 snoop_response_cmd = pkt->cmd; 679 snoop_response_latency = latency; 680 681 if (snoopFilter) { 682 // Handle responses by the snoopers and differentiate between 683 // responses to requests from above and snoops from below 684 if (source_master_port_id != InvalidPortID) { 685 // Getting a response for a snoop from below 686 assert(exclude_slave_port_id == InvalidPortID); 687 snoopFilter->updateSnoopForward(pkt, *p, 688 *masterPorts[source_master_port_id]); 689 } else { 690 // Getting a response for a request from above 691 assert(source_master_port_id == InvalidPortID); 692 snoopFilter->updateSnoopResponse(pkt, *p, 693 *slavePorts[exclude_slave_port_id]); 694 } 695 } 696 // restore original packet state for remaining snoopers 697 pkt->cmd = orig_cmd; 698 } 699 700 // Stats for fanout 701 snoopFanout.sample(fanout); 702 703 // the packet is restored as part of the loop and any potential 704 // snoop response is part of the returned pair 705 return std::make_pair(snoop_response_cmd, snoop_response_latency); 706} 707 708void 709CoherentXBar::recvFunctional(PacketPtr pkt, PortID slave_port_id) 710{ 711 if (!pkt->isPrint()) { 712 // don't do DPRINTFs on PrintReq as it clutters up the output 713 DPRINTF(CoherentXBar, 714 "recvFunctional: packet src %s addr 0x%x cmd %s\n", 715 slavePorts[slave_port_id]->name(), pkt->getAddr(), 716 pkt->cmdString()); 717 } 718 719 // uncacheable requests need never be snooped 720 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 721 // forward to all snoopers but the source 722 forwardFunctional(pkt, slave_port_id); 723 } 724 725 // there is no need to continue if the snooping has found what we 726 // were looking for and the packet is already a response 727 if (!pkt->isResponse()) { 728 PortID dest_id = findPort(pkt->getAddr()); 729 730 masterPorts[dest_id]->sendFunctional(pkt); 731 } 732} 733 734void 735CoherentXBar::recvFunctionalSnoop(PacketPtr pkt, PortID master_port_id) 736{ 737 if (!pkt->isPrint()) { 738 // don't do DPRINTFs on PrintReq as it clutters up the output 739 DPRINTF(CoherentXBar, 740 "recvFunctionalSnoop: packet src %s addr 0x%x cmd %s\n", 741 masterPorts[master_port_id]->name(), pkt->getAddr(), 742 pkt->cmdString()); 743 } 744 745 // forward to all snoopers 746 forwardFunctional(pkt, InvalidPortID); 747} 748 749void 750CoherentXBar::forwardFunctional(PacketPtr pkt, PortID exclude_slave_port_id) 751{ 752 // snoops should only happen if the system isn't bypassing caches 753 assert(!system->bypassCaches()); 754 755 for (const auto& p: snoopPorts) { 756 // we could have gotten this request from a snooping master 757 // (corresponding to our own slave port that is also in 758 // snoopPorts) and should not send it back to where it came 759 // from 760 if (exclude_slave_port_id == InvalidPortID || 761 p->getId() != exclude_slave_port_id) 762 p->sendFunctionalSnoop(pkt); 763 764 // if we get a response we are done 765 if (pkt->isResponse()) { 766 break; 767 } 768 } 769} 770 771unsigned int 772CoherentXBar::drain(DrainManager *dm) 773{ 774 // sum up the individual layers 775 unsigned int total = 0; 776 for (auto l: reqLayers) 777 total += l->drain(dm); 778 for (auto l: respLayers) 779 total += l->drain(dm); 780 for (auto l: snoopLayers) 781 total += l->drain(dm); 782 return total; 783} 784 785void 786CoherentXBar::regStats() 787{ 788 // register the stats of the base class and our layers 789 BaseXBar::regStats(); 790 for (auto l: reqLayers) 791 l->regStats(); 792 for (auto l: respLayers) 793 l->regStats(); 794 for (auto l: snoopLayers) 795 l->regStats(); 796 797 snoops 798 .name(name() + ".snoops") 799 .desc("Total snoops (count)") 800 ; 801 802 snoopFanout 803 .init(0, snoopPorts.size(), 1) 804 .name(name() + ".snoop_fanout") 805 .desc("Request fanout histogram") 806 ; 807} 808 809CoherentXBar * 810CoherentXBarParams::create() 811{ 812 return new CoherentXBar(this); 813} 814