coherent_xbar.cc (10572:fc4c90a7d2f5) | coherent_xbar.cc (10656:bd376adfb7d4) |
---|---|
1/* 2 * Copyright (c) 2011-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 128 unchanged lines hidden (view full) --- 137bool 138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 139{ 140 // determine the source port based on the id 141 SlavePort *src_port = slavePorts[slave_port_id]; 142 143 // remember if the packet is an express snoop 144 bool is_express_snoop = pkt->isExpressSnoop(); | 1/* 2 * Copyright (c) 2011-2014 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software --- 128 unchanged lines hidden (view full) --- 137bool 138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id) 139{ 140 // determine the source port based on the id 141 SlavePort *src_port = slavePorts[slave_port_id]; 142 143 // remember if the packet is an express snoop 144 bool is_express_snoop = pkt->isExpressSnoop(); |
145 bool is_inhibited = pkt->memInhibitAsserted(); 146 // for normal requests, going downstream, the express snoop flag 147 // and the inhibited flag should always be the same 148 assert(is_express_snoop == is_inhibited); |
|
145 146 // determine the destination based on the address 147 PortID master_port_id = findPort(pkt->getAddr()); 148 149 // test if the crossbar should be considered occupied for the current 150 // port, and exclude express snoops from the check 151 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 152 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", --- 5 unchanged lines hidden (view full) --- 158 src_port->name(), pkt->cmdString(), is_express_snoop, 159 pkt->getAddr()); 160 161 // store size and command as they might be modified when 162 // forwarding the packet 163 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 164 unsigned int pkt_cmd = pkt->cmdToIndex(); 165 | 149 150 // determine the destination based on the address 151 PortID master_port_id = findPort(pkt->getAddr()); 152 153 // test if the crossbar should be considered occupied for the current 154 // port, and exclude express snoops from the check 155 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) { 156 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n", --- 5 unchanged lines hidden (view full) --- 162 src_port->name(), pkt->cmdString(), is_express_snoop, 163 pkt->getAddr()); 164 165 // store size and command as they might be modified when 166 // forwarding the packet 167 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 168 unsigned int pkt_cmd = pkt->cmdToIndex(); 169 |
166 // set the source port for routing of the response 167 pkt->setSrc(slave_port_id); 168 | |
169 calcPacketTiming(pkt); 170 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 171 172 // uncacheable requests need never be snooped 173 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 174 // the packet is a memory-mapped request and should be 175 // broadcasted to our snoopers but the source 176 if (snoopFilter) { --- 5 unchanged lines hidden (view full) --- 182 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 183 sf_res.second); 184 forwardTiming(pkt, slave_port_id, sf_res.first); 185 } else { 186 forwardTiming(pkt, slave_port_id); 187 } 188 } 189 | 170 calcPacketTiming(pkt); 171 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 172 173 // uncacheable requests need never be snooped 174 if (!pkt->req->isUncacheable() && !system->bypassCaches()) { 175 // the packet is a memory-mapped request and should be 176 // broadcasted to our snoopers but the source 177 if (snoopFilter) { --- 5 unchanged lines hidden (view full) --- 183 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 184 sf_res.second); 185 forwardTiming(pkt, slave_port_id, sf_res.first); 186 } else { 187 forwardTiming(pkt, slave_port_id); 188 } 189 } 190 |
190 // remember if we add an outstanding req so we can undo it if 191 // necessary, if the packet needs a response, we should add it 192 // as outstanding and express snoops never fail so there is 193 // not need to worry about them 194 bool add_outstanding = !is_express_snoop && pkt->needsResponse(); | 191 // remember if the packet will generate a snoop response 192 const bool expect_snoop_resp = !is_inhibited && pkt->memInhibitAsserted(); 193 const bool expect_response = pkt->needsResponse() && 194 !pkt->memInhibitAsserted(); |
195 | 195 |
196 // keep track that we have an outstanding request packet 197 // matching this request, this is used by the coherency 198 // mechanism in determining what to do with snoop responses 199 // (in recvTimingSnoop) 200 if (add_outstanding) { 201 // we should never have an exsiting request outstanding 202 assert(outstandingReq.find(pkt->req) == outstandingReq.end()); 203 outstandingReq.insert(pkt->req); 204 } 205 | |
206 // Note: Cannot create a copy of the full packet, here. 207 MemCmd orig_cmd(pkt->cmd); 208 209 // since it is a normal request, attempt to send the packet 210 bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 211 212 if (snoopFilter && !pkt->req->isUncacheable() 213 && !system->bypassCaches()) { --- 5 unchanged lines hidden (view full) --- 219 // phase tracking. 220 MemCmd tmp_cmd(pkt->cmd); 221 pkt->cmd = orig_cmd; 222 // Let the snoop filter know about the success of the send operation 223 snoopFilter->updateRequest(pkt, *src_port, !success); 224 pkt->cmd = tmp_cmd; 225 } 226 | 196 // Note: Cannot create a copy of the full packet, here. 197 MemCmd orig_cmd(pkt->cmd); 198 199 // since it is a normal request, attempt to send the packet 200 bool success = masterPorts[master_port_id]->sendTimingReq(pkt); 201 202 if (snoopFilter && !pkt->req->isUncacheable() 203 && !system->bypassCaches()) { --- 5 unchanged lines hidden (view full) --- 209 // phase tracking. 210 MemCmd tmp_cmd(pkt->cmd); 211 pkt->cmd = orig_cmd; 212 // Let the snoop filter know about the success of the send operation 213 snoopFilter->updateRequest(pkt, *src_port, !success); 214 pkt->cmd = tmp_cmd; 215 } 216 |
227 // if this is an express snoop, we are done at this point 228 if (is_express_snoop) { 229 assert(success); 230 snoops++; | 217 // check if we were successful in sending the packet onwards 218 if (!success) { 219 // express snoops and inhibited packets should never be forced 220 // to retry 221 assert(!is_express_snoop); 222 assert(!pkt->memInhibitAsserted()); 223 224 // undo the calculation so we can check for 0 again 225 pkt->firstWordDelay = pkt->lastWordDelay = 0; 226 227 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 228 src_port->name(), pkt->cmdString(), pkt->getAddr()); 229 230 // update the layer state and schedule an idle event 231 reqLayers[master_port_id]->failedTiming(src_port, 232 clockEdge(headerCycles)); |
231 } else { | 233 } else { |
232 // for normal requests, check if successful 233 if (!success) { 234 // inhibited packets should never be forced to retry 235 assert(!pkt->memInhibitAsserted()); | 234 // express snoops currently bypass the crossbar state entirely 235 if (!is_express_snoop) { 236 // if this particular request will generate a snoop 237 // response 238 if (expect_snoop_resp) { 239 // we should never have an exsiting request outstanding 240 assert(outstandingSnoop.find(pkt->req) == 241 outstandingSnoop.end()); 242 outstandingSnoop.insert(pkt->req); |
236 | 243 |
237 // if it was added as outstanding and the send failed, then 238 // erase it again 239 if (add_outstanding) 240 outstandingReq.erase(pkt->req); | 244 // basic sanity check on the outstanding snoops 245 panic_if(outstandingSnoop.size() > 512, 246 "Outstanding snoop requests exceeded 512\n"); 247 } |
241 | 248 |
242 // undo the calculation so we can check for 0 again 243 pkt->firstWordDelay = pkt->lastWordDelay = 0; | 249 // remember where to route the normal response to 250 if (expect_response || expect_snoop_resp) { 251 assert(routeTo.find(pkt->req) == routeTo.end()); 252 routeTo[pkt->req] = slave_port_id; |
244 | 253 |
245 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n", 246 src_port->name(), pkt->cmdString(), pkt->getAddr()); | 254 panic_if(routeTo.size() > 512, 255 "Routing table exceeds 512 packets\n"); 256 } |
247 248 // update the layer state and schedule an idle event | 257 258 // update the layer state and schedule an idle event |
249 reqLayers[master_port_id]->failedTiming(src_port, 250 clockEdge(headerCycles)); 251 } else { 252 // update the layer state and schedule an idle event | |
253 reqLayers[master_port_id]->succeededTiming(packetFinishTime); 254 } | 259 reqLayers[master_port_id]->succeededTiming(packetFinishTime); 260 } |
255 } | |
256 | 261 |
257 // stats updates only consider packets that were successfully sent 258 if (success) { | 262 // stats updates only consider packets that were successfully sent |
259 pktCount[slave_port_id][master_port_id]++; 260 pktSize[slave_port_id][master_port_id] += pkt_size; 261 transDist[pkt_cmd]++; | 263 pktCount[slave_port_id][master_port_id]++; 264 pktSize[slave_port_id][master_port_id] += pkt_size; 265 transDist[pkt_cmd]++; |
266 267 if (is_express_snoop) 268 snoops++; |
|
262 } 263 264 return success; 265} 266 267bool 268CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 269{ 270 // determine the source port based on the id 271 MasterPort *src_port = masterPorts[master_port_id]; 272 | 269 } 270 271 return success; 272} 273 274bool 275CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id) 276{ 277 // determine the source port based on the id 278 MasterPort *src_port = masterPorts[master_port_id]; 279 |
273 // determine the destination based on what is stored in the packet 274 PortID slave_port_id = pkt->getDest(); | 280 // determine the destination 281 const auto route_lookup = routeTo.find(pkt->req); 282 assert(route_lookup != routeTo.end()); 283 const PortID slave_port_id = route_lookup->second; |
275 assert(slave_port_id != InvalidPortID); 276 assert(slave_port_id < respLayers.size()); 277 278 // test if the crossbar should be considered occupied for the 279 // current port 280 if (!respLayers[slave_port_id]->tryTiming(src_port)) { 281 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 282 src_port->name(), pkt->cmdString(), pkt->getAddr()); --- 6 unchanged lines hidden (view full) --- 289 // store size and command as they might be modified when 290 // forwarding the packet 291 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 292 unsigned int pkt_cmd = pkt->cmdToIndex(); 293 294 calcPacketTiming(pkt); 295 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 296 | 284 assert(slave_port_id != InvalidPortID); 285 assert(slave_port_id < respLayers.size()); 286 287 // test if the crossbar should be considered occupied for the 288 // current port 289 if (!respLayers[slave_port_id]->tryTiming(src_port)) { 290 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n", 291 src_port->name(), pkt->cmdString(), pkt->getAddr()); --- 6 unchanged lines hidden (view full) --- 298 // store size and command as they might be modified when 299 // forwarding the packet 300 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0; 301 unsigned int pkt_cmd = pkt->cmdToIndex(); 302 303 calcPacketTiming(pkt); 304 Tick packetFinishTime = pkt->lastWordDelay + curTick(); 305 |
297 // the packet is a normal response to a request that we should 298 // have seen passing through the crossbar 299 assert(outstandingReq.find(pkt->req) != outstandingReq.end()); 300 | |
301 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 302 // let the snoop filter inspect the response and update its state 303 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 304 } 305 | 306 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) { 307 // let the snoop filter inspect the response and update its state 308 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]); 309 } 310 |
306 // remove it as outstanding 307 outstandingReq.erase(pkt->req); 308 | |
309 // send the packet through the destination slave port 310 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 311 312 // currently it is illegal to block responses... can lead to 313 // deadlock 314 assert(success); 315 | 311 // send the packet through the destination slave port 312 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt); 313 314 // currently it is illegal to block responses... can lead to 315 // deadlock 316 assert(success); 317 |
318 // remove the request from the routing table 319 routeTo.erase(route_lookup); 320 |
|
316 respLayers[slave_port_id]->succeededTiming(packetFinishTime); 317 318 // stats updates 319 pktCount[slave_port_id][master_port_id]++; 320 pktSize[slave_port_id][master_port_id] += pkt_size; 321 transDist[pkt_cmd]++; 322 323 return true; --- 8 unchanged lines hidden (view full) --- 332 333 // update stats here as we know the forwarding will succeed 334 transDist[pkt->cmdToIndex()]++; 335 snoops++; 336 337 // we should only see express snoops from caches 338 assert(pkt->isExpressSnoop()); 339 | 321 respLayers[slave_port_id]->succeededTiming(packetFinishTime); 322 323 // stats updates 324 pktCount[slave_port_id][master_port_id]++; 325 pktSize[slave_port_id][master_port_id] += pkt_size; 326 transDist[pkt_cmd]++; 327 328 return true; --- 8 unchanged lines hidden (view full) --- 337 338 // update stats here as we know the forwarding will succeed 339 transDist[pkt->cmdToIndex()]++; 340 snoops++; 341 342 // we should only see express snoops from caches 343 assert(pkt->isExpressSnoop()); 344 |
340 // set the source port for routing of the response 341 pkt->setSrc(master_port_id); | 345 // remeber if the packet is inhibited so we can see if it changes 346 const bool is_inhibited = pkt->memInhibitAsserted(); |
342 343 if (snoopFilter) { 344 // let the Snoop Filter work its magic and guide probing 345 auto sf_res = snoopFilter->lookupSnoop(pkt); 346 // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 347 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 348 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 349 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 350 sf_res.second); 351 352 // forward to all snoopers 353 forwardTiming(pkt, InvalidPortID, sf_res.first); 354 } else { 355 forwardTiming(pkt, InvalidPortID); 356 } 357 | 347 348 if (snoopFilter) { 349 // let the Snoop Filter work its magic and guide probing 350 auto sf_res = snoopFilter->lookupSnoop(pkt); 351 // No timing here: packetFinishTime += sf_res.second * clockPeriod(); 352 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\ 353 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(), 354 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(), 355 sf_res.second); 356 357 // forward to all snoopers 358 forwardTiming(pkt, InvalidPortID, sf_res.first); 359 } else { 360 forwardTiming(pkt, InvalidPortID); 361 } 362 |
363 // if we can expect a response, remember how to route it 364 if (!is_inhibited && pkt->memInhibitAsserted()) { 365 assert(routeTo.find(pkt->req) == routeTo.end()); 366 routeTo[pkt->req] = master_port_id; 367 } 368 |
|
358 // a snoop request came from a connected slave device (one of 359 // our master ports), and if it is not coming from the slave 360 // device responsible for the address range something is 361 // wrong, hence there is nothing further to do as the packet 362 // would be going back to where it came from 363 assert(master_port_id == findPort(pkt->getAddr())); 364} 365 366bool 367CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 368{ 369 // determine the source port based on the id 370 SlavePort* src_port = slavePorts[slave_port_id]; 371 | 369 // a snoop request came from a connected slave device (one of 370 // our master ports), and if it is not coming from the slave 371 // device responsible for the address range something is 372 // wrong, hence there is nothing further to do as the packet 373 // would be going back to where it came from 374 assert(master_port_id == findPort(pkt->getAddr())); 375} 376 377bool 378CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id) 379{ 380 // determine the source port based on the id 381 SlavePort* src_port = slavePorts[slave_port_id]; 382 |
372 // get the destination from the packet 373 PortID dest_port_id = pkt->getDest(); | 383 // get the destination 384 const auto route_lookup = routeTo.find(pkt->req); 385 assert(route_lookup != routeTo.end()); 386 const PortID dest_port_id = route_lookup->second; |
374 assert(dest_port_id != InvalidPortID); 375 376 // determine if the response is from a snoop request we 377 // created as the result of a normal request (in which case it | 387 assert(dest_port_id != InvalidPortID); 388 389 // determine if the response is from a snoop request we 390 // created as the result of a normal request (in which case it |
378 // should be in the outstandingReq), or if we merely forwarded | 391 // should be in the outstandingSnoop), or if we merely forwarded |
379 // someone else's snoop request | 392 // someone else's snoop request |
380 bool forwardAsSnoop = outstandingReq.find(pkt->req) == 381 outstandingReq.end(); | 393 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) == 394 outstandingSnoop.end(); |
382 383 // test if the crossbar should be considered occupied for the 384 // current port, note that the check is bypassed if the response 385 // is being passed on as a normal response since this is occupying 386 // the response layer rather than the snoop response layer 387 if (forwardAsSnoop) { 388 assert(dest_port_id < snoopLayers.size()); 389 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { --- 45 unchanged lines hidden (view full) --- 435 assert(success); 436 437 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 438 } else { 439 // we got a snoop response on one of our slave ports, 440 // i.e. from a coherent master connected to the crossbar, and 441 // since we created the snoop request as part of recvTiming, 442 // this should now be a normal response again | 395 396 // test if the crossbar should be considered occupied for the 397 // current port, note that the check is bypassed if the response 398 // is being passed on as a normal response since this is occupying 399 // the response layer rather than the snoop response layer 400 if (forwardAsSnoop) { 401 assert(dest_port_id < snoopLayers.size()); 402 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) { --- 45 unchanged lines hidden (view full) --- 448 assert(success); 449 450 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime); 451 } else { 452 // we got a snoop response on one of our slave ports, 453 // i.e. from a coherent master connected to the crossbar, and 454 // since we created the snoop request as part of recvTiming, 455 // this should now be a normal response again |
443 outstandingReq.erase(pkt->req); | 456 outstandingSnoop.erase(pkt->req); |
444 | 457 |
445 // this is a snoop response from a coherent master, with a 446 // destination field set on its way through the crossbar as 447 // request, hence it should never go back to where the snoop 448 // response came from, but instead to where the original 449 // request came from | 458 // this is a snoop response from a coherent master, hence it 459 // should never go back to where the snoop response came from, 460 // but instead to where the original request came from |
450 assert(slave_port_id != dest_port_id); 451 452 if (snoopFilter) { 453 // update the probe filter so that it can properly track the line 454 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 455 *slavePorts[dest_port_id]); 456 } 457 --- 13 unchanged lines hidden (view full) --- 471 472 // currently it is illegal to block responses... can lead 473 // to deadlock 474 assert(success); 475 476 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 477 } 478 | 461 assert(slave_port_id != dest_port_id); 462 463 if (snoopFilter) { 464 // update the probe filter so that it can properly track the line 465 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id], 466 *slavePorts[dest_port_id]); 467 } 468 --- 13 unchanged lines hidden (view full) --- 482 483 // currently it is illegal to block responses... can lead 484 // to deadlock 485 assert(success); 486 487 respLayers[dest_port_id]->succeededTiming(packetFinishTime); 488 } 489 |
490 // remove the request from the routing table 491 routeTo.erase(route_lookup); 492 |
|
479 // stats updates 480 transDist[pkt_cmd]++; 481 snoops++; 482 483 return true; 484} 485 486 --- 332 unchanged lines hidden --- | 493 // stats updates 494 transDist[pkt_cmd]++; 495 snoops++; 496 497 return true; 498} 499 500 --- 332 unchanged lines hidden --- |