coherent_xbar.cc (10572:fc4c90a7d2f5) coherent_xbar.cc (10656:bd376adfb7d4)
1/*
2 * Copyright (c) 2011-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 128 unchanged lines hidden (view full) ---

137bool
138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
139{
140 // determine the source port based on the id
141 SlavePort *src_port = slavePorts[slave_port_id];
142
143 // remember if the packet is an express snoop
144 bool is_express_snoop = pkt->isExpressSnoop();
1/*
2 * Copyright (c) 2011-2014 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software

--- 128 unchanged lines hidden (view full) ---

137bool
138CoherentXBar::recvTimingReq(PacketPtr pkt, PortID slave_port_id)
139{
140 // determine the source port based on the id
141 SlavePort *src_port = slavePorts[slave_port_id];
142
143 // remember if the packet is an express snoop
144 bool is_express_snoop = pkt->isExpressSnoop();
145 bool is_inhibited = pkt->memInhibitAsserted();
146 // for normal requests, going downstream, the express snoop flag
147 // and the inhibited flag should always be the same
148 assert(is_express_snoop == is_inhibited);
145
146 // determine the destination based on the address
147 PortID master_port_id = findPort(pkt->getAddr());
148
149 // test if the crossbar should be considered occupied for the current
150 // port, and exclude express snoops from the check
151 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) {
152 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n",

--- 5 unchanged lines hidden (view full) ---

158 src_port->name(), pkt->cmdString(), is_express_snoop,
159 pkt->getAddr());
160
161 // store size and command as they might be modified when
162 // forwarding the packet
163 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
164 unsigned int pkt_cmd = pkt->cmdToIndex();
165
149
150 // determine the destination based on the address
151 PortID master_port_id = findPort(pkt->getAddr());
152
153 // test if the crossbar should be considered occupied for the current
154 // port, and exclude express snoops from the check
155 if (!is_express_snoop && !reqLayers[master_port_id]->tryTiming(src_port)) {
156 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x BUSY\n",

--- 5 unchanged lines hidden (view full) ---

162 src_port->name(), pkt->cmdString(), is_express_snoop,
163 pkt->getAddr());
164
165 // store size and command as they might be modified when
166 // forwarding the packet
167 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
168 unsigned int pkt_cmd = pkt->cmdToIndex();
169
166 // set the source port for routing of the response
167 pkt->setSrc(slave_port_id);
168
169 calcPacketTiming(pkt);
170 Tick packetFinishTime = pkt->lastWordDelay + curTick();
171
172 // uncacheable requests need never be snooped
173 if (!pkt->req->isUncacheable() && !system->bypassCaches()) {
174 // the packet is a memory-mapped request and should be
175 // broadcasted to our snoopers but the source
176 if (snoopFilter) {

--- 5 unchanged lines hidden (view full) ---

182 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
183 sf_res.second);
184 forwardTiming(pkt, slave_port_id, sf_res.first);
185 } else {
186 forwardTiming(pkt, slave_port_id);
187 }
188 }
189
170 calcPacketTiming(pkt);
171 Tick packetFinishTime = pkt->lastWordDelay + curTick();
172
173 // uncacheable requests need never be snooped
174 if (!pkt->req->isUncacheable() && !system->bypassCaches()) {
175 // the packet is a memory-mapped request and should be
176 // broadcasted to our snoopers but the source
177 if (snoopFilter) {

--- 5 unchanged lines hidden (view full) ---

183 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
184 sf_res.second);
185 forwardTiming(pkt, slave_port_id, sf_res.first);
186 } else {
187 forwardTiming(pkt, slave_port_id);
188 }
189 }
190
190 // remember if we add an outstanding req so we can undo it if
191 // necessary, if the packet needs a response, we should add it
192 // as outstanding and express snoops never fail so there is
193 // not need to worry about them
194 bool add_outstanding = !is_express_snoop && pkt->needsResponse();
191 // remember if the packet will generate a snoop response
192 const bool expect_snoop_resp = !is_inhibited && pkt->memInhibitAsserted();
193 const bool expect_response = pkt->needsResponse() &&
194 !pkt->memInhibitAsserted();
195
195
196 // keep track that we have an outstanding request packet
197 // matching this request, this is used by the coherency
198 // mechanism in determining what to do with snoop responses
199 // (in recvTimingSnoop)
200 if (add_outstanding) {
201 // we should never have an exsiting request outstanding
202 assert(outstandingReq.find(pkt->req) == outstandingReq.end());
203 outstandingReq.insert(pkt->req);
204 }
205
206 // Note: Cannot create a copy of the full packet, here.
207 MemCmd orig_cmd(pkt->cmd);
208
209 // since it is a normal request, attempt to send the packet
210 bool success = masterPorts[master_port_id]->sendTimingReq(pkt);
211
212 if (snoopFilter && !pkt->req->isUncacheable()
213 && !system->bypassCaches()) {

--- 5 unchanged lines hidden (view full) ---

219 // phase tracking.
220 MemCmd tmp_cmd(pkt->cmd);
221 pkt->cmd = orig_cmd;
222 // Let the snoop filter know about the success of the send operation
223 snoopFilter->updateRequest(pkt, *src_port, !success);
224 pkt->cmd = tmp_cmd;
225 }
226
196 // Note: Cannot create a copy of the full packet, here.
197 MemCmd orig_cmd(pkt->cmd);
198
199 // since it is a normal request, attempt to send the packet
200 bool success = masterPorts[master_port_id]->sendTimingReq(pkt);
201
202 if (snoopFilter && !pkt->req->isUncacheable()
203 && !system->bypassCaches()) {

--- 5 unchanged lines hidden (view full) ---

209 // phase tracking.
210 MemCmd tmp_cmd(pkt->cmd);
211 pkt->cmd = orig_cmd;
212 // Let the snoop filter know about the success of the send operation
213 snoopFilter->updateRequest(pkt, *src_port, !success);
214 pkt->cmd = tmp_cmd;
215 }
216
227 // if this is an express snoop, we are done at this point
228 if (is_express_snoop) {
229 assert(success);
230 snoops++;
217 // check if we were successful in sending the packet onwards
218 if (!success) {
219 // express snoops and inhibited packets should never be forced
220 // to retry
221 assert(!is_express_snoop);
222 assert(!pkt->memInhibitAsserted());
223
224 // undo the calculation so we can check for 0 again
225 pkt->firstWordDelay = pkt->lastWordDelay = 0;
226
227 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n",
228 src_port->name(), pkt->cmdString(), pkt->getAddr());
229
230 // update the layer state and schedule an idle event
231 reqLayers[master_port_id]->failedTiming(src_port,
232 clockEdge(headerCycles));
231 } else {
233 } else {
232 // for normal requests, check if successful
233 if (!success) {
234 // inhibited packets should never be forced to retry
235 assert(!pkt->memInhibitAsserted());
234 // express snoops currently bypass the crossbar state entirely
235 if (!is_express_snoop) {
236 // if this particular request will generate a snoop
237 // response
238 if (expect_snoop_resp) {
239 // we should never have an exsiting request outstanding
240 assert(outstandingSnoop.find(pkt->req) ==
241 outstandingSnoop.end());
242 outstandingSnoop.insert(pkt->req);
236
243
237 // if it was added as outstanding and the send failed, then
238 // erase it again
239 if (add_outstanding)
240 outstandingReq.erase(pkt->req);
244 // basic sanity check on the outstanding snoops
245 panic_if(outstandingSnoop.size() > 512,
246 "Outstanding snoop requests exceeded 512\n");
247 }
241
248
242 // undo the calculation so we can check for 0 again
243 pkt->firstWordDelay = pkt->lastWordDelay = 0;
249 // remember where to route the normal response to
250 if (expect_response || expect_snoop_resp) {
251 assert(routeTo.find(pkt->req) == routeTo.end());
252 routeTo[pkt->req] = slave_port_id;
244
253
245 DPRINTF(CoherentXBar, "recvTimingReq: src %s %s 0x%x RETRY\n",
246 src_port->name(), pkt->cmdString(), pkt->getAddr());
254 panic_if(routeTo.size() > 512,
255 "Routing table exceeds 512 packets\n");
256 }
247
248 // update the layer state and schedule an idle event
257
258 // update the layer state and schedule an idle event
249 reqLayers[master_port_id]->failedTiming(src_port,
250 clockEdge(headerCycles));
251 } else {
252 // update the layer state and schedule an idle event
253 reqLayers[master_port_id]->succeededTiming(packetFinishTime);
254 }
259 reqLayers[master_port_id]->succeededTiming(packetFinishTime);
260 }
255 }
256
261
257 // stats updates only consider packets that were successfully sent
258 if (success) {
262 // stats updates only consider packets that were successfully sent
259 pktCount[slave_port_id][master_port_id]++;
260 pktSize[slave_port_id][master_port_id] += pkt_size;
261 transDist[pkt_cmd]++;
263 pktCount[slave_port_id][master_port_id]++;
264 pktSize[slave_port_id][master_port_id] += pkt_size;
265 transDist[pkt_cmd]++;
266
267 if (is_express_snoop)
268 snoops++;
262 }
263
264 return success;
265}
266
267bool
268CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
269{
270 // determine the source port based on the id
271 MasterPort *src_port = masterPorts[master_port_id];
272
269 }
270
271 return success;
272}
273
274bool
275CoherentXBar::recvTimingResp(PacketPtr pkt, PortID master_port_id)
276{
277 // determine the source port based on the id
278 MasterPort *src_port = masterPorts[master_port_id];
279
273 // determine the destination based on what is stored in the packet
274 PortID slave_port_id = pkt->getDest();
280 // determine the destination
281 const auto route_lookup = routeTo.find(pkt->req);
282 assert(route_lookup != routeTo.end());
283 const PortID slave_port_id = route_lookup->second;
275 assert(slave_port_id != InvalidPortID);
276 assert(slave_port_id < respLayers.size());
277
278 // test if the crossbar should be considered occupied for the
279 // current port
280 if (!respLayers[slave_port_id]->tryTiming(src_port)) {
281 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n",
282 src_port->name(), pkt->cmdString(), pkt->getAddr());

--- 6 unchanged lines hidden (view full) ---

289 // store size and command as they might be modified when
290 // forwarding the packet
291 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
292 unsigned int pkt_cmd = pkt->cmdToIndex();
293
294 calcPacketTiming(pkt);
295 Tick packetFinishTime = pkt->lastWordDelay + curTick();
296
284 assert(slave_port_id != InvalidPortID);
285 assert(slave_port_id < respLayers.size());
286
287 // test if the crossbar should be considered occupied for the
288 // current port
289 if (!respLayers[slave_port_id]->tryTiming(src_port)) {
290 DPRINTF(CoherentXBar, "recvTimingResp: src %s %s 0x%x BUSY\n",
291 src_port->name(), pkt->cmdString(), pkt->getAddr());

--- 6 unchanged lines hidden (view full) ---

298 // store size and command as they might be modified when
299 // forwarding the packet
300 unsigned int pkt_size = pkt->hasData() ? pkt->getSize() : 0;
301 unsigned int pkt_cmd = pkt->cmdToIndex();
302
303 calcPacketTiming(pkt);
304 Tick packetFinishTime = pkt->lastWordDelay + curTick();
305
297 // the packet is a normal response to a request that we should
298 // have seen passing through the crossbar
299 assert(outstandingReq.find(pkt->req) != outstandingReq.end());
300
301 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) {
302 // let the snoop filter inspect the response and update its state
303 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
304 }
305
306 if (snoopFilter && !pkt->req->isUncacheable() && !system->bypassCaches()) {
307 // let the snoop filter inspect the response and update its state
308 snoopFilter->updateResponse(pkt, *slavePorts[slave_port_id]);
309 }
310
306 // remove it as outstanding
307 outstandingReq.erase(pkt->req);
308
309 // send the packet through the destination slave port
310 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt);
311
312 // currently it is illegal to block responses... can lead to
313 // deadlock
314 assert(success);
315
311 // send the packet through the destination slave port
312 bool success M5_VAR_USED = slavePorts[slave_port_id]->sendTimingResp(pkt);
313
314 // currently it is illegal to block responses... can lead to
315 // deadlock
316 assert(success);
317
318 // remove the request from the routing table
319 routeTo.erase(route_lookup);
320
316 respLayers[slave_port_id]->succeededTiming(packetFinishTime);
317
318 // stats updates
319 pktCount[slave_port_id][master_port_id]++;
320 pktSize[slave_port_id][master_port_id] += pkt_size;
321 transDist[pkt_cmd]++;
322
323 return true;

--- 8 unchanged lines hidden (view full) ---

332
333 // update stats here as we know the forwarding will succeed
334 transDist[pkt->cmdToIndex()]++;
335 snoops++;
336
337 // we should only see express snoops from caches
338 assert(pkt->isExpressSnoop());
339
321 respLayers[slave_port_id]->succeededTiming(packetFinishTime);
322
323 // stats updates
324 pktCount[slave_port_id][master_port_id]++;
325 pktSize[slave_port_id][master_port_id] += pkt_size;
326 transDist[pkt_cmd]++;
327
328 return true;

--- 8 unchanged lines hidden (view full) ---

337
338 // update stats here as we know the forwarding will succeed
339 transDist[pkt->cmdToIndex()]++;
340 snoops++;
341
342 // we should only see express snoops from caches
343 assert(pkt->isExpressSnoop());
344
340 // set the source port for routing of the response
341 pkt->setSrc(master_port_id);
345 // remeber if the packet is inhibited so we can see if it changes
346 const bool is_inhibited = pkt->memInhibitAsserted();
342
343 if (snoopFilter) {
344 // let the Snoop Filter work its magic and guide probing
345 auto sf_res = snoopFilter->lookupSnoop(pkt);
346 // No timing here: packetFinishTime += sf_res.second * clockPeriod();
347 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\
348 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(),
349 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
350 sf_res.second);
351
352 // forward to all snoopers
353 forwardTiming(pkt, InvalidPortID, sf_res.first);
354 } else {
355 forwardTiming(pkt, InvalidPortID);
356 }
357
347
348 if (snoopFilter) {
349 // let the Snoop Filter work its magic and guide probing
350 auto sf_res = snoopFilter->lookupSnoop(pkt);
351 // No timing here: packetFinishTime += sf_res.second * clockPeriod();
352 DPRINTF(CoherentXBar, "recvTimingSnoopReq: src %s %s 0x%x"\
353 " SF size: %i lat: %i\n", masterPorts[master_port_id]->name(),
354 pkt->cmdString(), pkt->getAddr(), sf_res.first.size(),
355 sf_res.second);
356
357 // forward to all snoopers
358 forwardTiming(pkt, InvalidPortID, sf_res.first);
359 } else {
360 forwardTiming(pkt, InvalidPortID);
361 }
362
363 // if we can expect a response, remember how to route it
364 if (!is_inhibited && pkt->memInhibitAsserted()) {
365 assert(routeTo.find(pkt->req) == routeTo.end());
366 routeTo[pkt->req] = master_port_id;
367 }
368
358 // a snoop request came from a connected slave device (one of
359 // our master ports), and if it is not coming from the slave
360 // device responsible for the address range something is
361 // wrong, hence there is nothing further to do as the packet
362 // would be going back to where it came from
363 assert(master_port_id == findPort(pkt->getAddr()));
364}
365
366bool
367CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
368{
369 // determine the source port based on the id
370 SlavePort* src_port = slavePorts[slave_port_id];
371
369 // a snoop request came from a connected slave device (one of
370 // our master ports), and if it is not coming from the slave
371 // device responsible for the address range something is
372 // wrong, hence there is nothing further to do as the packet
373 // would be going back to where it came from
374 assert(master_port_id == findPort(pkt->getAddr()));
375}
376
377bool
378CoherentXBar::recvTimingSnoopResp(PacketPtr pkt, PortID slave_port_id)
379{
380 // determine the source port based on the id
381 SlavePort* src_port = slavePorts[slave_port_id];
382
372 // get the destination from the packet
373 PortID dest_port_id = pkt->getDest();
383 // get the destination
384 const auto route_lookup = routeTo.find(pkt->req);
385 assert(route_lookup != routeTo.end());
386 const PortID dest_port_id = route_lookup->second;
374 assert(dest_port_id != InvalidPortID);
375
376 // determine if the response is from a snoop request we
377 // created as the result of a normal request (in which case it
387 assert(dest_port_id != InvalidPortID);
388
389 // determine if the response is from a snoop request we
390 // created as the result of a normal request (in which case it
378 // should be in the outstandingReq), or if we merely forwarded
391 // should be in the outstandingSnoop), or if we merely forwarded
379 // someone else's snoop request
392 // someone else's snoop request
380 bool forwardAsSnoop = outstandingReq.find(pkt->req) ==
381 outstandingReq.end();
393 const bool forwardAsSnoop = outstandingSnoop.find(pkt->req) ==
394 outstandingSnoop.end();
382
383 // test if the crossbar should be considered occupied for the
384 // current port, note that the check is bypassed if the response
385 // is being passed on as a normal response since this is occupying
386 // the response layer rather than the snoop response layer
387 if (forwardAsSnoop) {
388 assert(dest_port_id < snoopLayers.size());
389 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) {

--- 45 unchanged lines hidden (view full) ---

435 assert(success);
436
437 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime);
438 } else {
439 // we got a snoop response on one of our slave ports,
440 // i.e. from a coherent master connected to the crossbar, and
441 // since we created the snoop request as part of recvTiming,
442 // this should now be a normal response again
395
396 // test if the crossbar should be considered occupied for the
397 // current port, note that the check is bypassed if the response
398 // is being passed on as a normal response since this is occupying
399 // the response layer rather than the snoop response layer
400 if (forwardAsSnoop) {
401 assert(dest_port_id < snoopLayers.size());
402 if (!snoopLayers[dest_port_id]->tryTiming(src_port)) {

--- 45 unchanged lines hidden (view full) ---

448 assert(success);
449
450 snoopLayers[dest_port_id]->succeededTiming(packetFinishTime);
451 } else {
452 // we got a snoop response on one of our slave ports,
453 // i.e. from a coherent master connected to the crossbar, and
454 // since we created the snoop request as part of recvTiming,
455 // this should now be a normal response again
443 outstandingReq.erase(pkt->req);
456 outstandingSnoop.erase(pkt->req);
444
457
445 // this is a snoop response from a coherent master, with a
446 // destination field set on its way through the crossbar as
447 // request, hence it should never go back to where the snoop
448 // response came from, but instead to where the original
449 // request came from
458 // this is a snoop response from a coherent master, hence it
459 // should never go back to where the snoop response came from,
460 // but instead to where the original request came from
450 assert(slave_port_id != dest_port_id);
451
452 if (snoopFilter) {
453 // update the probe filter so that it can properly track the line
454 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id],
455 *slavePorts[dest_port_id]);
456 }
457

--- 13 unchanged lines hidden (view full) ---

471
472 // currently it is illegal to block responses... can lead
473 // to deadlock
474 assert(success);
475
476 respLayers[dest_port_id]->succeededTiming(packetFinishTime);
477 }
478
461 assert(slave_port_id != dest_port_id);
462
463 if (snoopFilter) {
464 // update the probe filter so that it can properly track the line
465 snoopFilter->updateSnoopResponse(pkt, *slavePorts[slave_port_id],
466 *slavePorts[dest_port_id]);
467 }
468

--- 13 unchanged lines hidden (view full) ---

482
483 // currently it is illegal to block responses... can lead
484 // to deadlock
485 assert(success);
486
487 respLayers[dest_port_id]->succeededTiming(packetFinishTime);
488 }
489
490 // remove the request from the routing table
491 routeTo.erase(route_lookup);
492
479 // stats updates
480 transDist[pkt_cmd]++;
481 snoops++;
482
483 return true;
484}
485
486

--- 332 unchanged lines hidden --- 		493 // stats updates
494 transDist[pkt_cmd]++;
495 snoops++;
496
497 return true;
498}
499
500

--- 332 unchanged lines hidden ---