78 lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
79 globalSeqNum(0), wavefrontSize(p->wfSize),
80 kernelLaunchInst(new KernelLaunchStaticInst())
81{
82 /**
83 * This check is necessary because std::bitset only provides conversion
84 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
85 * there are * a few places in the code where to_ullong() is used, however
86 * if VSZ is larger than a value the host can support then bitset will
87 * throw a runtime exception. we should remove all use of to_long() or
88 * to_ullong() so we can have VSZ greater than 64b, however until that is
89 * done this assert is required.
90 */
91 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
92 p->wfSize <= 0,
93 "WF size is larger than the host can support");
94 fatal_if(!isPowerOf2(wavefrontSize),
95 "Wavefront size should be a power of 2");
96 // calculate how many cycles a vector load or store will need to transfer
97 // its data over the corresponding buses
98 numCyclesPerStoreTransfer =
99 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
100 (double)vrfToCoalescerBusWidth);
101
102 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
103 / coalescerToVrfBusWidth;
104
105 lastVaddrWF.resize(numSIMDs);
106 wfList.resize(numSIMDs);
107
108 for (int j = 0; j < numSIMDs; ++j) {
109 lastVaddrWF[j].resize(p->n_wf);
110
111 for (int i = 0; i < p->n_wf; ++i) {
112 lastVaddrWF[j][i].resize(wfSize());
113
114 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
115 wfList[j][i]->setParent(this);
116
117 for (int k = 0; k < wfSize(); ++k) {
118 lastVaddrWF[j][i][k] = 0;
119 }
120 }
121 }
122
123 lastVaddrSimd.resize(numSIMDs);
124
125 for (int i = 0; i < numSIMDs; ++i) {
126 lastVaddrSimd[i].resize(wfSize(), 0);
127 }
128
129 lastVaddrCU.resize(wfSize());
130
131 lds.setParent(this);
132
133 if (p->execPolicy == "OLDEST-FIRST") {
134 exec_policy = EXEC_POLICY::OLDEST;
135 } else if (p->execPolicy == "ROUND-ROBIN") {
136 exec_policy = EXEC_POLICY::RR;
137 } else {
138 fatal("Invalid WF execution policy (CU)\n");
139 }
140
141 memPort.resize(wfSize());
142
143 // resize the tlbPort vectorArray
144 int tlbPort_width = perLaneTLB ? wfSize() : 1;
145 tlbPort.resize(tlbPort_width);
146
147 cuExitCallback = new CUExitCallback(this);
148 registerExitCallback(cuExitCallback);
149
150 xactCasLoadMap.clear();
151 lastExecCycle.resize(numSIMDs, 0);
152
153 for (int i = 0; i < vrf.size(); ++i) {
154 vrf[i]->setParent(this);
155 }
156
157 numVecRegsPerSimd = vrf[0]->numRegs();
158}
159
160ComputeUnit::~ComputeUnit()
161{
162 // Delete wavefront slots
163 for (int j = 0; j < numSIMDs; ++j) {
164 for (int i = 0; i < shader->n_wf; ++i) {
165 delete wfList[j][i];
166 }
167 lastVaddrSimd[j].clear();
168 }
169 lastVaddrCU.clear();
170 readyList.clear();
171 waveStatusList.clear();
172 dispatchList.clear();
173 vectorAluInstAvail.clear();
174 delete cuExitCallback;
175 delete ldsPort;
176}
177
178void
179ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
180{
181 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
182
183 w->workGroupSz[0] = ndr->q.wgSize[0];
184 w->workGroupSz[1] = ndr->q.wgSize[1];
185 w->workGroupSz[2] = ndr->q.wgSize[2];
186 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
187 w->gridSz[0] = ndr->q.gdSize[0];
188 w->gridSz[1] = ndr->q.gdSize[1];
189 w->gridSz[2] = ndr->q.gdSize[2];
190 w->kernelArgs = ndr->q.args;
191 w->privSizePerItem = ndr->q.privMemPerItem;
192 w->spillSizePerItem = ndr->q.spillMemPerItem;
193 w->roBase = ndr->q.roMemStart;
194 w->roSize = ndr->q.roMemTotal;
195 w->computeActualWgSz(ndr);
196}
197
198void
199ComputeUnit::updateEvents() {
200
201 if (!timestampVec.empty()) {
202 uint32_t vecSize = timestampVec.size();
203 uint32_t i = 0;
204 while (i < vecSize) {
205 if (timestampVec[i] <= shader->tick_cnt) {
206 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
207 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
208 statusVec[i]);
209 timestampVec.erase(timestampVec.begin() + i);
210 regIdxVec.erase(regIdxVec.begin() + i);
211 statusVec.erase(statusVec.begin() + i);
212 --vecSize;
213 --i;
214 }
215 ++i;
216 }
217 }
218
219 for (int i = 0; i< numSIMDs; ++i) {
220 vrf[i]->updateEvents();
221 }
222}
223
224
225void
226ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
227 NDRange *ndr)
228{
229 static int _n_wave = 0;
230
231 VectorMask init_mask;
232 init_mask.reset();
233
234 for (int k = 0; k < wfSize(); ++k) {
235 if (k + waveId * wfSize() < w->actualWgSzTotal)
236 init_mask[k] = 1;
237 }
238
239 w->kernId = ndr->dispatchId;
240 w->wfId = waveId;
241 w->initMask = init_mask.to_ullong();
242
243 for (int k = 0; k < wfSize(); ++k) {
244 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
245 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
246 w->actualWgSz[1];
247 w->workItemId[2][k] = (k + waveId * wfSize()) /
248 (w->actualWgSz[0] * w->actualWgSz[1]);
249
250 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
251 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
252 w->workItemId[0][k];
253 }
254
255 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
256
257 w->barCnt.resize(wfSize(), 0);
258
259 w->maxBarCnt = 0;
260 w->oldBarrierCnt = 0;
261 w->barrierCnt = 0;
262
263 w->privBase = ndr->q.privMemStart;
264 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
265
266 w->spillBase = ndr->q.spillMemStart;
267 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
268
269 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
270
271 // WG state
272 w->wgId = ndr->globalWgId;
273 w->dispatchId = ndr->dispatchId;
274 w->workGroupId[0] = w->wgId % ndr->numWg[0];
275 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
276 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
277
278 w->barrierId = barrier_id;
279 w->stalledAtBarrier = false;
280
281 // set the wavefront context to have a pointer to this section of the LDS
282 w->ldsChunk = ldsChunk;
283
284 int32_t refCount M5_VAR_USED =
285 lds.increaseRefCounter(w->dispatchId, w->wgId);
286 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
287 cu_id, w->wgId, refCount);
288
289 w->instructionBuffer.clear();
290
291 if (w->pendingFetch)
292 w->dropFetch = true;
293
294 // is this the last wavefront in the workgroup
295 // if set the spillWidth to be the remaining work-items
296 // so that the vector access is correct
297 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
298 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
299 } else {
300 w->spillWidth = wfSize();
301 }
302
303 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
304 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
305
306 w->start(++_n_wave, ndr->q.code_ptr);
307}
308
309void
310ComputeUnit::StartWorkgroup(NDRange *ndr)
311{
312 // reserve the LDS capacity allocated to the work group
313 // disambiguated by the dispatch ID and workgroup ID, which should be
314 // globally unique
315 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
316 ndr->q.ldsSize);
317
318 // Send L1 cache acquire
319 // isKernel + isAcquire = Kernel Begin
320 if (shader->impl_kern_boundary_sync) {
321 GPUDynInstPtr gpuDynInst =
322 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
323 getAndIncSeqNum());
324
325 gpuDynInst->useContinuation = false;
326 injectGlobalMemFence(gpuDynInst, true);
327 }
328
329 // calculate the number of 32-bit vector registers required by wavefront
330 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
331 int wave_id = 0;
332
333 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
334 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
335 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
336 // Check if this wavefront slot is available:
337 // It must be stopped and not waiting
338 // for a release to complete S_RETURNING
339 if (w->status == Wavefront::S_STOPPED) {
340 fillKernelState(w, ndr);
341 // if we have scheduled all work items then stop
342 // scheduling wavefronts
343 if (wave_id * wfSize() >= w->actualWgSzTotal)
344 break;
345
346 // reserve vector registers for the scheduled wavefront
347 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
348 uint32_t normSize = 0;
349
350 w->startVgprIndex = vrf[m % numSIMDs]->manager->
351 allocateRegion(vregDemand, &normSize);
352
353 w->reservedVectorRegs = normSize;
354 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
355
356 startWavefront(w, wave_id, ldsChunk, ndr);
357 ++wave_id;
358 }
359 }
360 ++barrier_id;
361}
362
363int
364ComputeUnit::ReadyWorkgroup(NDRange *ndr)
365{
366 // Get true size of workgroup (after clamping to grid size)
367 int trueWgSize[3];
368 int trueWgSizeTotal = 1;
369
370 for (int d = 0; d < 3; ++d) {
371 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
372 ndr->wgId[d] * ndr->q.wgSize[d]);
373
374 trueWgSizeTotal *= trueWgSize[d];
375 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
376 }
377
378 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
379
380 // calculate the number of 32-bit vector registers required by each
381 // work item of the work group
382 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
383 bool vregAvail = true;
384 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
385 int freeWfSlots = 0;
386 // check if the total number of VGPRs required by all WFs of the WG
387 // fit in the VRFs of all SIMD units
388 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
389 int numMappedWfs = 0;
390 std::vector<int> numWfsPerSimd;
391 numWfsPerSimd.resize(numSIMDs, 0);
392 // find how many free WF slots we have across all SIMDs
393 for (int j = 0; j < shader->n_wf; ++j) {
394 for (int i = 0; i < numSIMDs; ++i) {
395 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
396 // count the number of free WF slots
397 ++freeWfSlots;
398 if (numMappedWfs < numWfs) {
399 // count the WFs to be assigned per SIMD
400 numWfsPerSimd[i]++;
401 }
402 numMappedWfs++;
403 }
404 }
405 }
406
407 // if there are enough free WF slots then find if there are enough
408 // free VGPRs per SIMD based on the WF->SIMD mapping
409 if (freeWfSlots >= numWfs) {
410 for (int j = 0; j < numSIMDs; ++j) {
411 // find if there are enough free VGPR regions in the SIMD's VRF
412 // to accommodate the WFs of the new WG that would be mapped to
413 // this SIMD unit
414 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
415 vregDemandPerWI);
416
417 // stop searching if there is at least one SIMD
418 // whose VRF does not have enough free VGPR pools.
419 // This is because a WG is scheduled only if ALL
420 // of its WFs can be scheduled
421 if (!vregAvail)
422 break;
423 }
424 }
425
426 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
427 freeWfSlots, vregAvail);
428
429 if (!vregAvail) {
430 ++numTimesWgBlockedDueVgprAlloc;
431 }
432
433 // Return true if enough WF slots to submit workgroup and if there are
434 // enough VGPRs to schedule all WFs to their SIMD units
435 if (!lds.canReserve(ndr->q.ldsSize)) {
436 wgBlockedDueLdsAllocation++;
437 }
438
439 // Return true if (a) there are enough free WF slots to submit
440 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
441 // SIMD units and (c) if there is enough space in LDS
442 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
443}
444
445int
446ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
447{
448 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
449 int ccnt = 0;
450
451 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
452 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
453 Wavefront *w = wfList[i_simd][i_wf];
454
455 if (w->status == Wavefront::S_RUNNING) {
456 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
457
458 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
459 w->barrierId, _barrier_id);
460
461 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
462 w->barrierCnt, bcnt);
463 }
464
465 if (w->status == Wavefront::S_RUNNING &&
466 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
467 !w->outstandingReqs) {
468 ++ccnt;
469
470 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
471 "%d\n", i_simd, i_wf, ccnt);
472 }
473 }
474 }
475
476 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
477 cu_id, ccnt, bslots);
478
479 return ccnt == bslots;
480}
481
482// Check if the current wavefront is blocked on additional resources.
483bool
484ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
485{
486 bool cede = false;
487
488 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
489 // magic instructions will impact the scheduling of wavefronts
490 if (xact_cas_mode) {
491 /*
492 * When a wavefront calls xact_cas_ld, it adds itself to a per address
493 * queue. All per address queues are managed by the xactCasLoadMap.
494 *
495 * A wavefront is not blocked if: it is not in ANY per address queue or
496 * if it is at the head of a per address queue.
497 */
498 for (auto itMap : xactCasLoadMap) {
499 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
500
501 if (!curWaveIDQueue.empty()) {
502 for (auto it : curWaveIDQueue) {
503 waveIdentifier cur_wave = it;
504
505 if (cur_wave.simdId == simdId &&
506 cur_wave.wfSlotId == wfSlotId) {
507 // 2 possibilities
508 // 1: this WF has a green light
509 // 2: another WF has a green light
510 waveIdentifier owner_wave = curWaveIDQueue.front();
511
512 if (owner_wave.simdId != cur_wave.simdId ||
513 owner_wave.wfSlotId != cur_wave.wfSlotId) {
514 // possibility 2
515 cede = true;
516 break;
517 } else {
518 // possibility 1
519 break;
520 }
521 }
522 }
523 }
524 }
525 }
526
527 return cede;
528}
529
530// Execute one clock worth of work on the ComputeUnit.
531void
532ComputeUnit::exec()
533{
534 updateEvents();
535 // Execute pipeline stages in reverse order to simulate
536 // the pipeline latency
537 globalMemoryPipe.exec();
538 localMemoryPipe.exec();
539 execStage.exec();
540 scheduleStage.exec();
541 scoreboardCheckStage.exec();
542 fetchStage.exec();
543
544 totalCycles++;
545}
546
547void
548ComputeUnit::init()
549{
550 // Initialize CU Bus models
551 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
552 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
553 nextGlbMemBus = 0;
554 nextLocMemBus = 0;
555 fatal_if(numGlbMemUnits > 1,
556 "No support for multiple Global Memory Pipelines exists!!!");
557 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
558 for (int j = 0; j < numGlbMemUnits; ++j) {
559 vrfToGlobalMemPipeBus[j] = WaitClass();
560 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
561 }
562
563 fatal_if(numLocMemUnits > 1,
564 "No support for multiple Local Memory Pipelines exists!!!");
565 vrfToLocalMemPipeBus.resize(numLocMemUnits);
566 for (int j = 0; j < numLocMemUnits; ++j) {
567 vrfToLocalMemPipeBus[j] = WaitClass();
568 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
569 }
570 vectorRegsReserved.resize(numSIMDs, 0);
571 aluPipe.resize(numSIMDs);
572 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
573
574 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
575 wfWait[i] = WaitClass();
576 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
577 }
578
579 for (int i = 0; i < numSIMDs; ++i) {
580 aluPipe[i] = WaitClass();
581 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
582 }
583
584 // Setup space for call args
585 for (int j = 0; j < numSIMDs; ++j) {
586 for (int i = 0; i < shader->n_wf; ++i) {
587 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
588 }
589 }
590
591 // Initializing pipeline resources
592 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
593 waveStatusList.resize(numSIMDs);
594
595 for (int j = 0; j < numSIMDs; ++j) {
596 for (int i = 0; i < shader->n_wf; ++i) {
597 waveStatusList[j].push_back(
598 std::make_pair(wfList[j][i], BLOCKED));
599 }
600 }
601
602 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
603 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
604 }
605
606 fetchStage.init(this);
607 scoreboardCheckStage.init(this);
608 scheduleStage.init(this);
609 execStage.init(this);
610 globalMemoryPipe.init(this);
611 localMemoryPipe.init(this);
612 // initialize state for statistics calculation
613 vectorAluInstAvail.resize(numSIMDs, false);
614 shrMemInstAvail = 0;
615 glbMemInstAvail = 0;
616}
617
618bool
619ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
620{
621 // Ruby has completed the memory op. Schedule the mem_resp_event at the
622 // appropriate cycle to process the timing memory response
623 // This delay represents the pipeline delay
624 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
625 int index = sender_state->port_index;
626 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
627
628 // Is the packet returned a Kernel End or Barrier
629 if (pkt->req->isKernel() && pkt->req->isRelease()) {
630 Wavefront *w =
631 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
632
633 // Check if we are waiting on Kernel End Release
634 if (w->status == Wavefront::S_RETURNING) {
635 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
636 computeUnit->cu_id, w->simdId, w->wfSlotId,
637 w->wfDynId, w->kernId);
638
639 computeUnit->shader->dispatcher->notifyWgCompl(w);
640 w->status = Wavefront::S_STOPPED;
641 } else {
642 w->outstandingReqs--;
643 }
644
645 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
646 computeUnit->cu_id, gpuDynInst->simdId,
647 gpuDynInst->wfSlotId, w->barrierCnt);
648
649 if (gpuDynInst->useContinuation) {
650 assert(!gpuDynInst->isNoScope());
651 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
652 gpuDynInst);
653 }
654
655 delete pkt->senderState;
656 delete pkt->req;
657 delete pkt;
658 return true;
659 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
660 if (gpuDynInst->useContinuation) {
661 assert(!gpuDynInst->isNoScope());
662 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
663 gpuDynInst);
664 }
665
666 delete pkt->senderState;
667 delete pkt->req;
668 delete pkt;
669 return true;
670 }
671
672 EventFunctionWrapper *mem_resp_event =
673 computeUnit->memPort[index]->createMemRespEvent(pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
751 pkt->req->getFlags(), pkt->req->masterId(),
752 pkt->req->getPC());
753
754 // figure out the type of the request to set read/write
755 BaseTLB::Mode TLB_mode;
756 assert(pkt->isRead() || pkt->isWrite());
757
758 // Check write before read for atomic operations
759 // since atomic operations should use BaseTLB::Write
760 if (pkt->isWrite()){
761 TLB_mode = BaseTLB::Write;
762 } else if (pkt->isRead()) {
763 TLB_mode = BaseTLB::Read;
764 } else {
765 fatal("pkt is not a read nor a write\n");
766 }
767
768 tlbCycles -= curTick();
769 ++tlbRequests;
770
771 int tlbPort_index = perLaneTLB ? index : 0;
772
773 if (shader->timingSim) {
774 if (debugSegFault) {
775 Process *p = shader->gpuTc->getProcessPtr();
776 Addr vaddr = pkt->req->getVaddr();
777 unsigned size = pkt->getSize();
778
779 if ((vaddr + size - 1) % 64 < vaddr % 64) {
780 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
781 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
782 }
783
784 Addr paddr;
785
786 if (!p->pTable->translate(vaddr, paddr)) {
787 if (!p->fixupStackFault(vaddr)) {
788 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
789 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
790 vaddr);
791 }
792 }
793 }
794
795 // This is the SenderState needed upon return
796 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
797
798 // This is the senderState needed by the TLB hierarchy to function
799 TheISA::GpuTLB::TranslationState *translation_state =
800 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
801 pkt->senderState);
802
803 pkt->senderState = translation_state;
804
805 if (functionalTLB) {
806 tlbPort[tlbPort_index]->sendFunctional(pkt);
807
808 // update the hitLevel distribution
809 int hit_level = translation_state->hitLevel;
810 assert(hit_level != -1);
811 hitsPerTLBLevel[hit_level]++;
812
813 // New SenderState for the memory access
814 X86ISA::GpuTLB::TranslationState *sender_state =
815 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
816
817 delete sender_state->tlbEntry;
818 delete sender_state->saved;
819 delete sender_state;
820
821 assert(pkt->req->hasPaddr());
822 assert(pkt->req->hasSize());
823
824 uint8_t *tmpData = pkt->getPtr<uint8_t>();
825
826 // this is necessary because the GPU TLB receives packets instead
827 // of requests. when the translation is complete, all relevent
828 // fields in the request will be populated, but not in the packet.
829 // here we create the new packet so we can set the size, addr,
830 // and proper flags.
831 PacketPtr oldPkt = pkt;
832 pkt = new Packet(oldPkt->req, oldPkt->cmd);
833 delete oldPkt;
834 pkt->dataStatic(tmpData);
835
836
837 // New SenderState for the memory access
838 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
839 index, nullptr);
840
841 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
842 gpuDynInst->tlbHitLevel[index] = hit_level;
843
844
845 // translation is done. Schedule the mem_req_event at the
846 // appropriate cycle to send the timing memory request to ruby
847 EventFunctionWrapper *mem_req_event =
848 memPort[index]->createMemReqEvent(pkt);
849
850 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
851 "scheduled\n", cu_id, gpuDynInst->simdId,
852 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
853
854 schedule(mem_req_event, curTick() + req_tick_latency);
855 } else if (tlbPort[tlbPort_index]->isStalled()) {
856 assert(tlbPort[tlbPort_index]->retries.size() > 0);
857
858 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
859 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
860 tmp_vaddr);
861
862 tlbPort[tlbPort_index]->retries.push_back(pkt);
863 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
864 // Stall the data port;
865 // No more packet will be issued till
866 // ruby indicates resources are freed by
867 // a recvReqRetry() call back on this port.
868 tlbPort[tlbPort_index]->stallPort();
869
870 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
871 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
872 tmp_vaddr);
873
874 tlbPort[tlbPort_index]->retries.push_back(pkt);
875 } else {
876 DPRINTF(GPUTLB,
877 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
878 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
879 }
880 } else {
881 if (pkt->cmd == MemCmd::MemFenceReq) {
882 gpuDynInst->statusBitVector = VectorMask(0);
883 } else {
884 gpuDynInst->statusBitVector &= (~(1ll << index));
885 }
886
887 // New SenderState for the memory access
888 delete pkt->senderState;
889
890 // Because it's atomic operation, only need TLB translation state
891 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
892 shader->gpuTc);
893
894 tlbPort[tlbPort_index]->sendFunctional(pkt);
895
896 // the addr of the packet is not modified, so we need to create a new
897 // packet, or otherwise the memory access will have the old virtual
898 // address sent in the translation packet, instead of the physical
899 // address returned by the translation.
900 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
901 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
902
903 // Translation is done. It is safe to send the packet to memory.
904 memPort[0]->sendFunctional(new_pkt);
905
906 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
907 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
908 new_pkt->req->getPaddr());
909
910 // safe_cast the senderState
911 TheISA::GpuTLB::TranslationState *sender_state =
912 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
913
914 delete sender_state->tlbEntry;
915 delete new_pkt;
916 delete pkt->senderState;
917 delete pkt->req;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 EventFunctionWrapper *mem_req_event =
926 memPort[index]->createMemReqEvent(pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 Request* req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
948 }
949 req->setPaddr(0);
950 if (kernelLaunch) {
951 req->setFlags(Request::KERNEL);
952 }
953
954 // for non-kernel MemFence operations, memorder flags are set depending
955 // on which type of request is currently being sent, so this
956 // should be set by the caller (e.g. if an inst has acq-rel
957 // semantics, it will send one acquire req an one release req)
958 gpuDynInst->setRequestFlags(req, kernelLaunch);
959
960 // a mem fence must correspond to an acquire/release request
961 assert(req->isAcquire() || req->isRelease());
962
963 // create packet
964 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
965
966 // set packet's sender state
967 pkt->senderState =
968 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
969
970 // send the packet
971 sendSyncRequest(gpuDynInst, 0, pkt);
972}
973
974void
975ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt)
976{
977 DataPort::SenderState *sender_state =
978 safe_cast<DataPort::SenderState*>(pkt->senderState);
979
980 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
981 ComputeUnit *compute_unit = computeUnit;
982
983 assert(gpuDynInst);
984
985 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
986 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
987 pkt->req->getPaddr(), index);
988
989 Addr paddr = pkt->req->getPaddr();
990
991 if (pkt->cmd != MemCmd::MemFenceResp) {
992 int index = gpuDynInst->memStatusVector[paddr].back();
993
994 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
995 pkt->req->getPaddr(), index);
996
997 gpuDynInst->memStatusVector[paddr].pop_back();
998 gpuDynInst->pAddr = pkt->req->getPaddr();
999
1000 if (pkt->isRead() || pkt->isWrite()) {
1001
1002 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1003 gpuDynInst->statusBitVector &= (~(1ULL << index));
1004 } else {
1005 assert(gpuDynInst->statusVector[index] > 0);
1006 gpuDynInst->statusVector[index]--;
1007
1008 if (!gpuDynInst->statusVector[index])
1009 gpuDynInst->statusBitVector &= (~(1ULL << index));
1010 }
1011
1012 DPRINTF(GPUMem, "bitvector is now %#x\n",
1013 gpuDynInst->statusBitVector);
1014
1015 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1016 auto iter = gpuDynInst->memStatusVector.begin();
1017 auto end = gpuDynInst->memStatusVector.end();
1018
1019 while (iter != end) {
1020 assert(iter->second.empty());
1021 ++iter;
1022 }
1023
1024 gpuDynInst->memStatusVector.clear();
1025
1026 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1027 gpuDynInst->statusVector.clear();
1028
1029 compute_unit->globalMemoryPipe.handleResponse(gpuDynInst);
1030
1031 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1032 compute_unit->cu_id, gpuDynInst->simdId,
1033 gpuDynInst->wfSlotId);
1034
1035 // after clearing the status vectors,
1036 // see if there is a continuation to perform
1037 // the continuation may generate more work for
1038 // this memory request
1039 if (gpuDynInst->useContinuation) {
1040 assert(!gpuDynInst->isNoScope());
1041 gpuDynInst->execContinuation(
1042 gpuDynInst->staticInstruction(),
1043 gpuDynInst);
1044 }
1045 }
1046 }
1047 } else {
1048 gpuDynInst->statusBitVector = VectorMask(0);
1049
1050 if (gpuDynInst->useContinuation) {
1051 assert(!gpuDynInst->isNoScope());
1052 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1053 gpuDynInst);
1054 }
1055 }
1056
1057 delete pkt->senderState;
1058 delete pkt->req;
1059 delete pkt;
1060}
1061
1062ComputeUnit*
1063ComputeUnitParams::create()
1064{
1065 return new ComputeUnit(this);
1066}
1067
1068bool
1069ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1070{
1071 Addr line = pkt->req->getPaddr();
1072
1073 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1074 pkt->req->getVaddr(), line);
1075
1076 assert(pkt->senderState);
1077 computeUnit->tlbCycles += curTick();
1078
1079 // pop off the TLB translation state
1080 TheISA::GpuTLB::TranslationState *translation_state =
1081 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1082
1083 // no PageFaults are permitted for data accesses
1084 if (!translation_state->tlbEntry->valid) {
1085 DTLBPort::SenderState *sender_state =
1086 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1087
1088 Wavefront *w M5_VAR_USED =
1089 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1090 [sender_state->_gpuDynInst->wfSlotId];
1091
1092 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1093 pkt->req->getVaddr());
1094 }
1095
1096 assert(translation_state->tlbEntry->valid);
1097
1098 // update the hitLevel distribution
1099 int hit_level = translation_state->hitLevel;
1100 computeUnit->hitsPerTLBLevel[hit_level]++;
1101
1102 delete translation_state->tlbEntry;
1103 assert(!translation_state->ports.size());
1104 pkt->senderState = translation_state->saved;
1105
1106 // for prefetch pkt
1107 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1108
1109 delete translation_state;
1110
1111 // use the original sender state to know how to close this transaction
1112 DTLBPort::SenderState *sender_state =
1113 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1114
1115 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1116 int mp_index = sender_state->portIndex;
1117 Addr vaddr = pkt->req->getVaddr();
1118 gpuDynInst->memStatusVector[line].push_back(mp_index);
1119 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1120
1121 MemCmd requestCmd;
1122
1123 if (pkt->cmd == MemCmd::ReadResp) {
1124 requestCmd = MemCmd::ReadReq;
1125 } else if (pkt->cmd == MemCmd::WriteResp) {
1126 requestCmd = MemCmd::WriteReq;
1127 } else if (pkt->cmd == MemCmd::SwapResp) {
1128 requestCmd = MemCmd::SwapReq;
1129 } else {
1130 panic("unsupported response to request conversion %s\n",
1131 pkt->cmd.toString());
1132 }
1133
1134 if (computeUnit->prefetchDepth) {
1135 int simdId = gpuDynInst->simdId;
1136 int wfSlotId = gpuDynInst->wfSlotId;
1137 Addr last = 0;
1138
1139 switch(computeUnit->prefetchType) {
1140 case Enums::PF_CU:
1141 last = computeUnit->lastVaddrCU[mp_index];
1142 break;
1143 case Enums::PF_PHASE:
1144 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1145 break;
1146 case Enums::PF_WF:
1147 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1148 default:
1149 break;
1150 }
1151
1152 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1153 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1154
1155 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1156 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1157 : 0;
1158
1159 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1160
1161 computeUnit->lastVaddrCU[mp_index] = vaddr;
1162 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1163 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1164
1165 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1166 computeUnit->prefetchStride: stride;
1167
1168 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1169 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1170
1171 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1172
1173 // Prefetch Next few pages atomically
1174 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1175 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1176 vaddr+stride*pf*TheISA::PageBytes);
1177
1178 if (!stride)
1179 break;
1180
1181 Request *prefetch_req = new Request(0, vaddr + stride * pf *
1182 TheISA::PageBytes,
1183 sizeof(uint8_t), 0,
1184 computeUnit->masterId(),
1185 0, 0, 0);
1186
1187 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1188 uint8_t foo = 0;
1189 prefetch_pkt->dataStatic(&foo);
1190
1191 // Because it's atomic operation, only need TLB translation state
1192 prefetch_pkt->senderState =
1193 new TheISA::GpuTLB::TranslationState(TLB_mode,
1194 computeUnit->shader->gpuTc,
1195 true);
1196
1197 // Currently prefetches are zero-latency, hence the sendFunctional
1198 sendFunctional(prefetch_pkt);
1199
1200 /* safe_cast the senderState */
1201 TheISA::GpuTLB::TranslationState *tlb_state =
1202 safe_cast<TheISA::GpuTLB::TranslationState*>(
1203 prefetch_pkt->senderState);
1204
1205
1206 delete tlb_state->tlbEntry;
1207 delete tlb_state;
1208 delete prefetch_pkt->req;
1209 delete prefetch_pkt;
1210 }
1211 }
1212
1213 // First we must convert the response cmd back to a request cmd so that
1214 // the request can be sent through the cu's master port
1215 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1216 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1217 delete pkt->senderState;
1218 delete pkt;
1219
1220 // New SenderState for the memory access
1221 new_pkt->senderState =
1222 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1223 nullptr);
1224
1225 // translation is done. Schedule the mem_req_event at the appropriate
1226 // cycle to send the timing memory request to ruby
1227 EventFunctionWrapper *mem_req_event =
1228 computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt);
1229
1230 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1231 computeUnit->cu_id, gpuDynInst->simdId,
1232 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1233
1234 computeUnit->schedule(mem_req_event, curTick() +
1235 computeUnit->req_tick_latency);
1236
1237 return true;
1238}
1239
1240EventFunctionWrapper*
1241ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt)
1242{
1243 return new EventFunctionWrapper(
1244 [this, pkt]{ processMemReqEvent(pkt); },
1245 "ComputeUnit memory request event", true);
1246}
1247
1248EventFunctionWrapper*
1249ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt)
1250{
1251 return new EventFunctionWrapper(
1252 [this, pkt]{ processMemRespEvent(pkt); },
1253 "ComputeUnit memory response event", true);
1254}
1255
1256void
1257ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt)
1258{
1259 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1260 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1261 ComputeUnit *compute_unit M5_VAR_USED = computeUnit;
1262
1263 if (!(sendTimingReq(pkt))) {
1264 retries.push_back(std::make_pair(pkt, gpuDynInst));
1265
1266 DPRINTF(GPUPort,
1267 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1268 compute_unit->cu_id, gpuDynInst->simdId,
1269 gpuDynInst->wfSlotId, index,
1270 pkt->req->getPaddr());
1271 } else {
1272 DPRINTF(GPUPort,
1273 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1274 compute_unit->cu_id, gpuDynInst->simdId,
1275 gpuDynInst->wfSlotId, index,
1276 pkt->req->getPaddr());
1277 }
1278}
1279
1280/*
1281 * The initial translation request could have been rejected,
1282 * if <retries> queue is not Retry sending the translation
1283 * request. sendRetry() is called from the peer port whenever
1284 * a translation completes.
1285 */
1286void
1287ComputeUnit::DTLBPort::recvReqRetry()
1288{
1289 int len = retries.size();
1290
1291 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1292 computeUnit->cu_id, len);
1293
1294 assert(len > 0);
1295 assert(isStalled());
1296 // recvReqRetry is an indication that the resource on which this
1297 // port was stalling on is freed. So, remove the stall first
1298 unstallPort();
1299
1300 for (int i = 0; i < len; ++i) {
1301 PacketPtr pkt = retries.front();
1302 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1303 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1304
1305 if (!sendTimingReq(pkt)) {
1306 // Stall port
1307 stallPort();
1308 DPRINTF(GPUTLB, ": failed again\n");
1309 break;
1310 } else {
1311 DPRINTF(GPUTLB, ": successful\n");
1312 retries.pop_front();
1313 }
1314 }
1315}
1316
1317bool
1318ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1319{
1320 Addr line M5_VAR_USED = pkt->req->getPaddr();
1321 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1322 computeUnit->cu_id, pkt->req->getVaddr(), line);
1323
1324 assert(pkt->senderState);
1325
1326 // pop off the TLB translation state
1327 TheISA::GpuTLB::TranslationState *translation_state =
1328 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1329
1330 bool success = translation_state->tlbEntry->valid;
1331 delete translation_state->tlbEntry;
1332 assert(!translation_state->ports.size());
1333 pkt->senderState = translation_state->saved;
1334 delete translation_state;
1335
1336 // use the original sender state to know how to close this transaction
1337 ITLBPort::SenderState *sender_state =
1338 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1339
1340 // get the wavefront associated with this translation request
1341 Wavefront *wavefront = sender_state->wavefront;
1342 delete pkt->senderState;
1343
1344 if (success) {
1345 // pkt is reused in fetch(), don't delete it here. However, we must
1346 // reset the command to be a request so that it can be sent through
1347 // the cu's master port
1348 assert(pkt->cmd == MemCmd::ReadResp);
1349 pkt->cmd = MemCmd::ReadReq;
1350
1351 computeUnit->fetchStage.fetch(pkt, wavefront);
1352 } else {
1353 if (wavefront->dropFetch) {
1354 assert(wavefront->instructionBuffer.empty());
1355 wavefront->dropFetch = false;
1356 }
1357
1358 wavefront->pendingFetch = 0;
1359 }
1360
1361 return true;
1362}
1363
1364/*
1365 * The initial translation request could have been rejected, if
1366 * <retries> queue is not empty. Retry sending the translation
1367 * request. sendRetry() is called from the peer port whenever
1368 * a translation completes.
1369 */
1370void
1371ComputeUnit::ITLBPort::recvReqRetry()
1372{
1373
1374 int len = retries.size();
1375 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1376
1377 assert(len > 0);
1378 assert(isStalled());
1379
1380 // recvReqRetry is an indication that the resource on which this
1381 // port was stalling on is freed. So, remove the stall first
1382 unstallPort();
1383
1384 for (int i = 0; i < len; ++i) {
1385 PacketPtr pkt = retries.front();
1386 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1387 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1388
1389 if (!sendTimingReq(pkt)) {
1390 stallPort(); // Stall port
1391 DPRINTF(GPUTLB, ": failed again\n");
1392 break;
1393 } else {
1394 DPRINTF(GPUTLB, ": successful\n");
1395 retries.pop_front();
1396 }
1397 }
1398}
1399
1400void
1401ComputeUnit::regStats()
1402{
1403 MemObject::regStats();
1404
1405 vALUInsts
1406 .name(name() + ".valu_insts")
1407 .desc("Number of vector ALU insts issued.")
1408 ;
1409 vALUInstsPerWF
1410 .name(name() + ".valu_insts_per_wf")
1411 .desc("The avg. number of vector ALU insts issued per-wavefront.")
1412 ;
1413 sALUInsts
1414 .name(name() + ".salu_insts")
1415 .desc("Number of scalar ALU insts issued.")
1416 ;
1417 sALUInstsPerWF
1418 .name(name() + ".salu_insts_per_wf")
1419 .desc("The avg. number of scalar ALU insts issued per-wavefront.")
1420 ;
1421 instCyclesVALU
1422 .name(name() + ".inst_cycles_valu")
1423 .desc("Number of cycles needed to execute VALU insts.")
1424 ;
1425 instCyclesSALU
1426 .name(name() + ".inst_cycles_salu")
1427 .desc("Number of cycles needed to execute SALU insts.")
1428 ;
1429 threadCyclesVALU
1430 .name(name() + ".thread_cycles_valu")
1431 .desc("Number of thread cycles used to execute vector ALU ops. "
1432 "Similar to instCyclesVALU but multiplied by the number of "
1433 "active threads.")
1434 ;
1435 vALUUtilization
1436 .name(name() + ".valu_utilization")
1437 .desc("Percentage of active vector ALU threads in a wave.")
1438 ;
1439 ldsNoFlatInsts
1440 .name(name() + ".lds_no_flat_insts")
1441 .desc("Number of LDS insts issued, not including FLAT "
1442 "accesses that resolve to LDS.")
1443 ;
1444 ldsNoFlatInstsPerWF
1445 .name(name() + ".lds_no_flat_insts_per_wf")
1446 .desc("The avg. number of LDS insts (not including FLAT "
1447 "accesses that resolve to LDS) per-wavefront.")
1448 ;
1449 flatVMemInsts
1450 .name(name() + ".flat_vmem_insts")
1451 .desc("The number of FLAT insts that resolve to vmem issued.")
1452 ;
1453 flatVMemInstsPerWF
1454 .name(name() + ".flat_vmem_insts_per_wf")
1455 .desc("The average number of FLAT insts that resolve to vmem "
1456 "issued per-wavefront.")
1457 ;
1458 flatLDSInsts
1459 .name(name() + ".flat_lds_insts")
1460 .desc("The number of FLAT insts that resolve to LDS issued.")
1461 ;
1462 flatLDSInstsPerWF
1463 .name(name() + ".flat_lds_insts_per_wf")
1464 .desc("The average number of FLAT insts that resolve to LDS "
1465 "issued per-wavefront.")
1466 ;
1467 vectorMemWrites
1468 .name(name() + ".vector_mem_writes")
1469 .desc("Number of vector mem write insts (excluding FLAT insts).")
1470 ;
1471 vectorMemWritesPerWF
1472 .name(name() + ".vector_mem_writes_per_wf")
1473 .desc("The average number of vector mem write insts "
1474 "(excluding FLAT insts) per-wavefront.")
1475 ;
1476 vectorMemReads
1477 .name(name() + ".vector_mem_reads")
1478 .desc("Number of vector mem read insts (excluding FLAT insts).")
1479 ;
1480 vectorMemReadsPerWF
1481 .name(name() + ".vector_mem_reads_per_wf")
1482 .desc("The avg. number of vector mem read insts (excluding "
1483 "FLAT insts) per-wavefront.")
1484 ;
1485 scalarMemWrites
1486 .name(name() + ".scalar_mem_writes")
1487 .desc("Number of scalar mem write insts.")
1488 ;
1489 scalarMemWritesPerWF
1490 .name(name() + ".scalar_mem_writes_per_wf")
1491 .desc("The average number of scalar mem write insts per-wavefront.")
1492 ;
1493 scalarMemReads
1494 .name(name() + ".scalar_mem_reads")
1495 .desc("Number of scalar mem read insts.")
1496 ;
1497 scalarMemReadsPerWF
1498 .name(name() + ".scalar_mem_reads_per_wf")
1499 .desc("The average number of scalar mem read insts per-wavefront.")
1500 ;
1501
1502 vALUInstsPerWF = vALUInsts / completedWfs;
1503 sALUInstsPerWF = sALUInsts / completedWfs;
1504 vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100;
1505 ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs;
1506 flatVMemInstsPerWF = flatVMemInsts / completedWfs;
1507 flatLDSInstsPerWF = flatLDSInsts / completedWfs;
1508 vectorMemWritesPerWF = vectorMemWrites / completedWfs;
1509 vectorMemReadsPerWF = vectorMemReads / completedWfs;
1510 scalarMemWritesPerWF = scalarMemWrites / completedWfs;
1511 scalarMemReadsPerWF = scalarMemReads / completedWfs;
1512
1513 tlbCycles
1514 .name(name() + ".tlb_cycles")
1515 .desc("total number of cycles for all uncoalesced requests")
1516 ;
1517
1518 tlbRequests
1519 .name(name() + ".tlb_requests")
1520 .desc("number of uncoalesced requests")
1521 ;
1522
1523 tlbLatency
1524 .name(name() + ".avg_translation_latency")
1525 .desc("Avg. translation latency for data translations")
1526 ;
1527
1528 tlbLatency = tlbCycles / tlbRequests;
1529
1530 hitsPerTLBLevel
1531 .init(4)
1532 .name(name() + ".TLB_hits_distribution")
1533 .desc("TLB hits distribution (0 for page table, x for Lx-TLB")
1534 ;
1535
1536 // fixed number of TLB levels
1537 for (int i = 0; i < 4; ++i) {
1538 if (!i)
1539 hitsPerTLBLevel.subname(i,"page_table");
1540 else
1541 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i));
1542 }
1543
1544 execRateDist
1545 .init(0, 10, 2)
1546 .name(name() + ".inst_exec_rate")
1547 .desc("Instruction Execution Rate: Number of executed vector "
1548 "instructions per cycle")
1549 ;
1550
1551 ldsBankConflictDist
1552 .init(0, wfSize(), 2)
1553 .name(name() + ".lds_bank_conflicts")
1554 .desc("Number of bank conflicts per LDS memory packet")
1555 ;
1556
1557 ldsBankAccesses
1558 .name(name() + ".lds_bank_access_cnt")
1559 .desc("Total number of LDS bank accesses")
1560 ;
1561
1562 pageDivergenceDist
1563 // A wavefront can touch up to N pages per memory instruction where
1564 // N is equal to the wavefront size
1565 // The number of pages per bin can be configured (here it's 4).
1566 .init(1, wfSize(), 4)
1567 .name(name() + ".page_divergence_dist")
1568 .desc("pages touched per wf (over all mem. instr.)")
1569 ;
1570
1571 controlFlowDivergenceDist
1572 .init(1, wfSize(), 4)
1573 .name(name() + ".warp_execution_dist")
1574 .desc("number of lanes active per instruction (oval all instructions)")
1575 ;
1576
1577 activeLanesPerGMemInstrDist
1578 .init(1, wfSize(), 4)
1579 .name(name() + ".gmem_lanes_execution_dist")
1580 .desc("number of active lanes per global memory instruction")
1581 ;
1582
1583 activeLanesPerLMemInstrDist
1584 .init(1, wfSize(), 4)
1585 .name(name() + ".lmem_lanes_execution_dist")
1586 .desc("number of active lanes per local memory instruction")
1587 ;
1588
1589 numInstrExecuted
1590 .name(name() + ".num_instr_executed")
1591 .desc("number of instructions executed")
1592 ;
1593
1594 numVecOpsExecuted
1595 .name(name() + ".num_vec_ops_executed")
1596 .desc("number of vec ops executed (e.g. WF size/inst)")
1597 ;
1598
1599 totalCycles
1600 .name(name() + ".num_total_cycles")
1601 .desc("number of cycles the CU ran for")
1602 ;
1603
1604 ipc
1605 .name(name() + ".ipc")
1606 .desc("Instructions per cycle (this CU only)")
1607 ;
1608
1609 vpc
1610 .name(name() + ".vpc")
1611 .desc("Vector Operations per cycle (this CU only)")
1612 ;
1613
1614 numALUInstsExecuted
1615 .name(name() + ".num_alu_insts_executed")
1616 .desc("Number of dynamic non-GM memory insts executed")
1617 ;
1618
1619 wgBlockedDueLdsAllocation
1620 .name(name() + ".wg_blocked_due_lds_alloc")
1621 .desc("Workgroup blocked due to LDS capacity")
1622 ;
1623
1624 ipc = numInstrExecuted / totalCycles;
1625 vpc = numVecOpsExecuted / totalCycles;
1626
1627 numTimesWgBlockedDueVgprAlloc
1628 .name(name() + ".times_wg_blocked_due_vgpr_alloc")
1629 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD")
1630 ;
1631
1632 dynamicGMemInstrCnt
1633 .name(name() + ".global_mem_instr_cnt")
1634 .desc("dynamic global memory instructions count")
1635 ;
1636
1637 dynamicLMemInstrCnt
1638 .name(name() + ".local_mem_instr_cnt")
1639 .desc("dynamic local memory intruction count")
1640 ;
1641
1642 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt -
1643 dynamicLMemInstrCnt;
1644
1645 completedWfs
1646 .name(name() + ".num_completed_wfs")
1647 .desc("number of completed wavefronts")
1648 ;
1649
1650 numCASOps
1651 .name(name() + ".num_CAS_ops")
1652 .desc("number of compare and swap operations")
1653 ;
1654
1655 numFailedCASOps
1656 .name(name() + ".num_failed_CAS_ops")
1657 .desc("number of compare and swap operations that failed")
1658 ;
1659
1660 // register stats of pipeline stages
1661 fetchStage.regStats();
1662 scoreboardCheckStage.regStats();
1663 scheduleStage.regStats();
1664 execStage.regStats();
1665
1666 // register stats of memory pipeline
1667 globalMemoryPipe.regStats();
1668 localMemoryPipe.regStats();
1669}
1670
1671void
1672ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst)
1673{
1674 if (gpuDynInst->isScalar()) {
1675 if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) {
1676 sALUInsts++;
1677 instCyclesSALU++;
1678 } else if (gpuDynInst->isLoad()) {
1679 scalarMemReads++;
1680 } else if (gpuDynInst->isStore()) {
1681 scalarMemWrites++;
1682 }
1683 } else {
1684 if (gpuDynInst->isALU()) {
1685 vALUInsts++;
1686 instCyclesVALU++;
1687 threadCyclesVALU += gpuDynInst->wavefront()->execMask().count();
1688 } else if (gpuDynInst->isFlat()) {
1689 if (gpuDynInst->isLocalMem()) {
1690 flatLDSInsts++;
1691 } else {
1692 flatVMemInsts++;
1693 }
1694 } else if (gpuDynInst->isLocalMem()) {
1695 ldsNoFlatInsts++;
1696 } else if (gpuDynInst->isLoad()) {
1697 vectorMemReads++;
1698 } else if (gpuDynInst->isStore()) {
1699 vectorMemWrites++;
1700 }
1701 }
1702}
1703
1704void
1705ComputeUnit::updatePageDivergenceDist(Addr addr)
1706{
1707 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes);
1708
1709 if (!pagesTouched.count(virt_page_addr))
1710 pagesTouched[virt_page_addr] = 1;
1711 else
1712 pagesTouched[virt_page_addr]++;
1713}
1714
1715void
1716ComputeUnit::CUExitCallback::process()
1717{
1718 if (computeUnit->countPages) {
1719 std::ostream *page_stat_file =
1720 simout.create(computeUnit->name().c_str())->stream();
1721
1722 *page_stat_file << "page, wavefront accesses, workitem accesses" <<
1723 std::endl;
1724
1725 for (auto iter : computeUnit->pageAccesses) {
1726 *page_stat_file << std::hex << iter.first << ",";
1727 *page_stat_file << std::dec << iter.second.first << ",";
1728 *page_stat_file << std::dec << iter.second.second << std::endl;
1729 }
1730 }
1731 }
1732
1733bool
1734ComputeUnit::isDone() const
1735{
1736 for (int i = 0; i < numSIMDs; ++i) {
1737 if (!isSimdDone(i)) {
1738 return false;
1739 }
1740 }
1741
1742 bool glbMemBusRdy = true;
1743 for (int j = 0; j < numGlbMemUnits; ++j) {
1744 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy();
1745 }
1746 bool locMemBusRdy = true;
1747 for (int j = 0; j < numLocMemUnits; ++j) {
1748 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy();
1749 }
1750
1751 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() ||
1752 !globalMemoryPipe.isGMStRespFIFOWrRdy() ||
1753 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy()
1754 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() ||
1755 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) {
1756 return false;
1757 }
1758
1759 return true;
1760}
1761
1762int32_t
1763ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const
1764{
1765 return lds.getRefCounter(dispatchId, wgId);
1766}
1767
1768bool
1769ComputeUnit::isSimdDone(uint32_t simdId) const
1770{
1771 assert(simdId < numSIMDs);
1772
1773 for (int i=0; i < numGlbMemUnits; ++i) {
1774 if (!vrfToGlobalMemPipeBus[i].rdy())
1775 return false;
1776 }
1777 for (int i=0; i < numLocMemUnits; ++i) {
1778 if (!vrfToLocalMemPipeBus[i].rdy())
1779 return false;
1780 }
1781 if (!aluPipe[simdId].rdy()) {
1782 return false;
1783 }
1784
1785 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){
1786 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) {
1787 return false;
1788 }
1789 }
1790
1791 return true;
1792}
1793
1794/**
1795 * send a general request to the LDS
1796 * make sure to look at the return value here as your request might be
1797 * NACK'd and returning false means that you have to have some backup plan
1798 */
1799bool
1800ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst)
1801{
1802 // this is just a request to carry the GPUDynInstPtr
1803 // back and forth
1804 Request *newRequest = new Request();
1805 newRequest->setPaddr(0x0);
1806
1807 // ReadReq is not evaluted by the LDS but the Packet ctor requires this
1808 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq);
1809
1810 // This is the SenderState needed upon return
1811 newPacket->senderState = new LDSPort::SenderState(gpuDynInst);
1812
1813 return ldsPort->sendTimingReq(newPacket);
1814}
1815
1816/**
1817 * get the result of packets sent to the LDS when they return
1818 */
1819bool
1820ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet)
1821{
1822 const ComputeUnit::LDSPort::SenderState *senderState =
1823 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState);
1824
1825 fatal_if(!senderState, "did not get the right sort of sender state");
1826
1827 GPUDynInstPtr gpuDynInst = senderState->getMemInst();
1828
1829 delete packet->senderState;
1830 delete packet->req;
1831 delete packet;
1832
1833 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst);
1834 return true;
1835}
1836
1837/**
1838 * attempt to send this packet, either the port is already stalled, the request
1839 * is nack'd and must stall or the request goes through
1840 * when a request cannot be sent, add it to the retries queue
1841 */
1842bool
1843ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt)
1844{
1845 ComputeUnit::LDSPort::SenderState *sender_state =
1846 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState);
1847 fatal_if(!sender_state, "packet without a valid sender state");
1848
1849 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst();
1850
1851 if (isStalled()) {
1852 fatal_if(retries.empty(), "must have retries waiting to be stalled");
1853
1854 retries.push(pkt);
1855
1856 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n",
1857 computeUnit->cu_id, gpuDynInst->simdId,
1858 gpuDynInst->wfSlotId);
1859 return false;
1860 } else if (!MasterPort::sendTimingReq(pkt)) {
1861 // need to stall the LDS port until a recvReqRetry() is received
1862 // this indicates that there is more space
1863 stallPort();
1864 retries.push(pkt);
1865
1866 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n",
1867 computeUnit->cu_id, gpuDynInst->simdId,
1868 gpuDynInst->wfSlotId, pkt->req->getPaddr());
1869 return false;
1870 } else {
1871 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n",
1872 computeUnit->cu_id, gpuDynInst->simdId,
1873 gpuDynInst->wfSlotId, pkt->req->getPaddr());
1874 return true;
1875 }
1876}
1877
1878/**
1879 * the bus is telling the port that there is now space so retrying stalled
1880 * requests should work now
1881 * this allows the port to have a request be nack'd and then have the receiver
1882 * say when there is space, rather than simply retrying the send every cycle
1883 */
1884void
1885ComputeUnit::LDSPort::recvReqRetry()
1886{
1887 auto queueSize = retries.size();
1888
1889 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n",
1890 computeUnit->cu_id, queueSize);
1891
1892 fatal_if(queueSize < 1,
1893 "why was there a recvReqRetry() with no pending reqs?");
1894 fatal_if(!isStalled(),
1895 "recvReqRetry() happened when the port was not stalled");
1896
1897 unstallPort();
1898
1899 while (!retries.empty()) {
1900 PacketPtr packet = retries.front();
1901
1902 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id);
1903
1904 if (!MasterPort::sendTimingReq(packet)) {
1905 // Stall port
1906 stallPort();
1907 DPRINTF(GPUPort, ": LDS send failed again\n");
1908 break;
1909 } else {
1910 DPRINTF(GPUTLB, ": LDS send successful\n");
1911 retries.pop();
1912 }
1913 }
1914}
| 78 lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
79 globalSeqNum(0), wavefrontSize(p->wfSize),
80 kernelLaunchInst(new KernelLaunchStaticInst())
81{
82 /**
83 * This check is necessary because std::bitset only provides conversion
84 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
85 * there are * a few places in the code where to_ullong() is used, however
86 * if VSZ is larger than a value the host can support then bitset will
87 * throw a runtime exception. we should remove all use of to_long() or
88 * to_ullong() so we can have VSZ greater than 64b, however until that is
89 * done this assert is required.
90 */
91 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
92 p->wfSize <= 0,
93 "WF size is larger than the host can support");
94 fatal_if(!isPowerOf2(wavefrontSize),
95 "Wavefront size should be a power of 2");
96 // calculate how many cycles a vector load or store will need to transfer
97 // its data over the corresponding buses
98 numCyclesPerStoreTransfer =
99 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
100 (double)vrfToCoalescerBusWidth);
101
102 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
103 / coalescerToVrfBusWidth;
104
105 lastVaddrWF.resize(numSIMDs);
106 wfList.resize(numSIMDs);
107
108 for (int j = 0; j < numSIMDs; ++j) {
109 lastVaddrWF[j].resize(p->n_wf);
110
111 for (int i = 0; i < p->n_wf; ++i) {
112 lastVaddrWF[j][i].resize(wfSize());
113
114 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
115 wfList[j][i]->setParent(this);
116
117 for (int k = 0; k < wfSize(); ++k) {
118 lastVaddrWF[j][i][k] = 0;
119 }
120 }
121 }
122
123 lastVaddrSimd.resize(numSIMDs);
124
125 for (int i = 0; i < numSIMDs; ++i) {
126 lastVaddrSimd[i].resize(wfSize(), 0);
127 }
128
129 lastVaddrCU.resize(wfSize());
130
131 lds.setParent(this);
132
133 if (p->execPolicy == "OLDEST-FIRST") {
134 exec_policy = EXEC_POLICY::OLDEST;
135 } else if (p->execPolicy == "ROUND-ROBIN") {
136 exec_policy = EXEC_POLICY::RR;
137 } else {
138 fatal("Invalid WF execution policy (CU)\n");
139 }
140
141 memPort.resize(wfSize());
142
143 // resize the tlbPort vectorArray
144 int tlbPort_width = perLaneTLB ? wfSize() : 1;
145 tlbPort.resize(tlbPort_width);
146
147 cuExitCallback = new CUExitCallback(this);
148 registerExitCallback(cuExitCallback);
149
150 xactCasLoadMap.clear();
151 lastExecCycle.resize(numSIMDs, 0);
152
153 for (int i = 0; i < vrf.size(); ++i) {
154 vrf[i]->setParent(this);
155 }
156
157 numVecRegsPerSimd = vrf[0]->numRegs();
158}
159
160ComputeUnit::~ComputeUnit()
161{
162 // Delete wavefront slots
163 for (int j = 0; j < numSIMDs; ++j) {
164 for (int i = 0; i < shader->n_wf; ++i) {
165 delete wfList[j][i];
166 }
167 lastVaddrSimd[j].clear();
168 }
169 lastVaddrCU.clear();
170 readyList.clear();
171 waveStatusList.clear();
172 dispatchList.clear();
173 vectorAluInstAvail.clear();
174 delete cuExitCallback;
175 delete ldsPort;
176}
177
178void
179ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
180{
181 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
182
183 w->workGroupSz[0] = ndr->q.wgSize[0];
184 w->workGroupSz[1] = ndr->q.wgSize[1];
185 w->workGroupSz[2] = ndr->q.wgSize[2];
186 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
187 w->gridSz[0] = ndr->q.gdSize[0];
188 w->gridSz[1] = ndr->q.gdSize[1];
189 w->gridSz[2] = ndr->q.gdSize[2];
190 w->kernelArgs = ndr->q.args;
191 w->privSizePerItem = ndr->q.privMemPerItem;
192 w->spillSizePerItem = ndr->q.spillMemPerItem;
193 w->roBase = ndr->q.roMemStart;
194 w->roSize = ndr->q.roMemTotal;
195 w->computeActualWgSz(ndr);
196}
197
198void
199ComputeUnit::updateEvents() {
200
201 if (!timestampVec.empty()) {
202 uint32_t vecSize = timestampVec.size();
203 uint32_t i = 0;
204 while (i < vecSize) {
205 if (timestampVec[i] <= shader->tick_cnt) {
206 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
207 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
208 statusVec[i]);
209 timestampVec.erase(timestampVec.begin() + i);
210 regIdxVec.erase(regIdxVec.begin() + i);
211 statusVec.erase(statusVec.begin() + i);
212 --vecSize;
213 --i;
214 }
215 ++i;
216 }
217 }
218
219 for (int i = 0; i< numSIMDs; ++i) {
220 vrf[i]->updateEvents();
221 }
222}
223
224
225void
226ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
227 NDRange *ndr)
228{
229 static int _n_wave = 0;
230
231 VectorMask init_mask;
232 init_mask.reset();
233
234 for (int k = 0; k < wfSize(); ++k) {
235 if (k + waveId * wfSize() < w->actualWgSzTotal)
236 init_mask[k] = 1;
237 }
238
239 w->kernId = ndr->dispatchId;
240 w->wfId = waveId;
241 w->initMask = init_mask.to_ullong();
242
243 for (int k = 0; k < wfSize(); ++k) {
244 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
245 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
246 w->actualWgSz[1];
247 w->workItemId[2][k] = (k + waveId * wfSize()) /
248 (w->actualWgSz[0] * w->actualWgSz[1]);
249
250 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
251 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
252 w->workItemId[0][k];
253 }
254
255 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
256
257 w->barCnt.resize(wfSize(), 0);
258
259 w->maxBarCnt = 0;
260 w->oldBarrierCnt = 0;
261 w->barrierCnt = 0;
262
263 w->privBase = ndr->q.privMemStart;
264 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
265
266 w->spillBase = ndr->q.spillMemStart;
267 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
268
269 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
270
271 // WG state
272 w->wgId = ndr->globalWgId;
273 w->dispatchId = ndr->dispatchId;
274 w->workGroupId[0] = w->wgId % ndr->numWg[0];
275 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
276 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
277
278 w->barrierId = barrier_id;
279 w->stalledAtBarrier = false;
280
281 // set the wavefront context to have a pointer to this section of the LDS
282 w->ldsChunk = ldsChunk;
283
284 int32_t refCount M5_VAR_USED =
285 lds.increaseRefCounter(w->dispatchId, w->wgId);
286 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
287 cu_id, w->wgId, refCount);
288
289 w->instructionBuffer.clear();
290
291 if (w->pendingFetch)
292 w->dropFetch = true;
293
294 // is this the last wavefront in the workgroup
295 // if set the spillWidth to be the remaining work-items
296 // so that the vector access is correct
297 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
298 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
299 } else {
300 w->spillWidth = wfSize();
301 }
302
303 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
304 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
305
306 w->start(++_n_wave, ndr->q.code_ptr);
307}
308
309void
310ComputeUnit::StartWorkgroup(NDRange *ndr)
311{
312 // reserve the LDS capacity allocated to the work group
313 // disambiguated by the dispatch ID and workgroup ID, which should be
314 // globally unique
315 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
316 ndr->q.ldsSize);
317
318 // Send L1 cache acquire
319 // isKernel + isAcquire = Kernel Begin
320 if (shader->impl_kern_boundary_sync) {
321 GPUDynInstPtr gpuDynInst =
322 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
323 getAndIncSeqNum());
324
325 gpuDynInst->useContinuation = false;
326 injectGlobalMemFence(gpuDynInst, true);
327 }
328
329 // calculate the number of 32-bit vector registers required by wavefront
330 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
331 int wave_id = 0;
332
333 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
334 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
335 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
336 // Check if this wavefront slot is available:
337 // It must be stopped and not waiting
338 // for a release to complete S_RETURNING
339 if (w->status == Wavefront::S_STOPPED) {
340 fillKernelState(w, ndr);
341 // if we have scheduled all work items then stop
342 // scheduling wavefronts
343 if (wave_id * wfSize() >= w->actualWgSzTotal)
344 break;
345
346 // reserve vector registers for the scheduled wavefront
347 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
348 uint32_t normSize = 0;
349
350 w->startVgprIndex = vrf[m % numSIMDs]->manager->
351 allocateRegion(vregDemand, &normSize);
352
353 w->reservedVectorRegs = normSize;
354 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
355
356 startWavefront(w, wave_id, ldsChunk, ndr);
357 ++wave_id;
358 }
359 }
360 ++barrier_id;
361}
362
363int
364ComputeUnit::ReadyWorkgroup(NDRange *ndr)
365{
366 // Get true size of workgroup (after clamping to grid size)
367 int trueWgSize[3];
368 int trueWgSizeTotal = 1;
369
370 for (int d = 0; d < 3; ++d) {
371 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
372 ndr->wgId[d] * ndr->q.wgSize[d]);
373
374 trueWgSizeTotal *= trueWgSize[d];
375 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
376 }
377
378 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
379
380 // calculate the number of 32-bit vector registers required by each
381 // work item of the work group
382 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
383 bool vregAvail = true;
384 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
385 int freeWfSlots = 0;
386 // check if the total number of VGPRs required by all WFs of the WG
387 // fit in the VRFs of all SIMD units
388 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
389 int numMappedWfs = 0;
390 std::vector<int> numWfsPerSimd;
391 numWfsPerSimd.resize(numSIMDs, 0);
392 // find how many free WF slots we have across all SIMDs
393 for (int j = 0; j < shader->n_wf; ++j) {
394 for (int i = 0; i < numSIMDs; ++i) {
395 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
396 // count the number of free WF slots
397 ++freeWfSlots;
398 if (numMappedWfs < numWfs) {
399 // count the WFs to be assigned per SIMD
400 numWfsPerSimd[i]++;
401 }
402 numMappedWfs++;
403 }
404 }
405 }
406
407 // if there are enough free WF slots then find if there are enough
408 // free VGPRs per SIMD based on the WF->SIMD mapping
409 if (freeWfSlots >= numWfs) {
410 for (int j = 0; j < numSIMDs; ++j) {
411 // find if there are enough free VGPR regions in the SIMD's VRF
412 // to accommodate the WFs of the new WG that would be mapped to
413 // this SIMD unit
414 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
415 vregDemandPerWI);
416
417 // stop searching if there is at least one SIMD
418 // whose VRF does not have enough free VGPR pools.
419 // This is because a WG is scheduled only if ALL
420 // of its WFs can be scheduled
421 if (!vregAvail)
422 break;
423 }
424 }
425
426 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
427 freeWfSlots, vregAvail);
428
429 if (!vregAvail) {
430 ++numTimesWgBlockedDueVgprAlloc;
431 }
432
433 // Return true if enough WF slots to submit workgroup and if there are
434 // enough VGPRs to schedule all WFs to their SIMD units
435 if (!lds.canReserve(ndr->q.ldsSize)) {
436 wgBlockedDueLdsAllocation++;
437 }
438
439 // Return true if (a) there are enough free WF slots to submit
440 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
441 // SIMD units and (c) if there is enough space in LDS
442 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
443}
444
445int
446ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
447{
448 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
449 int ccnt = 0;
450
451 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
452 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
453 Wavefront *w = wfList[i_simd][i_wf];
454
455 if (w->status == Wavefront::S_RUNNING) {
456 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
457
458 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
459 w->barrierId, _barrier_id);
460
461 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
462 w->barrierCnt, bcnt);
463 }
464
465 if (w->status == Wavefront::S_RUNNING &&
466 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
467 !w->outstandingReqs) {
468 ++ccnt;
469
470 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
471 "%d\n", i_simd, i_wf, ccnt);
472 }
473 }
474 }
475
476 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
477 cu_id, ccnt, bslots);
478
479 return ccnt == bslots;
480}
481
482// Check if the current wavefront is blocked on additional resources.
483bool
484ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
485{
486 bool cede = false;
487
488 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
489 // magic instructions will impact the scheduling of wavefronts
490 if (xact_cas_mode) {
491 /*
492 * When a wavefront calls xact_cas_ld, it adds itself to a per address
493 * queue. All per address queues are managed by the xactCasLoadMap.
494 *
495 * A wavefront is not blocked if: it is not in ANY per address queue or
496 * if it is at the head of a per address queue.
497 */
498 for (auto itMap : xactCasLoadMap) {
499 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
500
501 if (!curWaveIDQueue.empty()) {
502 for (auto it : curWaveIDQueue) {
503 waveIdentifier cur_wave = it;
504
505 if (cur_wave.simdId == simdId &&
506 cur_wave.wfSlotId == wfSlotId) {
507 // 2 possibilities
508 // 1: this WF has a green light
509 // 2: another WF has a green light
510 waveIdentifier owner_wave = curWaveIDQueue.front();
511
512 if (owner_wave.simdId != cur_wave.simdId ||
513 owner_wave.wfSlotId != cur_wave.wfSlotId) {
514 // possibility 2
515 cede = true;
516 break;
517 } else {
518 // possibility 1
519 break;
520 }
521 }
522 }
523 }
524 }
525 }
526
527 return cede;
528}
529
530// Execute one clock worth of work on the ComputeUnit.
531void
532ComputeUnit::exec()
533{
534 updateEvents();
535 // Execute pipeline stages in reverse order to simulate
536 // the pipeline latency
537 globalMemoryPipe.exec();
538 localMemoryPipe.exec();
539 execStage.exec();
540 scheduleStage.exec();
541 scoreboardCheckStage.exec();
542 fetchStage.exec();
543
544 totalCycles++;
545}
546
547void
548ComputeUnit::init()
549{
550 // Initialize CU Bus models
551 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
552 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
553 nextGlbMemBus = 0;
554 nextLocMemBus = 0;
555 fatal_if(numGlbMemUnits > 1,
556 "No support for multiple Global Memory Pipelines exists!!!");
557 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
558 for (int j = 0; j < numGlbMemUnits; ++j) {
559 vrfToGlobalMemPipeBus[j] = WaitClass();
560 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
561 }
562
563 fatal_if(numLocMemUnits > 1,
564 "No support for multiple Local Memory Pipelines exists!!!");
565 vrfToLocalMemPipeBus.resize(numLocMemUnits);
566 for (int j = 0; j < numLocMemUnits; ++j) {
567 vrfToLocalMemPipeBus[j] = WaitClass();
568 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
569 }
570 vectorRegsReserved.resize(numSIMDs, 0);
571 aluPipe.resize(numSIMDs);
572 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
573
574 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
575 wfWait[i] = WaitClass();
576 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
577 }
578
579 for (int i = 0; i < numSIMDs; ++i) {
580 aluPipe[i] = WaitClass();
581 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
582 }
583
584 // Setup space for call args
585 for (int j = 0; j < numSIMDs; ++j) {
586 for (int i = 0; i < shader->n_wf; ++i) {
587 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
588 }
589 }
590
591 // Initializing pipeline resources
592 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
593 waveStatusList.resize(numSIMDs);
594
595 for (int j = 0; j < numSIMDs; ++j) {
596 for (int i = 0; i < shader->n_wf; ++i) {
597 waveStatusList[j].push_back(
598 std::make_pair(wfList[j][i], BLOCKED));
599 }
600 }
601
602 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
603 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
604 }
605
606 fetchStage.init(this);
607 scoreboardCheckStage.init(this);
608 scheduleStage.init(this);
609 execStage.init(this);
610 globalMemoryPipe.init(this);
611 localMemoryPipe.init(this);
612 // initialize state for statistics calculation
613 vectorAluInstAvail.resize(numSIMDs, false);
614 shrMemInstAvail = 0;
615 glbMemInstAvail = 0;
616}
617
618bool
619ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
620{
621 // Ruby has completed the memory op. Schedule the mem_resp_event at the
622 // appropriate cycle to process the timing memory response
623 // This delay represents the pipeline delay
624 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
625 int index = sender_state->port_index;
626 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
627
628 // Is the packet returned a Kernel End or Barrier
629 if (pkt->req->isKernel() && pkt->req->isRelease()) {
630 Wavefront *w =
631 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
632
633 // Check if we are waiting on Kernel End Release
634 if (w->status == Wavefront::S_RETURNING) {
635 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
636 computeUnit->cu_id, w->simdId, w->wfSlotId,
637 w->wfDynId, w->kernId);
638
639 computeUnit->shader->dispatcher->notifyWgCompl(w);
640 w->status = Wavefront::S_STOPPED;
641 } else {
642 w->outstandingReqs--;
643 }
644
645 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
646 computeUnit->cu_id, gpuDynInst->simdId,
647 gpuDynInst->wfSlotId, w->barrierCnt);
648
649 if (gpuDynInst->useContinuation) {
650 assert(!gpuDynInst->isNoScope());
651 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
652 gpuDynInst);
653 }
654
655 delete pkt->senderState;
656 delete pkt->req;
657 delete pkt;
658 return true;
659 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
660 if (gpuDynInst->useContinuation) {
661 assert(!gpuDynInst->isNoScope());
662 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
663 gpuDynInst);
664 }
665
666 delete pkt->senderState;
667 delete pkt->req;
668 delete pkt;
669 return true;
670 }
671
672 EventFunctionWrapper *mem_resp_event =
673 computeUnit->memPort[index]->createMemRespEvent(pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
751 pkt->req->getFlags(), pkt->req->masterId(),
752 pkt->req->getPC());
753
754 // figure out the type of the request to set read/write
755 BaseTLB::Mode TLB_mode;
756 assert(pkt->isRead() || pkt->isWrite());
757
758 // Check write before read for atomic operations
759 // since atomic operations should use BaseTLB::Write
760 if (pkt->isWrite()){
761 TLB_mode = BaseTLB::Write;
762 } else if (pkt->isRead()) {
763 TLB_mode = BaseTLB::Read;
764 } else {
765 fatal("pkt is not a read nor a write\n");
766 }
767
768 tlbCycles -= curTick();
769 ++tlbRequests;
770
771 int tlbPort_index = perLaneTLB ? index : 0;
772
773 if (shader->timingSim) {
774 if (debugSegFault) {
775 Process *p = shader->gpuTc->getProcessPtr();
776 Addr vaddr = pkt->req->getVaddr();
777 unsigned size = pkt->getSize();
778
779 if ((vaddr + size - 1) % 64 < vaddr % 64) {
780 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
781 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
782 }
783
784 Addr paddr;
785
786 if (!p->pTable->translate(vaddr, paddr)) {
787 if (!p->fixupStackFault(vaddr)) {
788 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
789 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
790 vaddr);
791 }
792 }
793 }
794
795 // This is the SenderState needed upon return
796 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
797
798 // This is the senderState needed by the TLB hierarchy to function
799 TheISA::GpuTLB::TranslationState *translation_state =
800 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
801 pkt->senderState);
802
803 pkt->senderState = translation_state;
804
805 if (functionalTLB) {
806 tlbPort[tlbPort_index]->sendFunctional(pkt);
807
808 // update the hitLevel distribution
809 int hit_level = translation_state->hitLevel;
810 assert(hit_level != -1);
811 hitsPerTLBLevel[hit_level]++;
812
813 // New SenderState for the memory access
814 X86ISA::GpuTLB::TranslationState *sender_state =
815 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
816
817 delete sender_state->tlbEntry;
818 delete sender_state->saved;
819 delete sender_state;
820
821 assert(pkt->req->hasPaddr());
822 assert(pkt->req->hasSize());
823
824 uint8_t *tmpData = pkt->getPtr<uint8_t>();
825
826 // this is necessary because the GPU TLB receives packets instead
827 // of requests. when the translation is complete, all relevent
828 // fields in the request will be populated, but not in the packet.
829 // here we create the new packet so we can set the size, addr,
830 // and proper flags.
831 PacketPtr oldPkt = pkt;
832 pkt = new Packet(oldPkt->req, oldPkt->cmd);
833 delete oldPkt;
834 pkt->dataStatic(tmpData);
835
836
837 // New SenderState for the memory access
838 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
839 index, nullptr);
840
841 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
842 gpuDynInst->tlbHitLevel[index] = hit_level;
843
844
845 // translation is done. Schedule the mem_req_event at the
846 // appropriate cycle to send the timing memory request to ruby
847 EventFunctionWrapper *mem_req_event =
848 memPort[index]->createMemReqEvent(pkt);
849
850 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
851 "scheduled\n", cu_id, gpuDynInst->simdId,
852 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
853
854 schedule(mem_req_event, curTick() + req_tick_latency);
855 } else if (tlbPort[tlbPort_index]->isStalled()) {
856 assert(tlbPort[tlbPort_index]->retries.size() > 0);
857
858 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
859 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
860 tmp_vaddr);
861
862 tlbPort[tlbPort_index]->retries.push_back(pkt);
863 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
864 // Stall the data port;
865 // No more packet will be issued till
866 // ruby indicates resources are freed by
867 // a recvReqRetry() call back on this port.
868 tlbPort[tlbPort_index]->stallPort();
869
870 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
871 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
872 tmp_vaddr);
873
874 tlbPort[tlbPort_index]->retries.push_back(pkt);
875 } else {
876 DPRINTF(GPUTLB,
877 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
878 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
879 }
880 } else {
881 if (pkt->cmd == MemCmd::MemFenceReq) {
882 gpuDynInst->statusBitVector = VectorMask(0);
883 } else {
884 gpuDynInst->statusBitVector &= (~(1ll << index));
885 }
886
887 // New SenderState for the memory access
888 delete pkt->senderState;
889
890 // Because it's atomic operation, only need TLB translation state
891 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
892 shader->gpuTc);
893
894 tlbPort[tlbPort_index]->sendFunctional(pkt);
895
896 // the addr of the packet is not modified, so we need to create a new
897 // packet, or otherwise the memory access will have the old virtual
898 // address sent in the translation packet, instead of the physical
899 // address returned by the translation.
900 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
901 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
902
903 // Translation is done. It is safe to send the packet to memory.
904 memPort[0]->sendFunctional(new_pkt);
905
906 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
907 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
908 new_pkt->req->getPaddr());
909
910 // safe_cast the senderState
911 TheISA::GpuTLB::TranslationState *sender_state =
912 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
913
914 delete sender_state->tlbEntry;
915 delete new_pkt;
916 delete pkt->senderState;
917 delete pkt->req;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 EventFunctionWrapper *mem_req_event =
926 memPort[index]->createMemReqEvent(pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 Request* req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
948 }
949 req->setPaddr(0);
950 if (kernelLaunch) {
951 req->setFlags(Request::KERNEL);
952 }
953
954 // for non-kernel MemFence operations, memorder flags are set depending
955 // on which type of request is currently being sent, so this
956 // should be set by the caller (e.g. if an inst has acq-rel
957 // semantics, it will send one acquire req an one release req)
958 gpuDynInst->setRequestFlags(req, kernelLaunch);
959
960 // a mem fence must correspond to an acquire/release request
961 assert(req->isAcquire() || req->isRelease());
962
963 // create packet
964 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
965
966 // set packet's sender state
967 pkt->senderState =
968 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
969
970 // send the packet
971 sendSyncRequest(gpuDynInst, 0, pkt);
972}
973
974void
975ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt)
976{
977 DataPort::SenderState *sender_state =
978 safe_cast<DataPort::SenderState*>(pkt->senderState);
979
980 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
981 ComputeUnit *compute_unit = computeUnit;
982
983 assert(gpuDynInst);
984
985 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
986 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
987 pkt->req->getPaddr(), index);
988
989 Addr paddr = pkt->req->getPaddr();
990
991 if (pkt->cmd != MemCmd::MemFenceResp) {
992 int index = gpuDynInst->memStatusVector[paddr].back();
993
994 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
995 pkt->req->getPaddr(), index);
996
997 gpuDynInst->memStatusVector[paddr].pop_back();
998 gpuDynInst->pAddr = pkt->req->getPaddr();
999
1000 if (pkt->isRead() || pkt->isWrite()) {
1001
1002 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1003 gpuDynInst->statusBitVector &= (~(1ULL << index));
1004 } else {
1005 assert(gpuDynInst->statusVector[index] > 0);
1006 gpuDynInst->statusVector[index]--;
1007
1008 if (!gpuDynInst->statusVector[index])
1009 gpuDynInst->statusBitVector &= (~(1ULL << index));
1010 }
1011
1012 DPRINTF(GPUMem, "bitvector is now %#x\n",
1013 gpuDynInst->statusBitVector);
1014
1015 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1016 auto iter = gpuDynInst->memStatusVector.begin();
1017 auto end = gpuDynInst->memStatusVector.end();
1018
1019 while (iter != end) {
1020 assert(iter->second.empty());
1021 ++iter;
1022 }
1023
1024 gpuDynInst->memStatusVector.clear();
1025
1026 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1027 gpuDynInst->statusVector.clear();
1028
1029 compute_unit->globalMemoryPipe.handleResponse(gpuDynInst);
1030
1031 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1032 compute_unit->cu_id, gpuDynInst->simdId,
1033 gpuDynInst->wfSlotId);
1034
1035 // after clearing the status vectors,
1036 // see if there is a continuation to perform
1037 // the continuation may generate more work for
1038 // this memory request
1039 if (gpuDynInst->useContinuation) {
1040 assert(!gpuDynInst->isNoScope());
1041 gpuDynInst->execContinuation(
1042 gpuDynInst->staticInstruction(),
1043 gpuDynInst);
1044 }
1045 }
1046 }
1047 } else {
1048 gpuDynInst->statusBitVector = VectorMask(0);
1049
1050 if (gpuDynInst->useContinuation) {
1051 assert(!gpuDynInst->isNoScope());
1052 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1053 gpuDynInst);
1054 }
1055 }
1056
1057 delete pkt->senderState;
1058 delete pkt->req;
1059 delete pkt;
1060}
1061
1062ComputeUnit*
1063ComputeUnitParams::create()
1064{
1065 return new ComputeUnit(this);
1066}
1067
1068bool
1069ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1070{
1071 Addr line = pkt->req->getPaddr();
1072
1073 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1074 pkt->req->getVaddr(), line);
1075
1076 assert(pkt->senderState);
1077 computeUnit->tlbCycles += curTick();
1078
1079 // pop off the TLB translation state
1080 TheISA::GpuTLB::TranslationState *translation_state =
1081 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1082
1083 // no PageFaults are permitted for data accesses
1084 if (!translation_state->tlbEntry->valid) {
1085 DTLBPort::SenderState *sender_state =
1086 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1087
1088 Wavefront *w M5_VAR_USED =
1089 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1090 [sender_state->_gpuDynInst->wfSlotId];
1091
1092 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1093 pkt->req->getVaddr());
1094 }
1095
1096 assert(translation_state->tlbEntry->valid);
1097
1098 // update the hitLevel distribution
1099 int hit_level = translation_state->hitLevel;
1100 computeUnit->hitsPerTLBLevel[hit_level]++;
1101
1102 delete translation_state->tlbEntry;
1103 assert(!translation_state->ports.size());
1104 pkt->senderState = translation_state->saved;
1105
1106 // for prefetch pkt
1107 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1108
1109 delete translation_state;
1110
1111 // use the original sender state to know how to close this transaction
1112 DTLBPort::SenderState *sender_state =
1113 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1114
1115 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1116 int mp_index = sender_state->portIndex;
1117 Addr vaddr = pkt->req->getVaddr();
1118 gpuDynInst->memStatusVector[line].push_back(mp_index);
1119 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1120
1121 MemCmd requestCmd;
1122
1123 if (pkt->cmd == MemCmd::ReadResp) {
1124 requestCmd = MemCmd::ReadReq;
1125 } else if (pkt->cmd == MemCmd::WriteResp) {
1126 requestCmd = MemCmd::WriteReq;
1127 } else if (pkt->cmd == MemCmd::SwapResp) {
1128 requestCmd = MemCmd::SwapReq;
1129 } else {
1130 panic("unsupported response to request conversion %s\n",
1131 pkt->cmd.toString());
1132 }
1133
1134 if (computeUnit->prefetchDepth) {
1135 int simdId = gpuDynInst->simdId;
1136 int wfSlotId = gpuDynInst->wfSlotId;
1137 Addr last = 0;
1138
1139 switch(computeUnit->prefetchType) {
1140 case Enums::PF_CU:
1141 last = computeUnit->lastVaddrCU[mp_index];
1142 break;
1143 case Enums::PF_PHASE:
1144 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1145 break;
1146 case Enums::PF_WF:
1147 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1148 default:
1149 break;
1150 }
1151
1152 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1153 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1154
1155 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1156 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1157 : 0;
1158
1159 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1160
1161 computeUnit->lastVaddrCU[mp_index] = vaddr;
1162 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1163 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1164
1165 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1166 computeUnit->prefetchStride: stride;
1167
1168 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1169 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1170
1171 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1172
1173 // Prefetch Next few pages atomically
1174 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1175 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1176 vaddr+stride*pf*TheISA::PageBytes);
1177
1178 if (!stride)
1179 break;
1180
1181 Request *prefetch_req = new Request(0, vaddr + stride * pf *
1182 TheISA::PageBytes,
1183 sizeof(uint8_t), 0,
1184 computeUnit->masterId(),
1185 0, 0, 0);
1186
1187 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1188 uint8_t foo = 0;
1189 prefetch_pkt->dataStatic(&foo);
1190
1191 // Because it's atomic operation, only need TLB translation state
1192 prefetch_pkt->senderState =
1193 new TheISA::GpuTLB::TranslationState(TLB_mode,
1194 computeUnit->shader->gpuTc,
1195 true);
1196
1197 // Currently prefetches are zero-latency, hence the sendFunctional
1198 sendFunctional(prefetch_pkt);
1199
1200 /* safe_cast the senderState */
1201 TheISA::GpuTLB::TranslationState *tlb_state =
1202 safe_cast<TheISA::GpuTLB::TranslationState*>(
1203 prefetch_pkt->senderState);
1204
1205
1206 delete tlb_state->tlbEntry;
1207 delete tlb_state;
1208 delete prefetch_pkt->req;
1209 delete prefetch_pkt;
1210 }
1211 }
1212
1213 // First we must convert the response cmd back to a request cmd so that
1214 // the request can be sent through the cu's master port
1215 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1216 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1217 delete pkt->senderState;
1218 delete pkt;
1219
1220 // New SenderState for the memory access
1221 new_pkt->senderState =
1222 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1223 nullptr);
1224
1225 // translation is done. Schedule the mem_req_event at the appropriate
1226 // cycle to send the timing memory request to ruby
1227 EventFunctionWrapper *mem_req_event =
1228 computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt);
1229
1230 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1231 computeUnit->cu_id, gpuDynInst->simdId,
1232 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1233
1234 computeUnit->schedule(mem_req_event, curTick() +
1235 computeUnit->req_tick_latency);
1236
1237 return true;
1238}
1239
1240EventFunctionWrapper*
1241ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt)
1242{
1243 return new EventFunctionWrapper(
1244 [this, pkt]{ processMemReqEvent(pkt); },
1245 "ComputeUnit memory request event", true);
1246}
1247
1248EventFunctionWrapper*
1249ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt)
1250{
1251 return new EventFunctionWrapper(
1252 [this, pkt]{ processMemRespEvent(pkt); },
1253 "ComputeUnit memory response event", true);
1254}
1255
1256void
1257ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt)
1258{
1259 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1260 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1261 ComputeUnit *compute_unit M5_VAR_USED = computeUnit;
1262
1263 if (!(sendTimingReq(pkt))) {
1264 retries.push_back(std::make_pair(pkt, gpuDynInst));
1265
1266 DPRINTF(GPUPort,
1267 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1268 compute_unit->cu_id, gpuDynInst->simdId,
1269 gpuDynInst->wfSlotId, index,
1270 pkt->req->getPaddr());
1271 } else {
1272 DPRINTF(GPUPort,
1273 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1274 compute_unit->cu_id, gpuDynInst->simdId,
1275 gpuDynInst->wfSlotId, index,
1276 pkt->req->getPaddr());
1277 }
1278}
1279
1280/*
1281 * The initial translation request could have been rejected,
1282 * if <retries> queue is not Retry sending the translation
1283 * request. sendRetry() is called from the peer port whenever
1284 * a translation completes.
1285 */
1286void
1287ComputeUnit::DTLBPort::recvReqRetry()
1288{
1289 int len = retries.size();
1290
1291 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1292 computeUnit->cu_id, len);
1293
1294 assert(len > 0);
1295 assert(isStalled());
1296 // recvReqRetry is an indication that the resource on which this
1297 // port was stalling on is freed. So, remove the stall first
1298 unstallPort();
1299
1300 for (int i = 0; i < len; ++i) {
1301 PacketPtr pkt = retries.front();
1302 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1303 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1304
1305 if (!sendTimingReq(pkt)) {
1306 // Stall port
1307 stallPort();
1308 DPRINTF(GPUTLB, ": failed again\n");
1309 break;
1310 } else {
1311 DPRINTF(GPUTLB, ": successful\n");
1312 retries.pop_front();
1313 }
1314 }
1315}
1316
1317bool
1318ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1319{
1320 Addr line M5_VAR_USED = pkt->req->getPaddr();
1321 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1322 computeUnit->cu_id, pkt->req->getVaddr(), line);
1323
1324 assert(pkt->senderState);
1325
1326 // pop off the TLB translation state
1327 TheISA::GpuTLB::TranslationState *translation_state =
1328 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1329
1330 bool success = translation_state->tlbEntry->valid;
1331 delete translation_state->tlbEntry;
1332 assert(!translation_state->ports.size());
1333 pkt->senderState = translation_state->saved;
1334 delete translation_state;
1335
1336 // use the original sender state to know how to close this transaction
1337 ITLBPort::SenderState *sender_state =
1338 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1339
1340 // get the wavefront associated with this translation request
1341 Wavefront *wavefront = sender_state->wavefront;
1342 delete pkt->senderState;
1343
1344 if (success) {
1345 // pkt is reused in fetch(), don't delete it here. However, we must
1346 // reset the command to be a request so that it can be sent through
1347 // the cu's master port
1348 assert(pkt->cmd == MemCmd::ReadResp);
1349 pkt->cmd = MemCmd::ReadReq;
1350
1351 computeUnit->fetchStage.fetch(pkt, wavefront);
1352 } else {
1353 if (wavefront->dropFetch) {
1354 assert(wavefront->instructionBuffer.empty());
1355 wavefront->dropFetch = false;
1356 }
1357
1358 wavefront->pendingFetch = 0;
1359 }
1360
1361 return true;
1362}
1363
1364/*
1365 * The initial translation request could have been rejected, if
1366 * <retries> queue is not empty. Retry sending the translation
1367 * request. sendRetry() is called from the peer port whenever
1368 * a translation completes.
1369 */
1370void
1371ComputeUnit::ITLBPort::recvReqRetry()
1372{
1373
1374 int len = retries.size();
1375 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1376
1377 assert(len > 0);
1378 assert(isStalled());
1379
1380 // recvReqRetry is an indication that the resource on which this
1381 // port was stalling on is freed. So, remove the stall first
1382 unstallPort();
1383
1384 for (int i = 0; i < len; ++i) {
1385 PacketPtr pkt = retries.front();
1386 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1387 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1388
1389 if (!sendTimingReq(pkt)) {
1390 stallPort(); // Stall port
1391 DPRINTF(GPUTLB, ": failed again\n");
1392 break;
1393 } else {
1394 DPRINTF(GPUTLB, ": successful\n");
1395 retries.pop_front();
1396 }
1397 }
1398}
1399
1400void
1401ComputeUnit::regStats()
1402{
1403 MemObject::regStats();
1404
1405 vALUInsts
1406 .name(name() + ".valu_insts")
1407 .desc("Number of vector ALU insts issued.")
1408 ;
1409 vALUInstsPerWF
1410 .name(name() + ".valu_insts_per_wf")
1411 .desc("The avg. number of vector ALU insts issued per-wavefront.")
1412 ;
1413 sALUInsts
1414 .name(name() + ".salu_insts")
1415 .desc("Number of scalar ALU insts issued.")
1416 ;
1417 sALUInstsPerWF
1418 .name(name() + ".salu_insts_per_wf")
1419 .desc("The avg. number of scalar ALU insts issued per-wavefront.")
1420 ;
1421 instCyclesVALU
1422 .name(name() + ".inst_cycles_valu")
1423 .desc("Number of cycles needed to execute VALU insts.")
1424 ;
1425 instCyclesSALU
1426 .name(name() + ".inst_cycles_salu")
1427 .desc("Number of cycles needed to execute SALU insts.")
1428 ;
1429 threadCyclesVALU
1430 .name(name() + ".thread_cycles_valu")
1431 .desc("Number of thread cycles used to execute vector ALU ops. "
1432 "Similar to instCyclesVALU but multiplied by the number of "
1433 "active threads.")
1434 ;
1435 vALUUtilization
1436 .name(name() + ".valu_utilization")
1437 .desc("Percentage of active vector ALU threads in a wave.")
1438 ;
1439 ldsNoFlatInsts
1440 .name(name() + ".lds_no_flat_insts")
1441 .desc("Number of LDS insts issued, not including FLAT "
1442 "accesses that resolve to LDS.")
1443 ;
1444 ldsNoFlatInstsPerWF
1445 .name(name() + ".lds_no_flat_insts_per_wf")
1446 .desc("The avg. number of LDS insts (not including FLAT "
1447 "accesses that resolve to LDS) per-wavefront.")
1448 ;
1449 flatVMemInsts
1450 .name(name() + ".flat_vmem_insts")
1451 .desc("The number of FLAT insts that resolve to vmem issued.")
1452 ;
1453 flatVMemInstsPerWF
1454 .name(name() + ".flat_vmem_insts_per_wf")
1455 .desc("The average number of FLAT insts that resolve to vmem "
1456 "issued per-wavefront.")
1457 ;
1458 flatLDSInsts
1459 .name(name() + ".flat_lds_insts")
1460 .desc("The number of FLAT insts that resolve to LDS issued.")
1461 ;
1462 flatLDSInstsPerWF
1463 .name(name() + ".flat_lds_insts_per_wf")
1464 .desc("The average number of FLAT insts that resolve to LDS "
1465 "issued per-wavefront.")
1466 ;
1467 vectorMemWrites
1468 .name(name() + ".vector_mem_writes")
1469 .desc("Number of vector mem write insts (excluding FLAT insts).")
1470 ;
1471 vectorMemWritesPerWF
1472 .name(name() + ".vector_mem_writes_per_wf")
1473 .desc("The average number of vector mem write insts "
1474 "(excluding FLAT insts) per-wavefront.")
1475 ;
1476 vectorMemReads
1477 .name(name() + ".vector_mem_reads")
1478 .desc("Number of vector mem read insts (excluding FLAT insts).")
1479 ;
1480 vectorMemReadsPerWF
1481 .name(name() + ".vector_mem_reads_per_wf")
1482 .desc("The avg. number of vector mem read insts (excluding "
1483 "FLAT insts) per-wavefront.")
1484 ;
1485 scalarMemWrites
1486 .name(name() + ".scalar_mem_writes")
1487 .desc("Number of scalar mem write insts.")
1488 ;
1489 scalarMemWritesPerWF
1490 .name(name() + ".scalar_mem_writes_per_wf")
1491 .desc("The average number of scalar mem write insts per-wavefront.")
1492 ;
1493 scalarMemReads
1494 .name(name() + ".scalar_mem_reads")
1495 .desc("Number of scalar mem read insts.")
1496 ;
1497 scalarMemReadsPerWF
1498 .name(name() + ".scalar_mem_reads_per_wf")
1499 .desc("The average number of scalar mem read insts per-wavefront.")
1500 ;
1501
1502 vALUInstsPerWF = vALUInsts / completedWfs;
1503 sALUInstsPerWF = sALUInsts / completedWfs;
1504 vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100;
1505 ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs;
1506 flatVMemInstsPerWF = flatVMemInsts / completedWfs;
1507 flatLDSInstsPerWF = flatLDSInsts / completedWfs;
1508 vectorMemWritesPerWF = vectorMemWrites / completedWfs;
1509 vectorMemReadsPerWF = vectorMemReads / completedWfs;
1510 scalarMemWritesPerWF = scalarMemWrites / completedWfs;
1511 scalarMemReadsPerWF = scalarMemReads / completedWfs;
1512
1513 tlbCycles
1514 .name(name() + ".tlb_cycles")
1515 .desc("total number of cycles for all uncoalesced requests")
1516 ;
1517
1518 tlbRequests
1519 .name(name() + ".tlb_requests")
1520 .desc("number of uncoalesced requests")
1521 ;
1522
1523 tlbLatency
1524 .name(name() + ".avg_translation_latency")
1525 .desc("Avg. translation latency for data translations")
1526 ;
1527
1528 tlbLatency = tlbCycles / tlbRequests;
1529
1530 hitsPerTLBLevel
1531 .init(4)
1532 .name(name() + ".TLB_hits_distribution")
1533 .desc("TLB hits distribution (0 for page table, x for Lx-TLB")
1534 ;
1535
1536 // fixed number of TLB levels
1537 for (int i = 0; i < 4; ++i) {
1538 if (!i)
1539 hitsPerTLBLevel.subname(i,"page_table");
1540 else
1541 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i));
1542 }
1543
1544 execRateDist
1545 .init(0, 10, 2)
1546 .name(name() + ".inst_exec_rate")
1547 .desc("Instruction Execution Rate: Number of executed vector "
1548 "instructions per cycle")
1549 ;
1550
1551 ldsBankConflictDist
1552 .init(0, wfSize(), 2)
1553 .name(name() + ".lds_bank_conflicts")
1554 .desc("Number of bank conflicts per LDS memory packet")
1555 ;
1556
1557 ldsBankAccesses
1558 .name(name() + ".lds_bank_access_cnt")
1559 .desc("Total number of LDS bank accesses")
1560 ;
1561
1562 pageDivergenceDist
1563 // A wavefront can touch up to N pages per memory instruction where
1564 // N is equal to the wavefront size
1565 // The number of pages per bin can be configured (here it's 4).
1566 .init(1, wfSize(), 4)
1567 .name(name() + ".page_divergence_dist")
1568 .desc("pages touched per wf (over all mem. instr.)")
1569 ;
1570
1571 controlFlowDivergenceDist
1572 .init(1, wfSize(), 4)
1573 .name(name() + ".warp_execution_dist")
1574 .desc("number of lanes active per instruction (oval all instructions)")
1575 ;
1576
1577 activeLanesPerGMemInstrDist
1578 .init(1, wfSize(), 4)
1579 .name(name() + ".gmem_lanes_execution_dist")
1580 .desc("number of active lanes per global memory instruction")
1581 ;
1582
1583 activeLanesPerLMemInstrDist
1584 .init(1, wfSize(), 4)
1585 .name(name() + ".lmem_lanes_execution_dist")
1586 .desc("number of active lanes per local memory instruction")
1587 ;
1588
1589 numInstrExecuted
1590 .name(name() + ".num_instr_executed")
1591 .desc("number of instructions executed")
1592 ;
1593
1594 numVecOpsExecuted
1595 .name(name() + ".num_vec_ops_executed")
1596 .desc("number of vec ops executed (e.g. WF size/inst)")
1597 ;
1598
1599 totalCycles
1600 .name(name() + ".num_total_cycles")
1601 .desc("number of cycles the CU ran for")
1602 ;
1603
1604 ipc
1605 .name(name() + ".ipc")
1606 .desc("Instructions per cycle (this CU only)")
1607 ;
1608
1609 vpc
1610 .name(name() + ".vpc")
1611 .desc("Vector Operations per cycle (this CU only)")
1612 ;
1613
1614 numALUInstsExecuted
1615 .name(name() + ".num_alu_insts_executed")
1616 .desc("Number of dynamic non-GM memory insts executed")
1617 ;
1618
1619 wgBlockedDueLdsAllocation
1620 .name(name() + ".wg_blocked_due_lds_alloc")
1621 .desc("Workgroup blocked due to LDS capacity")
1622 ;
1623
1624 ipc = numInstrExecuted / totalCycles;
1625 vpc = numVecOpsExecuted / totalCycles;
1626
1627 numTimesWgBlockedDueVgprAlloc
1628 .name(name() + ".times_wg_blocked_due_vgpr_alloc")
1629 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD")
1630 ;
1631
1632 dynamicGMemInstrCnt
1633 .name(name() + ".global_mem_instr_cnt")
1634 .desc("dynamic global memory instructions count")
1635 ;
1636
1637 dynamicLMemInstrCnt
1638 .name(name() + ".local_mem_instr_cnt")
1639 .desc("dynamic local memory intruction count")
1640 ;
1641
1642 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt -
1643 dynamicLMemInstrCnt;
1644
1645 completedWfs
1646 .name(name() + ".num_completed_wfs")
1647 .desc("number of completed wavefronts")
1648 ;
1649
1650 numCASOps
1651 .name(name() + ".num_CAS_ops")
1652 .desc("number of compare and swap operations")
1653 ;
1654
1655 numFailedCASOps
1656 .name(name() + ".num_failed_CAS_ops")
1657 .desc("number of compare and swap operations that failed")
1658 ;
1659
1660 // register stats of pipeline stages
1661 fetchStage.regStats();
1662 scoreboardCheckStage.regStats();
1663 scheduleStage.regStats();
1664 execStage.regStats();
1665
1666 // register stats of memory pipeline
1667 globalMemoryPipe.regStats();
1668 localMemoryPipe.regStats();
1669}
1670
1671void
1672ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst)
1673{
1674 if (gpuDynInst->isScalar()) {
1675 if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) {
1676 sALUInsts++;
1677 instCyclesSALU++;
1678 } else if (gpuDynInst->isLoad()) {
1679 scalarMemReads++;
1680 } else if (gpuDynInst->isStore()) {
1681 scalarMemWrites++;
1682 }
1683 } else {
1684 if (gpuDynInst->isALU()) {
1685 vALUInsts++;
1686 instCyclesVALU++;
1687 threadCyclesVALU += gpuDynInst->wavefront()->execMask().count();
1688 } else if (gpuDynInst->isFlat()) {
1689 if (gpuDynInst->isLocalMem()) {
1690 flatLDSInsts++;
1691 } else {
1692 flatVMemInsts++;
1693 }
1694 } else if (gpuDynInst->isLocalMem()) {
1695 ldsNoFlatInsts++;
1696 } else if (gpuDynInst->isLoad()) {
1697 vectorMemReads++;
1698 } else if (gpuDynInst->isStore()) {
1699 vectorMemWrites++;
1700 }
1701 }
1702}
1703
1704void
1705ComputeUnit::updatePageDivergenceDist(Addr addr)
1706{
1707 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes);
1708
1709 if (!pagesTouched.count(virt_page_addr))
1710 pagesTouched[virt_page_addr] = 1;
1711 else
1712 pagesTouched[virt_page_addr]++;
1713}
1714
1715void
1716ComputeUnit::CUExitCallback::process()
1717{
1718 if (computeUnit->countPages) {
1719 std::ostream *page_stat_file =
1720 simout.create(computeUnit->name().c_str())->stream();
1721
1722 *page_stat_file << "page, wavefront accesses, workitem accesses" <<
1723 std::endl;
1724
1725 for (auto iter : computeUnit->pageAccesses) {
1726 *page_stat_file << std::hex << iter.first << ",";
1727 *page_stat_file << std::dec << iter.second.first << ",";
1728 *page_stat_file << std::dec << iter.second.second << std::endl;
1729 }
1730 }
1731 }
1732
1733bool
1734ComputeUnit::isDone() const
1735{
1736 for (int i = 0; i < numSIMDs; ++i) {
1737 if (!isSimdDone(i)) {
1738 return false;
1739 }
1740 }
1741
1742 bool glbMemBusRdy = true;
1743 for (int j = 0; j < numGlbMemUnits; ++j) {
1744 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy();
1745 }
1746 bool locMemBusRdy = true;
1747 for (int j = 0; j < numLocMemUnits; ++j) {
1748 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy();
1749 }
1750
1751 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() ||
1752 !globalMemoryPipe.isGMStRespFIFOWrRdy() ||
1753 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy()
1754 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() ||
1755 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) {
1756 return false;
1757 }
1758
1759 return true;
1760}
1761
1762int32_t
1763ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const
1764{
1765 return lds.getRefCounter(dispatchId, wgId);
1766}
1767
1768bool
1769ComputeUnit::isSimdDone(uint32_t simdId) const
1770{
1771 assert(simdId < numSIMDs);
1772
1773 for (int i=0; i < numGlbMemUnits; ++i) {
1774 if (!vrfToGlobalMemPipeBus[i].rdy())
1775 return false;
1776 }
1777 for (int i=0; i < numLocMemUnits; ++i) {
1778 if (!vrfToLocalMemPipeBus[i].rdy())
1779 return false;
1780 }
1781 if (!aluPipe[simdId].rdy()) {
1782 return false;
1783 }
1784
1785 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){
1786 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) {
1787 return false;
1788 }
1789 }
1790
1791 return true;
1792}
1793
1794/**
1795 * send a general request to the LDS
1796 * make sure to look at the return value here as your request might be
1797 * NACK'd and returning false means that you have to have some backup plan
1798 */
1799bool
1800ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst)
1801{
1802 // this is just a request to carry the GPUDynInstPtr
1803 // back and forth
1804 Request *newRequest = new Request();
1805 newRequest->setPaddr(0x0);
1806
1807 // ReadReq is not evaluted by the LDS but the Packet ctor requires this
1808 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq);
1809
1810 // This is the SenderState needed upon return
1811 newPacket->senderState = new LDSPort::SenderState(gpuDynInst);
1812
1813 return ldsPort->sendTimingReq(newPacket);
1814}
1815
1816/**
1817 * get the result of packets sent to the LDS when they return
1818 */
1819bool
1820ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet)
1821{
1822 const ComputeUnit::LDSPort::SenderState *senderState =
1823 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState);
1824
1825 fatal_if(!senderState, "did not get the right sort of sender state");
1826
1827 GPUDynInstPtr gpuDynInst = senderState->getMemInst();
1828
1829 delete packet->senderState;
1830 delete packet->req;
1831 delete packet;
1832
1833 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst);
1834 return true;
1835}
1836
1837/**
1838 * attempt to send this packet, either the port is already stalled, the request
1839 * is nack'd and must stall or the request goes through
1840 * when a request cannot be sent, add it to the retries queue
1841 */
1842bool
1843ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt)
1844{
1845 ComputeUnit::LDSPort::SenderState *sender_state =
1846 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState);
1847 fatal_if(!sender_state, "packet without a valid sender state");
1848
1849 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst();
1850
1851 if (isStalled()) {
1852 fatal_if(retries.empty(), "must have retries waiting to be stalled");
1853
1854 retries.push(pkt);
1855
1856 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n",
1857 computeUnit->cu_id, gpuDynInst->simdId,
1858 gpuDynInst->wfSlotId);
1859 return false;
1860 } else if (!MasterPort::sendTimingReq(pkt)) {
1861 // need to stall the LDS port until a recvReqRetry() is received
1862 // this indicates that there is more space
1863 stallPort();
1864 retries.push(pkt);
1865
1866 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n",
1867 computeUnit->cu_id, gpuDynInst->simdId,
1868 gpuDynInst->wfSlotId, pkt->req->getPaddr());
1869 return false;
1870 } else {
1871 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n",
1872 computeUnit->cu_id, gpuDynInst->simdId,
1873 gpuDynInst->wfSlotId, pkt->req->getPaddr());
1874 return true;
1875 }
1876}
1877
1878/**
1879 * the bus is telling the port that there is now space so retrying stalled
1880 * requests should work now
1881 * this allows the port to have a request be nack'd and then have the receiver
1882 * say when there is space, rather than simply retrying the send every cycle
1883 */
1884void
1885ComputeUnit::LDSPort::recvReqRetry()
1886{
1887 auto queueSize = retries.size();
1888
1889 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n",
1890 computeUnit->cu_id, queueSize);
1891
1892 fatal_if(queueSize < 1,
1893 "why was there a recvReqRetry() with no pending reqs?");
1894 fatal_if(!isStalled(),
1895 "recvReqRetry() happened when the port was not stalled");
1896
1897 unstallPort();
1898
1899 while (!retries.empty()) {
1900 PacketPtr packet = retries.front();
1901
1902 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id);
1903
1904 if (!MasterPort::sendTimingReq(packet)) {
1905 // Stall port
1906 stallPort();
1907 DPRINTF(GPUPort, ": LDS send failed again\n");
1908 break;
1909 } else {
1910 DPRINTF(GPUTLB, ": LDS send successful\n");
1911 retries.pop();
1912 }
1913 }
1914}
|