62 scoreboardCheckStage(p), scheduleStage(p), execStage(p),
63 globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
64 cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
65 spBypassPipeLength(p->spbypass_pipe_length),
66 dpBypassPipeLength(p->dpbypass_pipe_length),
67 issuePeriod(p->issue_period),
68 numGlbMemUnits(p->num_global_mem_pipes),
69 numLocMemUnits(p->num_shared_mem_pipes),
70 perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
71 prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
72 xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
73 functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
74 countPages(p->countPages), barrier_id(0),
75 vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
76 coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
77 req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
78 resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
79 _masterId(p->system->getMasterId(this, "ComputeUnit")),
80 lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
81 globalSeqNum(0), wavefrontSize(p->wfSize),
82 kernelLaunchInst(new KernelLaunchStaticInst())
83{
84 /**
85 * This check is necessary because std::bitset only provides conversion
86 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
87 * there are * a few places in the code where to_ullong() is used, however
88 * if VSZ is larger than a value the host can support then bitset will
89 * throw a runtime exception. we should remove all use of to_long() or
90 * to_ullong() so we can have VSZ greater than 64b, however until that is
91 * done this assert is required.
92 */
93 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
94 p->wfSize <= 0,
95 "WF size is larger than the host can support");
96 fatal_if(!isPowerOf2(wavefrontSize),
97 "Wavefront size should be a power of 2");
98 // calculate how many cycles a vector load or store will need to transfer
99 // its data over the corresponding buses
100 numCyclesPerStoreTransfer =
101 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
102 (double)vrfToCoalescerBusWidth);
103
104 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
105 / coalescerToVrfBusWidth;
106
107 lastVaddrWF.resize(numSIMDs);
108 wfList.resize(numSIMDs);
109
110 for (int j = 0; j < numSIMDs; ++j) {
111 lastVaddrWF[j].resize(p->n_wf);
112
113 for (int i = 0; i < p->n_wf; ++i) {
114 lastVaddrWF[j][i].resize(wfSize());
115
116 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
117 wfList[j][i]->setParent(this);
118
119 for (int k = 0; k < wfSize(); ++k) {
120 lastVaddrWF[j][i][k] = 0;
121 }
122 }
123 }
124
125 lastVaddrSimd.resize(numSIMDs);
126
127 for (int i = 0; i < numSIMDs; ++i) {
128 lastVaddrSimd[i].resize(wfSize(), 0);
129 }
130
131 lastVaddrCU.resize(wfSize());
132
133 lds.setParent(this);
134
135 if (p->execPolicy == "OLDEST-FIRST") {
136 exec_policy = EXEC_POLICY::OLDEST;
137 } else if (p->execPolicy == "ROUND-ROBIN") {
138 exec_policy = EXEC_POLICY::RR;
139 } else {
140 fatal("Invalid WF execution policy (CU)\n");
141 }
142
143 memPort.resize(wfSize());
144
145 // resize the tlbPort vectorArray
146 int tlbPort_width = perLaneTLB ? wfSize() : 1;
147 tlbPort.resize(tlbPort_width);
148
149 cuExitCallback = new CUExitCallback(this);
150 registerExitCallback(cuExitCallback);
151
152 xactCasLoadMap.clear();
153 lastExecCycle.resize(numSIMDs, 0);
154
155 for (int i = 0; i < vrf.size(); ++i) {
156 vrf[i]->setParent(this);
157 }
158
159 numVecRegsPerSimd = vrf[0]->numRegs();
160}
161
162ComputeUnit::~ComputeUnit()
163{
164 // Delete wavefront slots
165 for (int j = 0; j < numSIMDs; ++j) {
166 for (int i = 0; i < shader->n_wf; ++i) {
167 delete wfList[j][i];
168 }
169 lastVaddrSimd[j].clear();
170 }
171 lastVaddrCU.clear();
172 readyList.clear();
173 waveStatusList.clear();
174 dispatchList.clear();
175 vectorAluInstAvail.clear();
176 delete cuExitCallback;
177 delete ldsPort;
178}
179
180void
181ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
182{
183 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
184
185 w->workGroupSz[0] = ndr->q.wgSize[0];
186 w->workGroupSz[1] = ndr->q.wgSize[1];
187 w->workGroupSz[2] = ndr->q.wgSize[2];
188 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
189 w->gridSz[0] = ndr->q.gdSize[0];
190 w->gridSz[1] = ndr->q.gdSize[1];
191 w->gridSz[2] = ndr->q.gdSize[2];
192 w->kernelArgs = ndr->q.args;
193 w->privSizePerItem = ndr->q.privMemPerItem;
194 w->spillSizePerItem = ndr->q.spillMemPerItem;
195 w->roBase = ndr->q.roMemStart;
196 w->roSize = ndr->q.roMemTotal;
197 w->computeActualWgSz(ndr);
198}
199
200void
201ComputeUnit::updateEvents() {
202
203 if (!timestampVec.empty()) {
204 uint32_t vecSize = timestampVec.size();
205 uint32_t i = 0;
206 while (i < vecSize) {
207 if (timestampVec[i] <= shader->tick_cnt) {
208 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
209 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
210 statusVec[i]);
211 timestampVec.erase(timestampVec.begin() + i);
212 regIdxVec.erase(regIdxVec.begin() + i);
213 statusVec.erase(statusVec.begin() + i);
214 --vecSize;
215 --i;
216 }
217 ++i;
218 }
219 }
220
221 for (int i = 0; i< numSIMDs; ++i) {
222 vrf[i]->updateEvents();
223 }
224}
225
226
227void
228ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
229 NDRange *ndr)
230{
231 static int _n_wave = 0;
232
233 VectorMask init_mask;
234 init_mask.reset();
235
236 for (int k = 0; k < wfSize(); ++k) {
237 if (k + waveId * wfSize() < w->actualWgSzTotal)
238 init_mask[k] = 1;
239 }
240
241 w->kernId = ndr->dispatchId;
242 w->wfId = waveId;
243 w->initMask = init_mask.to_ullong();
244
245 for (int k = 0; k < wfSize(); ++k) {
246 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
247 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
248 w->actualWgSz[1];
249 w->workItemId[2][k] = (k + waveId * wfSize()) /
250 (w->actualWgSz[0] * w->actualWgSz[1]);
251
252 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
253 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
254 w->workItemId[0][k];
255 }
256
257 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
258
259 w->barCnt.resize(wfSize(), 0);
260
261 w->maxBarCnt = 0;
262 w->oldBarrierCnt = 0;
263 w->barrierCnt = 0;
264
265 w->privBase = ndr->q.privMemStart;
266 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
267
268 w->spillBase = ndr->q.spillMemStart;
269 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
270
271 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
272
273 // WG state
274 w->wgId = ndr->globalWgId;
275 w->dispatchId = ndr->dispatchId;
276 w->workGroupId[0] = w->wgId % ndr->numWg[0];
277 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
278 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
279
280 w->barrierId = barrier_id;
281 w->stalledAtBarrier = false;
282
283 // set the wavefront context to have a pointer to this section of the LDS
284 w->ldsChunk = ldsChunk;
285
286 int32_t refCount M5_VAR_USED =
287 lds.increaseRefCounter(w->dispatchId, w->wgId);
288 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
289 cu_id, w->wgId, refCount);
290
291 w->instructionBuffer.clear();
292
293 if (w->pendingFetch)
294 w->dropFetch = true;
295
296 // is this the last wavefront in the workgroup
297 // if set the spillWidth to be the remaining work-items
298 // so that the vector access is correct
299 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
300 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
301 } else {
302 w->spillWidth = wfSize();
303 }
304
305 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
306 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
307
308 w->start(++_n_wave, ndr->q.code_ptr);
309}
310
311void
312ComputeUnit::StartWorkgroup(NDRange *ndr)
313{
314 // reserve the LDS capacity allocated to the work group
315 // disambiguated by the dispatch ID and workgroup ID, which should be
316 // globally unique
317 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
318 ndr->q.ldsSize);
319
320 // Send L1 cache acquire
321 // isKernel + isAcquire = Kernel Begin
322 if (shader->impl_kern_boundary_sync) {
323 GPUDynInstPtr gpuDynInst =
324 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
325 getAndIncSeqNum());
326
327 gpuDynInst->useContinuation = false;
328 injectGlobalMemFence(gpuDynInst, true);
329 }
330
331 // calculate the number of 32-bit vector registers required by wavefront
332 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
333 int wave_id = 0;
334
335 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
336 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
337 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
338 // Check if this wavefront slot is available:
339 // It must be stopped and not waiting
340 // for a release to complete S_RETURNING
341 if (w->status == Wavefront::S_STOPPED) {
342 fillKernelState(w, ndr);
343 // if we have scheduled all work items then stop
344 // scheduling wavefronts
345 if (wave_id * wfSize() >= w->actualWgSzTotal)
346 break;
347
348 // reserve vector registers for the scheduled wavefront
349 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
350 uint32_t normSize = 0;
351
352 w->startVgprIndex = vrf[m % numSIMDs]->manager->
353 allocateRegion(vregDemand, &normSize);
354
355 w->reservedVectorRegs = normSize;
356 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
357
358 startWavefront(w, wave_id, ldsChunk, ndr);
359 ++wave_id;
360 }
361 }
362 ++barrier_id;
363}
364
365int
366ComputeUnit::ReadyWorkgroup(NDRange *ndr)
367{
368 // Get true size of workgroup (after clamping to grid size)
369 int trueWgSize[3];
370 int trueWgSizeTotal = 1;
371
372 for (int d = 0; d < 3; ++d) {
373 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
374 ndr->wgId[d] * ndr->q.wgSize[d]);
375
376 trueWgSizeTotal *= trueWgSize[d];
377 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
378 }
379
380 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
381
382 // calculate the number of 32-bit vector registers required by each
383 // work item of the work group
384 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
385 bool vregAvail = true;
386 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
387 int freeWfSlots = 0;
388 // check if the total number of VGPRs required by all WFs of the WG
389 // fit in the VRFs of all SIMD units
390 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
391 int numMappedWfs = 0;
392 std::vector<int> numWfsPerSimd;
393 numWfsPerSimd.resize(numSIMDs, 0);
394 // find how many free WF slots we have across all SIMDs
395 for (int j = 0; j < shader->n_wf; ++j) {
396 for (int i = 0; i < numSIMDs; ++i) {
397 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
398 // count the number of free WF slots
399 ++freeWfSlots;
400 if (numMappedWfs < numWfs) {
401 // count the WFs to be assigned per SIMD
402 numWfsPerSimd[i]++;
403 }
404 numMappedWfs++;
405 }
406 }
407 }
408
409 // if there are enough free WF slots then find if there are enough
410 // free VGPRs per SIMD based on the WF->SIMD mapping
411 if (freeWfSlots >= numWfs) {
412 for (int j = 0; j < numSIMDs; ++j) {
413 // find if there are enough free VGPR regions in the SIMD's VRF
414 // to accommodate the WFs of the new WG that would be mapped to
415 // this SIMD unit
416 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
417 vregDemandPerWI);
418
419 // stop searching if there is at least one SIMD
420 // whose VRF does not have enough free VGPR pools.
421 // This is because a WG is scheduled only if ALL
422 // of its WFs can be scheduled
423 if (!vregAvail)
424 break;
425 }
426 }
427
428 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
429 freeWfSlots, vregAvail);
430
431 if (!vregAvail) {
432 ++numTimesWgBlockedDueVgprAlloc;
433 }
434
435 // Return true if enough WF slots to submit workgroup and if there are
436 // enough VGPRs to schedule all WFs to their SIMD units
437 if (!lds.canReserve(ndr->q.ldsSize)) {
438 wgBlockedDueLdsAllocation++;
439 }
440
441 // Return true if (a) there are enough free WF slots to submit
442 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
443 // SIMD units and (c) if there is enough space in LDS
444 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
445}
446
447int
448ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
449{
450 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
451 int ccnt = 0;
452
453 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
454 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
455 Wavefront *w = wfList[i_simd][i_wf];
456
457 if (w->status == Wavefront::S_RUNNING) {
458 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
459
460 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
461 w->barrierId, _barrier_id);
462
463 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
464 w->barrierCnt, bcnt);
465 }
466
467 if (w->status == Wavefront::S_RUNNING &&
468 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
469 !w->outstandingReqs) {
470 ++ccnt;
471
472 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
473 "%d\n", i_simd, i_wf, ccnt);
474 }
475 }
476 }
477
478 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
479 cu_id, ccnt, bslots);
480
481 return ccnt == bslots;
482}
483
484// Check if the current wavefront is blocked on additional resources.
485bool
486ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
487{
488 bool cede = false;
489
490 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
491 // magic instructions will impact the scheduling of wavefronts
492 if (xact_cas_mode) {
493 /*
494 * When a wavefront calls xact_cas_ld, it adds itself to a per address
495 * queue. All per address queues are managed by the xactCasLoadMap.
496 *
497 * A wavefront is not blocked if: it is not in ANY per address queue or
498 * if it is at the head of a per address queue.
499 */
500 for (auto itMap : xactCasLoadMap) {
501 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
502
503 if (!curWaveIDQueue.empty()) {
504 for (auto it : curWaveIDQueue) {
505 waveIdentifier cur_wave = it;
506
507 if (cur_wave.simdId == simdId &&
508 cur_wave.wfSlotId == wfSlotId) {
509 // 2 possibilities
510 // 1: this WF has a green light
511 // 2: another WF has a green light
512 waveIdentifier owner_wave = curWaveIDQueue.front();
513
514 if (owner_wave.simdId != cur_wave.simdId ||
515 owner_wave.wfSlotId != cur_wave.wfSlotId) {
516 // possibility 2
517 cede = true;
518 break;
519 } else {
520 // possibility 1
521 break;
522 }
523 }
524 }
525 }
526 }
527 }
528
529 return cede;
530}
531
532// Execute one clock worth of work on the ComputeUnit.
533void
534ComputeUnit::exec()
535{
536 updateEvents();
537 // Execute pipeline stages in reverse order to simulate
538 // the pipeline latency
539 globalMemoryPipe.exec();
540 localMemoryPipe.exec();
541 execStage.exec();
542 scheduleStage.exec();
543 scoreboardCheckStage.exec();
544 fetchStage.exec();
545
546 totalCycles++;
547}
548
549void
550ComputeUnit::init()
551{
552 // Initialize CU Bus models
553 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
554 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
555 nextGlbMemBus = 0;
556 nextLocMemBus = 0;
557 fatal_if(numGlbMemUnits > 1,
558 "No support for multiple Global Memory Pipelines exists!!!");
559 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
560 for (int j = 0; j < numGlbMemUnits; ++j) {
561 vrfToGlobalMemPipeBus[j] = WaitClass();
562 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
563 }
564
565 fatal_if(numLocMemUnits > 1,
566 "No support for multiple Local Memory Pipelines exists!!!");
567 vrfToLocalMemPipeBus.resize(numLocMemUnits);
568 for (int j = 0; j < numLocMemUnits; ++j) {
569 vrfToLocalMemPipeBus[j] = WaitClass();
570 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
571 }
572 vectorRegsReserved.resize(numSIMDs, 0);
573 aluPipe.resize(numSIMDs);
574 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
575
576 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
577 wfWait[i] = WaitClass();
578 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
579 }
580
581 for (int i = 0; i < numSIMDs; ++i) {
582 aluPipe[i] = WaitClass();
583 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
584 }
585
586 // Setup space for call args
587 for (int j = 0; j < numSIMDs; ++j) {
588 for (int i = 0; i < shader->n_wf; ++i) {
589 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
590 }
591 }
592
593 // Initializing pipeline resources
594 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
595 waveStatusList.resize(numSIMDs);
596
597 for (int j = 0; j < numSIMDs; ++j) {
598 for (int i = 0; i < shader->n_wf; ++i) {
599 waveStatusList[j].push_back(
600 std::make_pair(wfList[j][i], BLOCKED));
601 }
602 }
603
604 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
605 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
606 }
607
608 fetchStage.init(this);
609 scoreboardCheckStage.init(this);
610 scheduleStage.init(this);
611 execStage.init(this);
612 globalMemoryPipe.init(this);
613 localMemoryPipe.init(this);
614 // initialize state for statistics calculation
615 vectorAluInstAvail.resize(numSIMDs, false);
616 shrMemInstAvail = 0;
617 glbMemInstAvail = 0;
618}
619
620bool
621ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
622{
623 // Ruby has completed the memory op. Schedule the mem_resp_event at the
624 // appropriate cycle to process the timing memory response
625 // This delay represents the pipeline delay
626 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
627 int index = sender_state->port_index;
628 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
629
630 // Is the packet returned a Kernel End or Barrier
631 if (pkt->req->isKernel() && pkt->req->isRelease()) {
632 Wavefront *w =
633 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
634
635 // Check if we are waiting on Kernel End Release
636 if (w->status == Wavefront::S_RETURNING) {
637 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
638 computeUnit->cu_id, w->simdId, w->wfSlotId,
639 w->wfDynId, w->kernId);
640
641 computeUnit->shader->dispatcher->notifyWgCompl(w);
642 w->status = Wavefront::S_STOPPED;
643 } else {
644 w->outstandingReqs--;
645 }
646
647 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
648 computeUnit->cu_id, gpuDynInst->simdId,
649 gpuDynInst->wfSlotId, w->barrierCnt);
650
651 if (gpuDynInst->useContinuation) {
652 assert(!gpuDynInst->isNoScope());
653 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
654 gpuDynInst);
655 }
656
657 delete pkt->senderState;
658 delete pkt;
659 return true;
660 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
661 if (gpuDynInst->useContinuation) {
662 assert(!gpuDynInst->isNoScope());
663 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
664 gpuDynInst);
665 }
666
667 delete pkt->senderState;
668 delete pkt;
669 return true;
670 }
671
672 EventFunctionWrapper *mem_resp_event =
673 computeUnit->memPort[index]->createMemRespEvent(pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 // set PC in request
751 pkt->req->setPC(gpuDynInst->wavefront()->pc());
752
753 pkt->req->setReqInstSeqNum(gpuDynInst->seqNum());
754
755 // figure out the type of the request to set read/write
756 BaseTLB::Mode TLB_mode;
757 assert(pkt->isRead() || pkt->isWrite());
758
759 // Check write before read for atomic operations
760 // since atomic operations should use BaseTLB::Write
761 if (pkt->isWrite()){
762 TLB_mode = BaseTLB::Write;
763 } else if (pkt->isRead()) {
764 TLB_mode = BaseTLB::Read;
765 } else {
766 fatal("pkt is not a read nor a write\n");
767 }
768
769 tlbCycles -= curTick();
770 ++tlbRequests;
771
772 int tlbPort_index = perLaneTLB ? index : 0;
773
774 if (shader->timingSim) {
775 if (debugSegFault) {
776 Process *p = shader->gpuTc->getProcessPtr();
777 Addr vaddr = pkt->req->getVaddr();
778 unsigned size = pkt->getSize();
779
780 if ((vaddr + size - 1) % 64 < vaddr % 64) {
781 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
782 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
783 }
784
785 Addr paddr;
786
787 if (!p->pTable->translate(vaddr, paddr)) {
788 if (!p->fixupStackFault(vaddr)) {
789 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
790 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
791 vaddr);
792 }
793 }
794 }
795
796 // This is the SenderState needed upon return
797 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
798
799 // This is the senderState needed by the TLB hierarchy to function
800 TheISA::GpuTLB::TranslationState *translation_state =
801 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
802 pkt->senderState);
803
804 pkt->senderState = translation_state;
805
806 if (functionalTLB) {
807 tlbPort[tlbPort_index]->sendFunctional(pkt);
808
809 // update the hitLevel distribution
810 int hit_level = translation_state->hitLevel;
811 assert(hit_level != -1);
812 hitsPerTLBLevel[hit_level]++;
813
814 // New SenderState for the memory access
815 X86ISA::GpuTLB::TranslationState *sender_state =
816 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
817
818 delete sender_state->tlbEntry;
819 delete sender_state->saved;
820 delete sender_state;
821
822 assert(pkt->req->hasPaddr());
823 assert(pkt->req->hasSize());
824
825 uint8_t *tmpData = pkt->getPtr<uint8_t>();
826
827 // this is necessary because the GPU TLB receives packets instead
828 // of requests. when the translation is complete, all relevent
829 // fields in the request will be populated, but not in the packet.
830 // here we create the new packet so we can set the size, addr,
831 // and proper flags.
832 PacketPtr oldPkt = pkt;
833 pkt = new Packet(oldPkt->req, oldPkt->cmd);
834 delete oldPkt;
835 pkt->dataStatic(tmpData);
836
837
838 // New SenderState for the memory access
839 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
840 index, nullptr);
841
842 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
843 gpuDynInst->tlbHitLevel[index] = hit_level;
844
845
846 // translation is done. Schedule the mem_req_event at the
847 // appropriate cycle to send the timing memory request to ruby
848 EventFunctionWrapper *mem_req_event =
849 memPort[index]->createMemReqEvent(pkt);
850
851 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
852 "scheduled\n", cu_id, gpuDynInst->simdId,
853 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
854
855 schedule(mem_req_event, curTick() + req_tick_latency);
856 } else if (tlbPort[tlbPort_index]->isStalled()) {
857 assert(tlbPort[tlbPort_index]->retries.size() > 0);
858
859 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
860 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
861 tmp_vaddr);
862
863 tlbPort[tlbPort_index]->retries.push_back(pkt);
864 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
865 // Stall the data port;
866 // No more packet will be issued till
867 // ruby indicates resources are freed by
868 // a recvReqRetry() call back on this port.
869 tlbPort[tlbPort_index]->stallPort();
870
871 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
872 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
873 tmp_vaddr);
874
875 tlbPort[tlbPort_index]->retries.push_back(pkt);
876 } else {
877 DPRINTF(GPUTLB,
878 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
879 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
880 }
881 } else {
882 if (pkt->cmd == MemCmd::MemFenceReq) {
883 gpuDynInst->statusBitVector = VectorMask(0);
884 } else {
885 gpuDynInst->statusBitVector &= (~(1ll << index));
886 }
887
888 // New SenderState for the memory access
889 delete pkt->senderState;
890
891 // Because it's atomic operation, only need TLB translation state
892 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
893 shader->gpuTc);
894
895 tlbPort[tlbPort_index]->sendFunctional(pkt);
896
897 // the addr of the packet is not modified, so we need to create a new
898 // packet, or otherwise the memory access will have the old virtual
899 // address sent in the translation packet, instead of the physical
900 // address returned by the translation.
901 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
902 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
903
904 // Translation is done. It is safe to send the packet to memory.
905 memPort[0]->sendFunctional(new_pkt);
906
907 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
908 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
909 new_pkt->req->getPaddr());
910
911 // safe_cast the senderState
912 TheISA::GpuTLB::TranslationState *sender_state =
913 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
914
915 delete sender_state->tlbEntry;
916 delete new_pkt;
917 delete pkt->senderState;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 EventFunctionWrapper *mem_req_event =
926 memPort[index]->createMemReqEvent(pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 RequestPtr req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = std::make_shared<Request>(
948 0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
949 }
950 req->setPaddr(0);
951 if (kernelLaunch) {
952 req->setFlags(Request::KERNEL);
953 }
954
955 // for non-kernel MemFence operations, memorder flags are set depending
956 // on which type of request is currently being sent, so this
957 // should be set by the caller (e.g. if an inst has acq-rel
958 // semantics, it will send one acquire req an one release req)
959 gpuDynInst->setRequestFlags(req, kernelLaunch);
960
961 // a mem fence must correspond to an acquire/release request
962 assert(req->isAcquire() || req->isRelease());
963
964 // create packet
965 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
966
967 // set packet's sender state
968 pkt->senderState =
969 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
970
971 // send the packet
972 sendSyncRequest(gpuDynInst, 0, pkt);
973}
974
975void
976ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt)
977{
978 DataPort::SenderState *sender_state =
979 safe_cast<DataPort::SenderState*>(pkt->senderState);
980
981 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
982 ComputeUnit *compute_unit = computeUnit;
983
984 assert(gpuDynInst);
985
986 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
987 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
988 pkt->req->getPaddr(), index);
989
990 Addr paddr = pkt->req->getPaddr();
991
992 if (pkt->cmd != MemCmd::MemFenceResp) {
993 int index = gpuDynInst->memStatusVector[paddr].back();
994
995 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
996 pkt->req->getPaddr(), index);
997
998 gpuDynInst->memStatusVector[paddr].pop_back();
999 gpuDynInst->pAddr = pkt->req->getPaddr();
1000
1001 if (pkt->isRead() || pkt->isWrite()) {
1002
1003 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1004 gpuDynInst->statusBitVector &= (~(1ULL << index));
1005 } else {
1006 assert(gpuDynInst->statusVector[index] > 0);
1007 gpuDynInst->statusVector[index]--;
1008
1009 if (!gpuDynInst->statusVector[index])
1010 gpuDynInst->statusBitVector &= (~(1ULL << index));
1011 }
1012
1013 DPRINTF(GPUMem, "bitvector is now %#x\n",
1014 gpuDynInst->statusBitVector);
1015
1016 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1017 auto iter = gpuDynInst->memStatusVector.begin();
1018 auto end = gpuDynInst->memStatusVector.end();
1019
1020 while (iter != end) {
1021 assert(iter->second.empty());
1022 ++iter;
1023 }
1024
1025 gpuDynInst->memStatusVector.clear();
1026
1027 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1028 gpuDynInst->statusVector.clear();
1029
1030 compute_unit->globalMemoryPipe.handleResponse(gpuDynInst);
1031
1032 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1033 compute_unit->cu_id, gpuDynInst->simdId,
1034 gpuDynInst->wfSlotId);
1035
1036 // after clearing the status vectors,
1037 // see if there is a continuation to perform
1038 // the continuation may generate more work for
1039 // this memory request
1040 if (gpuDynInst->useContinuation) {
1041 assert(!gpuDynInst->isNoScope());
1042 gpuDynInst->execContinuation(
1043 gpuDynInst->staticInstruction(),
1044 gpuDynInst);
1045 }
1046 }
1047 }
1048 } else {
1049 gpuDynInst->statusBitVector = VectorMask(0);
1050
1051 if (gpuDynInst->useContinuation) {
1052 assert(!gpuDynInst->isNoScope());
1053 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1054 gpuDynInst);
1055 }
1056 }
1057
1058 delete pkt->senderState;
1059 delete pkt;
1060}
1061
1062ComputeUnit*
1063ComputeUnitParams::create()
1064{
1065 return new ComputeUnit(this);
1066}
1067
1068bool
1069ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1070{
1071 Addr line = pkt->req->getPaddr();
1072
1073 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1074 pkt->req->getVaddr(), line);
1075
1076 assert(pkt->senderState);
1077 computeUnit->tlbCycles += curTick();
1078
1079 // pop off the TLB translation state
1080 TheISA::GpuTLB::TranslationState *translation_state =
1081 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1082
1083 // no PageFaults are permitted for data accesses
1084 if (!translation_state->tlbEntry) {
1085 DTLBPort::SenderState *sender_state =
1086 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1087
1088 Wavefront *w M5_VAR_USED =
1089 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1090 [sender_state->_gpuDynInst->wfSlotId];
1091
1092 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1093 pkt->req->getVaddr());
1094 }
1095
1096 // update the hitLevel distribution
1097 int hit_level = translation_state->hitLevel;
1098 computeUnit->hitsPerTLBLevel[hit_level]++;
1099
1100 delete translation_state->tlbEntry;
1101 assert(!translation_state->ports.size());
1102 pkt->senderState = translation_state->saved;
1103
1104 // for prefetch pkt
1105 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1106
1107 delete translation_state;
1108
1109 // use the original sender state to know how to close this transaction
1110 DTLBPort::SenderState *sender_state =
1111 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1112
1113 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1114 int mp_index = sender_state->portIndex;
1115 Addr vaddr = pkt->req->getVaddr();
1116 gpuDynInst->memStatusVector[line].push_back(mp_index);
1117 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1118
1119 MemCmd requestCmd;
1120
1121 if (pkt->cmd == MemCmd::ReadResp) {
1122 requestCmd = MemCmd::ReadReq;
1123 } else if (pkt->cmd == MemCmd::WriteResp) {
1124 requestCmd = MemCmd::WriteReq;
1125 } else if (pkt->cmd == MemCmd::SwapResp) {
1126 requestCmd = MemCmd::SwapReq;
1127 } else {
1128 panic("unsupported response to request conversion %s\n",
1129 pkt->cmd.toString());
1130 }
1131
1132 if (computeUnit->prefetchDepth) {
1133 int simdId = gpuDynInst->simdId;
1134 int wfSlotId = gpuDynInst->wfSlotId;
1135 Addr last = 0;
1136
1137 switch(computeUnit->prefetchType) {
1138 case Enums::PF_CU:
1139 last = computeUnit->lastVaddrCU[mp_index];
1140 break;
1141 case Enums::PF_PHASE:
1142 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1143 break;
1144 case Enums::PF_WF:
1145 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1146 default:
1147 break;
1148 }
1149
1150 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1151 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1152
1153 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1154 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1155 : 0;
1156
1157 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1158
1159 computeUnit->lastVaddrCU[mp_index] = vaddr;
1160 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1161 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1162
1163 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1164 computeUnit->prefetchStride: stride;
1165
1166 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1167 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1168
1169 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1170
1171 // Prefetch Next few pages atomically
1172 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1173 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1174 vaddr+stride*pf*TheISA::PageBytes);
1175
1176 if (!stride)
1177 break;
1178
1179 RequestPtr prefetch_req = std::make_shared<Request>(
1180 0, vaddr + stride * pf * TheISA::PageBytes,
1181 sizeof(uint8_t), 0,
1182 computeUnit->masterId(),
1183 0, 0, nullptr);
1184
1185 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1186 uint8_t foo = 0;
1187 prefetch_pkt->dataStatic(&foo);
1188
1189 // Because it's atomic operation, only need TLB translation state
1190 prefetch_pkt->senderState =
1191 new TheISA::GpuTLB::TranslationState(TLB_mode,
1192 computeUnit->shader->gpuTc,
1193 true);
1194
1195 // Currently prefetches are zero-latency, hence the sendFunctional
1196 sendFunctional(prefetch_pkt);
1197
1198 /* safe_cast the senderState */
1199 TheISA::GpuTLB::TranslationState *tlb_state =
1200 safe_cast<TheISA::GpuTLB::TranslationState*>(
1201 prefetch_pkt->senderState);
1202
1203
1204 delete tlb_state->tlbEntry;
1205 delete tlb_state;
1206 delete prefetch_pkt;
1207 }
1208 }
1209
1210 // First we must convert the response cmd back to a request cmd so that
1211 // the request can be sent through the cu's master port
1212 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1213 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1214 delete pkt->senderState;
1215 delete pkt;
1216
1217 // New SenderState for the memory access
1218 new_pkt->senderState =
1219 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1220 nullptr);
1221
1222 // translation is done. Schedule the mem_req_event at the appropriate
1223 // cycle to send the timing memory request to ruby
1224 EventFunctionWrapper *mem_req_event =
1225 computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt);
1226
1227 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1228 computeUnit->cu_id, gpuDynInst->simdId,
1229 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1230
1231 computeUnit->schedule(mem_req_event, curTick() +
1232 computeUnit->req_tick_latency);
1233
1234 return true;
1235}
1236
1237EventFunctionWrapper*
1238ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt)
1239{
1240 return new EventFunctionWrapper(
1241 [this, pkt]{ processMemReqEvent(pkt); },
1242 "ComputeUnit memory request event", true);
1243}
1244
1245EventFunctionWrapper*
1246ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt)
1247{
1248 return new EventFunctionWrapper(
1249 [this, pkt]{ processMemRespEvent(pkt); },
1250 "ComputeUnit memory response event", true);
1251}
1252
1253void
1254ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt)
1255{
1256 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1257 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1258 ComputeUnit *compute_unit M5_VAR_USED = computeUnit;
1259
1260 if (!(sendTimingReq(pkt))) {
1261 retries.push_back(std::make_pair(pkt, gpuDynInst));
1262
1263 DPRINTF(GPUPort,
1264 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1265 compute_unit->cu_id, gpuDynInst->simdId,
1266 gpuDynInst->wfSlotId, index,
1267 pkt->req->getPaddr());
1268 } else {
1269 DPRINTF(GPUPort,
1270 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1271 compute_unit->cu_id, gpuDynInst->simdId,
1272 gpuDynInst->wfSlotId, index,
1273 pkt->req->getPaddr());
1274 }
1275}
1276
1277/*
1278 * The initial translation request could have been rejected,
1279 * if <retries> queue is not Retry sending the translation
1280 * request. sendRetry() is called from the peer port whenever
1281 * a translation completes.
1282 */
1283void
1284ComputeUnit::DTLBPort::recvReqRetry()
1285{
1286 int len = retries.size();
1287
1288 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1289 computeUnit->cu_id, len);
1290
1291 assert(len > 0);
1292 assert(isStalled());
1293 // recvReqRetry is an indication that the resource on which this
1294 // port was stalling on is freed. So, remove the stall first
1295 unstallPort();
1296
1297 for (int i = 0; i < len; ++i) {
1298 PacketPtr pkt = retries.front();
1299 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1300 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1301
1302 if (!sendTimingReq(pkt)) {
1303 // Stall port
1304 stallPort();
1305 DPRINTF(GPUTLB, ": failed again\n");
1306 break;
1307 } else {
1308 DPRINTF(GPUTLB, ": successful\n");
1309 retries.pop_front();
1310 }
1311 }
1312}
1313
1314bool
1315ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1316{
1317 Addr line M5_VAR_USED = pkt->req->getPaddr();
1318 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1319 computeUnit->cu_id, pkt->req->getVaddr(), line);
1320
1321 assert(pkt->senderState);
1322
1323 // pop off the TLB translation state
1324 TheISA::GpuTLB::TranslationState *translation_state =
1325 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1326
1327 bool success = translation_state->tlbEntry != nullptr;
1328 delete translation_state->tlbEntry;
1329 assert(!translation_state->ports.size());
1330 pkt->senderState = translation_state->saved;
1331 delete translation_state;
1332
1333 // use the original sender state to know how to close this transaction
1334 ITLBPort::SenderState *sender_state =
1335 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1336
1337 // get the wavefront associated with this translation request
1338 Wavefront *wavefront = sender_state->wavefront;
1339 delete pkt->senderState;
1340
1341 if (success) {
1342 // pkt is reused in fetch(), don't delete it here. However, we must
1343 // reset the command to be a request so that it can be sent through
1344 // the cu's master port
1345 assert(pkt->cmd == MemCmd::ReadResp);
1346 pkt->cmd = MemCmd::ReadReq;
1347
1348 computeUnit->fetchStage.fetch(pkt, wavefront);
1349 } else {
1350 if (wavefront->dropFetch) {
1351 assert(wavefront->instructionBuffer.empty());
1352 wavefront->dropFetch = false;
1353 }
1354
1355 wavefront->pendingFetch = 0;
1356 }
1357
1358 return true;
1359}
1360
1361/*
1362 * The initial translation request could have been rejected, if
1363 * <retries> queue is not empty. Retry sending the translation
1364 * request. sendRetry() is called from the peer port whenever
1365 * a translation completes.
1366 */
1367void
1368ComputeUnit::ITLBPort::recvReqRetry()
1369{
1370
1371 int len = retries.size();
1372 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1373
1374 assert(len > 0);
1375 assert(isStalled());
1376
1377 // recvReqRetry is an indication that the resource on which this
1378 // port was stalling on is freed. So, remove the stall first
1379 unstallPort();
1380
1381 for (int i = 0; i < len; ++i) {
1382 PacketPtr pkt = retries.front();
1383 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1384 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1385
1386 if (!sendTimingReq(pkt)) {
1387 stallPort(); // Stall port
1388 DPRINTF(GPUTLB, ": failed again\n");
1389 break;
1390 } else {
1391 DPRINTF(GPUTLB, ": successful\n");
1392 retries.pop_front();
1393 }
1394 }
1395}
1396
1397void
1398ComputeUnit::regStats()
1399{
| 62 scoreboardCheckStage(p), scheduleStage(p), execStage(p),
63 globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
64 cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
65 spBypassPipeLength(p->spbypass_pipe_length),
66 dpBypassPipeLength(p->dpbypass_pipe_length),
67 issuePeriod(p->issue_period),
68 numGlbMemUnits(p->num_global_mem_pipes),
69 numLocMemUnits(p->num_shared_mem_pipes),
70 perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
71 prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
72 xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
73 functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
74 countPages(p->countPages), barrier_id(0),
75 vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
76 coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
77 req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
78 resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
79 _masterId(p->system->getMasterId(this, "ComputeUnit")),
80 lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
81 globalSeqNum(0), wavefrontSize(p->wfSize),
82 kernelLaunchInst(new KernelLaunchStaticInst())
83{
84 /**
85 * This check is necessary because std::bitset only provides conversion
86 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
87 * there are * a few places in the code where to_ullong() is used, however
88 * if VSZ is larger than a value the host can support then bitset will
89 * throw a runtime exception. we should remove all use of to_long() or
90 * to_ullong() so we can have VSZ greater than 64b, however until that is
91 * done this assert is required.
92 */
93 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
94 p->wfSize <= 0,
95 "WF size is larger than the host can support");
96 fatal_if(!isPowerOf2(wavefrontSize),
97 "Wavefront size should be a power of 2");
98 // calculate how many cycles a vector load or store will need to transfer
99 // its data over the corresponding buses
100 numCyclesPerStoreTransfer =
101 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
102 (double)vrfToCoalescerBusWidth);
103
104 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
105 / coalescerToVrfBusWidth;
106
107 lastVaddrWF.resize(numSIMDs);
108 wfList.resize(numSIMDs);
109
110 for (int j = 0; j < numSIMDs; ++j) {
111 lastVaddrWF[j].resize(p->n_wf);
112
113 for (int i = 0; i < p->n_wf; ++i) {
114 lastVaddrWF[j][i].resize(wfSize());
115
116 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
117 wfList[j][i]->setParent(this);
118
119 for (int k = 0; k < wfSize(); ++k) {
120 lastVaddrWF[j][i][k] = 0;
121 }
122 }
123 }
124
125 lastVaddrSimd.resize(numSIMDs);
126
127 for (int i = 0; i < numSIMDs; ++i) {
128 lastVaddrSimd[i].resize(wfSize(), 0);
129 }
130
131 lastVaddrCU.resize(wfSize());
132
133 lds.setParent(this);
134
135 if (p->execPolicy == "OLDEST-FIRST") {
136 exec_policy = EXEC_POLICY::OLDEST;
137 } else if (p->execPolicy == "ROUND-ROBIN") {
138 exec_policy = EXEC_POLICY::RR;
139 } else {
140 fatal("Invalid WF execution policy (CU)\n");
141 }
142
143 memPort.resize(wfSize());
144
145 // resize the tlbPort vectorArray
146 int tlbPort_width = perLaneTLB ? wfSize() : 1;
147 tlbPort.resize(tlbPort_width);
148
149 cuExitCallback = new CUExitCallback(this);
150 registerExitCallback(cuExitCallback);
151
152 xactCasLoadMap.clear();
153 lastExecCycle.resize(numSIMDs, 0);
154
155 for (int i = 0; i < vrf.size(); ++i) {
156 vrf[i]->setParent(this);
157 }
158
159 numVecRegsPerSimd = vrf[0]->numRegs();
160}
161
162ComputeUnit::~ComputeUnit()
163{
164 // Delete wavefront slots
165 for (int j = 0; j < numSIMDs; ++j) {
166 for (int i = 0; i < shader->n_wf; ++i) {
167 delete wfList[j][i];
168 }
169 lastVaddrSimd[j].clear();
170 }
171 lastVaddrCU.clear();
172 readyList.clear();
173 waveStatusList.clear();
174 dispatchList.clear();
175 vectorAluInstAvail.clear();
176 delete cuExitCallback;
177 delete ldsPort;
178}
179
180void
181ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
182{
183 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
184
185 w->workGroupSz[0] = ndr->q.wgSize[0];
186 w->workGroupSz[1] = ndr->q.wgSize[1];
187 w->workGroupSz[2] = ndr->q.wgSize[2];
188 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
189 w->gridSz[0] = ndr->q.gdSize[0];
190 w->gridSz[1] = ndr->q.gdSize[1];
191 w->gridSz[2] = ndr->q.gdSize[2];
192 w->kernelArgs = ndr->q.args;
193 w->privSizePerItem = ndr->q.privMemPerItem;
194 w->spillSizePerItem = ndr->q.spillMemPerItem;
195 w->roBase = ndr->q.roMemStart;
196 w->roSize = ndr->q.roMemTotal;
197 w->computeActualWgSz(ndr);
198}
199
200void
201ComputeUnit::updateEvents() {
202
203 if (!timestampVec.empty()) {
204 uint32_t vecSize = timestampVec.size();
205 uint32_t i = 0;
206 while (i < vecSize) {
207 if (timestampVec[i] <= shader->tick_cnt) {
208 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
209 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
210 statusVec[i]);
211 timestampVec.erase(timestampVec.begin() + i);
212 regIdxVec.erase(regIdxVec.begin() + i);
213 statusVec.erase(statusVec.begin() + i);
214 --vecSize;
215 --i;
216 }
217 ++i;
218 }
219 }
220
221 for (int i = 0; i< numSIMDs; ++i) {
222 vrf[i]->updateEvents();
223 }
224}
225
226
227void
228ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
229 NDRange *ndr)
230{
231 static int _n_wave = 0;
232
233 VectorMask init_mask;
234 init_mask.reset();
235
236 for (int k = 0; k < wfSize(); ++k) {
237 if (k + waveId * wfSize() < w->actualWgSzTotal)
238 init_mask[k] = 1;
239 }
240
241 w->kernId = ndr->dispatchId;
242 w->wfId = waveId;
243 w->initMask = init_mask.to_ullong();
244
245 for (int k = 0; k < wfSize(); ++k) {
246 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
247 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
248 w->actualWgSz[1];
249 w->workItemId[2][k] = (k + waveId * wfSize()) /
250 (w->actualWgSz[0] * w->actualWgSz[1]);
251
252 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
253 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
254 w->workItemId[0][k];
255 }
256
257 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
258
259 w->barCnt.resize(wfSize(), 0);
260
261 w->maxBarCnt = 0;
262 w->oldBarrierCnt = 0;
263 w->barrierCnt = 0;
264
265 w->privBase = ndr->q.privMemStart;
266 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
267
268 w->spillBase = ndr->q.spillMemStart;
269 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
270
271 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
272
273 // WG state
274 w->wgId = ndr->globalWgId;
275 w->dispatchId = ndr->dispatchId;
276 w->workGroupId[0] = w->wgId % ndr->numWg[0];
277 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
278 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
279
280 w->barrierId = barrier_id;
281 w->stalledAtBarrier = false;
282
283 // set the wavefront context to have a pointer to this section of the LDS
284 w->ldsChunk = ldsChunk;
285
286 int32_t refCount M5_VAR_USED =
287 lds.increaseRefCounter(w->dispatchId, w->wgId);
288 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
289 cu_id, w->wgId, refCount);
290
291 w->instructionBuffer.clear();
292
293 if (w->pendingFetch)
294 w->dropFetch = true;
295
296 // is this the last wavefront in the workgroup
297 // if set the spillWidth to be the remaining work-items
298 // so that the vector access is correct
299 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
300 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
301 } else {
302 w->spillWidth = wfSize();
303 }
304
305 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
306 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
307
308 w->start(++_n_wave, ndr->q.code_ptr);
309}
310
311void
312ComputeUnit::StartWorkgroup(NDRange *ndr)
313{
314 // reserve the LDS capacity allocated to the work group
315 // disambiguated by the dispatch ID and workgroup ID, which should be
316 // globally unique
317 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
318 ndr->q.ldsSize);
319
320 // Send L1 cache acquire
321 // isKernel + isAcquire = Kernel Begin
322 if (shader->impl_kern_boundary_sync) {
323 GPUDynInstPtr gpuDynInst =
324 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
325 getAndIncSeqNum());
326
327 gpuDynInst->useContinuation = false;
328 injectGlobalMemFence(gpuDynInst, true);
329 }
330
331 // calculate the number of 32-bit vector registers required by wavefront
332 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
333 int wave_id = 0;
334
335 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
336 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
337 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
338 // Check if this wavefront slot is available:
339 // It must be stopped and not waiting
340 // for a release to complete S_RETURNING
341 if (w->status == Wavefront::S_STOPPED) {
342 fillKernelState(w, ndr);
343 // if we have scheduled all work items then stop
344 // scheduling wavefronts
345 if (wave_id * wfSize() >= w->actualWgSzTotal)
346 break;
347
348 // reserve vector registers for the scheduled wavefront
349 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
350 uint32_t normSize = 0;
351
352 w->startVgprIndex = vrf[m % numSIMDs]->manager->
353 allocateRegion(vregDemand, &normSize);
354
355 w->reservedVectorRegs = normSize;
356 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
357
358 startWavefront(w, wave_id, ldsChunk, ndr);
359 ++wave_id;
360 }
361 }
362 ++barrier_id;
363}
364
365int
366ComputeUnit::ReadyWorkgroup(NDRange *ndr)
367{
368 // Get true size of workgroup (after clamping to grid size)
369 int trueWgSize[3];
370 int trueWgSizeTotal = 1;
371
372 for (int d = 0; d < 3; ++d) {
373 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
374 ndr->wgId[d] * ndr->q.wgSize[d]);
375
376 trueWgSizeTotal *= trueWgSize[d];
377 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
378 }
379
380 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
381
382 // calculate the number of 32-bit vector registers required by each
383 // work item of the work group
384 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
385 bool vregAvail = true;
386 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
387 int freeWfSlots = 0;
388 // check if the total number of VGPRs required by all WFs of the WG
389 // fit in the VRFs of all SIMD units
390 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
391 int numMappedWfs = 0;
392 std::vector<int> numWfsPerSimd;
393 numWfsPerSimd.resize(numSIMDs, 0);
394 // find how many free WF slots we have across all SIMDs
395 for (int j = 0; j < shader->n_wf; ++j) {
396 for (int i = 0; i < numSIMDs; ++i) {
397 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
398 // count the number of free WF slots
399 ++freeWfSlots;
400 if (numMappedWfs < numWfs) {
401 // count the WFs to be assigned per SIMD
402 numWfsPerSimd[i]++;
403 }
404 numMappedWfs++;
405 }
406 }
407 }
408
409 // if there are enough free WF slots then find if there are enough
410 // free VGPRs per SIMD based on the WF->SIMD mapping
411 if (freeWfSlots >= numWfs) {
412 for (int j = 0; j < numSIMDs; ++j) {
413 // find if there are enough free VGPR regions in the SIMD's VRF
414 // to accommodate the WFs of the new WG that would be mapped to
415 // this SIMD unit
416 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
417 vregDemandPerWI);
418
419 // stop searching if there is at least one SIMD
420 // whose VRF does not have enough free VGPR pools.
421 // This is because a WG is scheduled only if ALL
422 // of its WFs can be scheduled
423 if (!vregAvail)
424 break;
425 }
426 }
427
428 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
429 freeWfSlots, vregAvail);
430
431 if (!vregAvail) {
432 ++numTimesWgBlockedDueVgprAlloc;
433 }
434
435 // Return true if enough WF slots to submit workgroup and if there are
436 // enough VGPRs to schedule all WFs to their SIMD units
437 if (!lds.canReserve(ndr->q.ldsSize)) {
438 wgBlockedDueLdsAllocation++;
439 }
440
441 // Return true if (a) there are enough free WF slots to submit
442 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
443 // SIMD units and (c) if there is enough space in LDS
444 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
445}
446
447int
448ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
449{
450 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
451 int ccnt = 0;
452
453 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
454 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
455 Wavefront *w = wfList[i_simd][i_wf];
456
457 if (w->status == Wavefront::S_RUNNING) {
458 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
459
460 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
461 w->barrierId, _barrier_id);
462
463 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
464 w->barrierCnt, bcnt);
465 }
466
467 if (w->status == Wavefront::S_RUNNING &&
468 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
469 !w->outstandingReqs) {
470 ++ccnt;
471
472 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
473 "%d\n", i_simd, i_wf, ccnt);
474 }
475 }
476 }
477
478 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
479 cu_id, ccnt, bslots);
480
481 return ccnt == bslots;
482}
483
484// Check if the current wavefront is blocked on additional resources.
485bool
486ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
487{
488 bool cede = false;
489
490 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
491 // magic instructions will impact the scheduling of wavefronts
492 if (xact_cas_mode) {
493 /*
494 * When a wavefront calls xact_cas_ld, it adds itself to a per address
495 * queue. All per address queues are managed by the xactCasLoadMap.
496 *
497 * A wavefront is not blocked if: it is not in ANY per address queue or
498 * if it is at the head of a per address queue.
499 */
500 for (auto itMap : xactCasLoadMap) {
501 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
502
503 if (!curWaveIDQueue.empty()) {
504 for (auto it : curWaveIDQueue) {
505 waveIdentifier cur_wave = it;
506
507 if (cur_wave.simdId == simdId &&
508 cur_wave.wfSlotId == wfSlotId) {
509 // 2 possibilities
510 // 1: this WF has a green light
511 // 2: another WF has a green light
512 waveIdentifier owner_wave = curWaveIDQueue.front();
513
514 if (owner_wave.simdId != cur_wave.simdId ||
515 owner_wave.wfSlotId != cur_wave.wfSlotId) {
516 // possibility 2
517 cede = true;
518 break;
519 } else {
520 // possibility 1
521 break;
522 }
523 }
524 }
525 }
526 }
527 }
528
529 return cede;
530}
531
532// Execute one clock worth of work on the ComputeUnit.
533void
534ComputeUnit::exec()
535{
536 updateEvents();
537 // Execute pipeline stages in reverse order to simulate
538 // the pipeline latency
539 globalMemoryPipe.exec();
540 localMemoryPipe.exec();
541 execStage.exec();
542 scheduleStage.exec();
543 scoreboardCheckStage.exec();
544 fetchStage.exec();
545
546 totalCycles++;
547}
548
549void
550ComputeUnit::init()
551{
552 // Initialize CU Bus models
553 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
554 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
555 nextGlbMemBus = 0;
556 nextLocMemBus = 0;
557 fatal_if(numGlbMemUnits > 1,
558 "No support for multiple Global Memory Pipelines exists!!!");
559 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
560 for (int j = 0; j < numGlbMemUnits; ++j) {
561 vrfToGlobalMemPipeBus[j] = WaitClass();
562 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
563 }
564
565 fatal_if(numLocMemUnits > 1,
566 "No support for multiple Local Memory Pipelines exists!!!");
567 vrfToLocalMemPipeBus.resize(numLocMemUnits);
568 for (int j = 0; j < numLocMemUnits; ++j) {
569 vrfToLocalMemPipeBus[j] = WaitClass();
570 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
571 }
572 vectorRegsReserved.resize(numSIMDs, 0);
573 aluPipe.resize(numSIMDs);
574 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
575
576 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
577 wfWait[i] = WaitClass();
578 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
579 }
580
581 for (int i = 0; i < numSIMDs; ++i) {
582 aluPipe[i] = WaitClass();
583 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
584 }
585
586 // Setup space for call args
587 for (int j = 0; j < numSIMDs; ++j) {
588 for (int i = 0; i < shader->n_wf; ++i) {
589 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
590 }
591 }
592
593 // Initializing pipeline resources
594 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
595 waveStatusList.resize(numSIMDs);
596
597 for (int j = 0; j < numSIMDs; ++j) {
598 for (int i = 0; i < shader->n_wf; ++i) {
599 waveStatusList[j].push_back(
600 std::make_pair(wfList[j][i], BLOCKED));
601 }
602 }
603
604 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
605 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
606 }
607
608 fetchStage.init(this);
609 scoreboardCheckStage.init(this);
610 scheduleStage.init(this);
611 execStage.init(this);
612 globalMemoryPipe.init(this);
613 localMemoryPipe.init(this);
614 // initialize state for statistics calculation
615 vectorAluInstAvail.resize(numSIMDs, false);
616 shrMemInstAvail = 0;
617 glbMemInstAvail = 0;
618}
619
620bool
621ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
622{
623 // Ruby has completed the memory op. Schedule the mem_resp_event at the
624 // appropriate cycle to process the timing memory response
625 // This delay represents the pipeline delay
626 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
627 int index = sender_state->port_index;
628 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
629
630 // Is the packet returned a Kernel End or Barrier
631 if (pkt->req->isKernel() && pkt->req->isRelease()) {
632 Wavefront *w =
633 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
634
635 // Check if we are waiting on Kernel End Release
636 if (w->status == Wavefront::S_RETURNING) {
637 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
638 computeUnit->cu_id, w->simdId, w->wfSlotId,
639 w->wfDynId, w->kernId);
640
641 computeUnit->shader->dispatcher->notifyWgCompl(w);
642 w->status = Wavefront::S_STOPPED;
643 } else {
644 w->outstandingReqs--;
645 }
646
647 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
648 computeUnit->cu_id, gpuDynInst->simdId,
649 gpuDynInst->wfSlotId, w->barrierCnt);
650
651 if (gpuDynInst->useContinuation) {
652 assert(!gpuDynInst->isNoScope());
653 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
654 gpuDynInst);
655 }
656
657 delete pkt->senderState;
658 delete pkt;
659 return true;
660 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
661 if (gpuDynInst->useContinuation) {
662 assert(!gpuDynInst->isNoScope());
663 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
664 gpuDynInst);
665 }
666
667 delete pkt->senderState;
668 delete pkt;
669 return true;
670 }
671
672 EventFunctionWrapper *mem_resp_event =
673 computeUnit->memPort[index]->createMemRespEvent(pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 // set PC in request
751 pkt->req->setPC(gpuDynInst->wavefront()->pc());
752
753 pkt->req->setReqInstSeqNum(gpuDynInst->seqNum());
754
755 // figure out the type of the request to set read/write
756 BaseTLB::Mode TLB_mode;
757 assert(pkt->isRead() || pkt->isWrite());
758
759 // Check write before read for atomic operations
760 // since atomic operations should use BaseTLB::Write
761 if (pkt->isWrite()){
762 TLB_mode = BaseTLB::Write;
763 } else if (pkt->isRead()) {
764 TLB_mode = BaseTLB::Read;
765 } else {
766 fatal("pkt is not a read nor a write\n");
767 }
768
769 tlbCycles -= curTick();
770 ++tlbRequests;
771
772 int tlbPort_index = perLaneTLB ? index : 0;
773
774 if (shader->timingSim) {
775 if (debugSegFault) {
776 Process *p = shader->gpuTc->getProcessPtr();
777 Addr vaddr = pkt->req->getVaddr();
778 unsigned size = pkt->getSize();
779
780 if ((vaddr + size - 1) % 64 < vaddr % 64) {
781 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
782 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
783 }
784
785 Addr paddr;
786
787 if (!p->pTable->translate(vaddr, paddr)) {
788 if (!p->fixupStackFault(vaddr)) {
789 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
790 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
791 vaddr);
792 }
793 }
794 }
795
796 // This is the SenderState needed upon return
797 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
798
799 // This is the senderState needed by the TLB hierarchy to function
800 TheISA::GpuTLB::TranslationState *translation_state =
801 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
802 pkt->senderState);
803
804 pkt->senderState = translation_state;
805
806 if (functionalTLB) {
807 tlbPort[tlbPort_index]->sendFunctional(pkt);
808
809 // update the hitLevel distribution
810 int hit_level = translation_state->hitLevel;
811 assert(hit_level != -1);
812 hitsPerTLBLevel[hit_level]++;
813
814 // New SenderState for the memory access
815 X86ISA::GpuTLB::TranslationState *sender_state =
816 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
817
818 delete sender_state->tlbEntry;
819 delete sender_state->saved;
820 delete sender_state;
821
822 assert(pkt->req->hasPaddr());
823 assert(pkt->req->hasSize());
824
825 uint8_t *tmpData = pkt->getPtr<uint8_t>();
826
827 // this is necessary because the GPU TLB receives packets instead
828 // of requests. when the translation is complete, all relevent
829 // fields in the request will be populated, but not in the packet.
830 // here we create the new packet so we can set the size, addr,
831 // and proper flags.
832 PacketPtr oldPkt = pkt;
833 pkt = new Packet(oldPkt->req, oldPkt->cmd);
834 delete oldPkt;
835 pkt->dataStatic(tmpData);
836
837
838 // New SenderState for the memory access
839 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
840 index, nullptr);
841
842 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
843 gpuDynInst->tlbHitLevel[index] = hit_level;
844
845
846 // translation is done. Schedule the mem_req_event at the
847 // appropriate cycle to send the timing memory request to ruby
848 EventFunctionWrapper *mem_req_event =
849 memPort[index]->createMemReqEvent(pkt);
850
851 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
852 "scheduled\n", cu_id, gpuDynInst->simdId,
853 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
854
855 schedule(mem_req_event, curTick() + req_tick_latency);
856 } else if (tlbPort[tlbPort_index]->isStalled()) {
857 assert(tlbPort[tlbPort_index]->retries.size() > 0);
858
859 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
860 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
861 tmp_vaddr);
862
863 tlbPort[tlbPort_index]->retries.push_back(pkt);
864 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
865 // Stall the data port;
866 // No more packet will be issued till
867 // ruby indicates resources are freed by
868 // a recvReqRetry() call back on this port.
869 tlbPort[tlbPort_index]->stallPort();
870
871 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
872 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
873 tmp_vaddr);
874
875 tlbPort[tlbPort_index]->retries.push_back(pkt);
876 } else {
877 DPRINTF(GPUTLB,
878 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
879 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
880 }
881 } else {
882 if (pkt->cmd == MemCmd::MemFenceReq) {
883 gpuDynInst->statusBitVector = VectorMask(0);
884 } else {
885 gpuDynInst->statusBitVector &= (~(1ll << index));
886 }
887
888 // New SenderState for the memory access
889 delete pkt->senderState;
890
891 // Because it's atomic operation, only need TLB translation state
892 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
893 shader->gpuTc);
894
895 tlbPort[tlbPort_index]->sendFunctional(pkt);
896
897 // the addr of the packet is not modified, so we need to create a new
898 // packet, or otherwise the memory access will have the old virtual
899 // address sent in the translation packet, instead of the physical
900 // address returned by the translation.
901 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
902 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
903
904 // Translation is done. It is safe to send the packet to memory.
905 memPort[0]->sendFunctional(new_pkt);
906
907 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
908 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
909 new_pkt->req->getPaddr());
910
911 // safe_cast the senderState
912 TheISA::GpuTLB::TranslationState *sender_state =
913 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
914
915 delete sender_state->tlbEntry;
916 delete new_pkt;
917 delete pkt->senderState;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 EventFunctionWrapper *mem_req_event =
926 memPort[index]->createMemReqEvent(pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 RequestPtr req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = std::make_shared<Request>(
948 0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
949 }
950 req->setPaddr(0);
951 if (kernelLaunch) {
952 req->setFlags(Request::KERNEL);
953 }
954
955 // for non-kernel MemFence operations, memorder flags are set depending
956 // on which type of request is currently being sent, so this
957 // should be set by the caller (e.g. if an inst has acq-rel
958 // semantics, it will send one acquire req an one release req)
959 gpuDynInst->setRequestFlags(req, kernelLaunch);
960
961 // a mem fence must correspond to an acquire/release request
962 assert(req->isAcquire() || req->isRelease());
963
964 // create packet
965 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
966
967 // set packet's sender state
968 pkt->senderState =
969 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
970
971 // send the packet
972 sendSyncRequest(gpuDynInst, 0, pkt);
973}
974
975void
976ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt)
977{
978 DataPort::SenderState *sender_state =
979 safe_cast<DataPort::SenderState*>(pkt->senderState);
980
981 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
982 ComputeUnit *compute_unit = computeUnit;
983
984 assert(gpuDynInst);
985
986 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
987 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
988 pkt->req->getPaddr(), index);
989
990 Addr paddr = pkt->req->getPaddr();
991
992 if (pkt->cmd != MemCmd::MemFenceResp) {
993 int index = gpuDynInst->memStatusVector[paddr].back();
994
995 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
996 pkt->req->getPaddr(), index);
997
998 gpuDynInst->memStatusVector[paddr].pop_back();
999 gpuDynInst->pAddr = pkt->req->getPaddr();
1000
1001 if (pkt->isRead() || pkt->isWrite()) {
1002
1003 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1004 gpuDynInst->statusBitVector &= (~(1ULL << index));
1005 } else {
1006 assert(gpuDynInst->statusVector[index] > 0);
1007 gpuDynInst->statusVector[index]--;
1008
1009 if (!gpuDynInst->statusVector[index])
1010 gpuDynInst->statusBitVector &= (~(1ULL << index));
1011 }
1012
1013 DPRINTF(GPUMem, "bitvector is now %#x\n",
1014 gpuDynInst->statusBitVector);
1015
1016 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1017 auto iter = gpuDynInst->memStatusVector.begin();
1018 auto end = gpuDynInst->memStatusVector.end();
1019
1020 while (iter != end) {
1021 assert(iter->second.empty());
1022 ++iter;
1023 }
1024
1025 gpuDynInst->memStatusVector.clear();
1026
1027 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1028 gpuDynInst->statusVector.clear();
1029
1030 compute_unit->globalMemoryPipe.handleResponse(gpuDynInst);
1031
1032 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1033 compute_unit->cu_id, gpuDynInst->simdId,
1034 gpuDynInst->wfSlotId);
1035
1036 // after clearing the status vectors,
1037 // see if there is a continuation to perform
1038 // the continuation may generate more work for
1039 // this memory request
1040 if (gpuDynInst->useContinuation) {
1041 assert(!gpuDynInst->isNoScope());
1042 gpuDynInst->execContinuation(
1043 gpuDynInst->staticInstruction(),
1044 gpuDynInst);
1045 }
1046 }
1047 }
1048 } else {
1049 gpuDynInst->statusBitVector = VectorMask(0);
1050
1051 if (gpuDynInst->useContinuation) {
1052 assert(!gpuDynInst->isNoScope());
1053 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1054 gpuDynInst);
1055 }
1056 }
1057
1058 delete pkt->senderState;
1059 delete pkt;
1060}
1061
1062ComputeUnit*
1063ComputeUnitParams::create()
1064{
1065 return new ComputeUnit(this);
1066}
1067
1068bool
1069ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1070{
1071 Addr line = pkt->req->getPaddr();
1072
1073 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1074 pkt->req->getVaddr(), line);
1075
1076 assert(pkt->senderState);
1077 computeUnit->tlbCycles += curTick();
1078
1079 // pop off the TLB translation state
1080 TheISA::GpuTLB::TranslationState *translation_state =
1081 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1082
1083 // no PageFaults are permitted for data accesses
1084 if (!translation_state->tlbEntry) {
1085 DTLBPort::SenderState *sender_state =
1086 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1087
1088 Wavefront *w M5_VAR_USED =
1089 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1090 [sender_state->_gpuDynInst->wfSlotId];
1091
1092 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1093 pkt->req->getVaddr());
1094 }
1095
1096 // update the hitLevel distribution
1097 int hit_level = translation_state->hitLevel;
1098 computeUnit->hitsPerTLBLevel[hit_level]++;
1099
1100 delete translation_state->tlbEntry;
1101 assert(!translation_state->ports.size());
1102 pkt->senderState = translation_state->saved;
1103
1104 // for prefetch pkt
1105 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1106
1107 delete translation_state;
1108
1109 // use the original sender state to know how to close this transaction
1110 DTLBPort::SenderState *sender_state =
1111 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1112
1113 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1114 int mp_index = sender_state->portIndex;
1115 Addr vaddr = pkt->req->getVaddr();
1116 gpuDynInst->memStatusVector[line].push_back(mp_index);
1117 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1118
1119 MemCmd requestCmd;
1120
1121 if (pkt->cmd == MemCmd::ReadResp) {
1122 requestCmd = MemCmd::ReadReq;
1123 } else if (pkt->cmd == MemCmd::WriteResp) {
1124 requestCmd = MemCmd::WriteReq;
1125 } else if (pkt->cmd == MemCmd::SwapResp) {
1126 requestCmd = MemCmd::SwapReq;
1127 } else {
1128 panic("unsupported response to request conversion %s\n",
1129 pkt->cmd.toString());
1130 }
1131
1132 if (computeUnit->prefetchDepth) {
1133 int simdId = gpuDynInst->simdId;
1134 int wfSlotId = gpuDynInst->wfSlotId;
1135 Addr last = 0;
1136
1137 switch(computeUnit->prefetchType) {
1138 case Enums::PF_CU:
1139 last = computeUnit->lastVaddrCU[mp_index];
1140 break;
1141 case Enums::PF_PHASE:
1142 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1143 break;
1144 case Enums::PF_WF:
1145 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1146 default:
1147 break;
1148 }
1149
1150 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1151 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1152
1153 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1154 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1155 : 0;
1156
1157 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1158
1159 computeUnit->lastVaddrCU[mp_index] = vaddr;
1160 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1161 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1162
1163 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1164 computeUnit->prefetchStride: stride;
1165
1166 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1167 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1168
1169 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1170
1171 // Prefetch Next few pages atomically
1172 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1173 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1174 vaddr+stride*pf*TheISA::PageBytes);
1175
1176 if (!stride)
1177 break;
1178
1179 RequestPtr prefetch_req = std::make_shared<Request>(
1180 0, vaddr + stride * pf * TheISA::PageBytes,
1181 sizeof(uint8_t), 0,
1182 computeUnit->masterId(),
1183 0, 0, nullptr);
1184
1185 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1186 uint8_t foo = 0;
1187 prefetch_pkt->dataStatic(&foo);
1188
1189 // Because it's atomic operation, only need TLB translation state
1190 prefetch_pkt->senderState =
1191 new TheISA::GpuTLB::TranslationState(TLB_mode,
1192 computeUnit->shader->gpuTc,
1193 true);
1194
1195 // Currently prefetches are zero-latency, hence the sendFunctional
1196 sendFunctional(prefetch_pkt);
1197
1198 /* safe_cast the senderState */
1199 TheISA::GpuTLB::TranslationState *tlb_state =
1200 safe_cast<TheISA::GpuTLB::TranslationState*>(
1201 prefetch_pkt->senderState);
1202
1203
1204 delete tlb_state->tlbEntry;
1205 delete tlb_state;
1206 delete prefetch_pkt;
1207 }
1208 }
1209
1210 // First we must convert the response cmd back to a request cmd so that
1211 // the request can be sent through the cu's master port
1212 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1213 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1214 delete pkt->senderState;
1215 delete pkt;
1216
1217 // New SenderState for the memory access
1218 new_pkt->senderState =
1219 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1220 nullptr);
1221
1222 // translation is done. Schedule the mem_req_event at the appropriate
1223 // cycle to send the timing memory request to ruby
1224 EventFunctionWrapper *mem_req_event =
1225 computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt);
1226
1227 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1228 computeUnit->cu_id, gpuDynInst->simdId,
1229 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1230
1231 computeUnit->schedule(mem_req_event, curTick() +
1232 computeUnit->req_tick_latency);
1233
1234 return true;
1235}
1236
1237EventFunctionWrapper*
1238ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt)
1239{
1240 return new EventFunctionWrapper(
1241 [this, pkt]{ processMemReqEvent(pkt); },
1242 "ComputeUnit memory request event", true);
1243}
1244
1245EventFunctionWrapper*
1246ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt)
1247{
1248 return new EventFunctionWrapper(
1249 [this, pkt]{ processMemRespEvent(pkt); },
1250 "ComputeUnit memory response event", true);
1251}
1252
1253void
1254ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt)
1255{
1256 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1257 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1258 ComputeUnit *compute_unit M5_VAR_USED = computeUnit;
1259
1260 if (!(sendTimingReq(pkt))) {
1261 retries.push_back(std::make_pair(pkt, gpuDynInst));
1262
1263 DPRINTF(GPUPort,
1264 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1265 compute_unit->cu_id, gpuDynInst->simdId,
1266 gpuDynInst->wfSlotId, index,
1267 pkt->req->getPaddr());
1268 } else {
1269 DPRINTF(GPUPort,
1270 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1271 compute_unit->cu_id, gpuDynInst->simdId,
1272 gpuDynInst->wfSlotId, index,
1273 pkt->req->getPaddr());
1274 }
1275}
1276
1277/*
1278 * The initial translation request could have been rejected,
1279 * if <retries> queue is not Retry sending the translation
1280 * request. sendRetry() is called from the peer port whenever
1281 * a translation completes.
1282 */
1283void
1284ComputeUnit::DTLBPort::recvReqRetry()
1285{
1286 int len = retries.size();
1287
1288 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1289 computeUnit->cu_id, len);
1290
1291 assert(len > 0);
1292 assert(isStalled());
1293 // recvReqRetry is an indication that the resource on which this
1294 // port was stalling on is freed. So, remove the stall first
1295 unstallPort();
1296
1297 for (int i = 0; i < len; ++i) {
1298 PacketPtr pkt = retries.front();
1299 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1300 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1301
1302 if (!sendTimingReq(pkt)) {
1303 // Stall port
1304 stallPort();
1305 DPRINTF(GPUTLB, ": failed again\n");
1306 break;
1307 } else {
1308 DPRINTF(GPUTLB, ": successful\n");
1309 retries.pop_front();
1310 }
1311 }
1312}
1313
1314bool
1315ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1316{
1317 Addr line M5_VAR_USED = pkt->req->getPaddr();
1318 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1319 computeUnit->cu_id, pkt->req->getVaddr(), line);
1320
1321 assert(pkt->senderState);
1322
1323 // pop off the TLB translation state
1324 TheISA::GpuTLB::TranslationState *translation_state =
1325 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1326
1327 bool success = translation_state->tlbEntry != nullptr;
1328 delete translation_state->tlbEntry;
1329 assert(!translation_state->ports.size());
1330 pkt->senderState = translation_state->saved;
1331 delete translation_state;
1332
1333 // use the original sender state to know how to close this transaction
1334 ITLBPort::SenderState *sender_state =
1335 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1336
1337 // get the wavefront associated with this translation request
1338 Wavefront *wavefront = sender_state->wavefront;
1339 delete pkt->senderState;
1340
1341 if (success) {
1342 // pkt is reused in fetch(), don't delete it here. However, we must
1343 // reset the command to be a request so that it can be sent through
1344 // the cu's master port
1345 assert(pkt->cmd == MemCmd::ReadResp);
1346 pkt->cmd = MemCmd::ReadReq;
1347
1348 computeUnit->fetchStage.fetch(pkt, wavefront);
1349 } else {
1350 if (wavefront->dropFetch) {
1351 assert(wavefront->instructionBuffer.empty());
1352 wavefront->dropFetch = false;
1353 }
1354
1355 wavefront->pendingFetch = 0;
1356 }
1357
1358 return true;
1359}
1360
1361/*
1362 * The initial translation request could have been rejected, if
1363 * <retries> queue is not empty. Retry sending the translation
1364 * request. sendRetry() is called from the peer port whenever
1365 * a translation completes.
1366 */
1367void
1368ComputeUnit::ITLBPort::recvReqRetry()
1369{
1370
1371 int len = retries.size();
1372 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1373
1374 assert(len > 0);
1375 assert(isStalled());
1376
1377 // recvReqRetry is an indication that the resource on which this
1378 // port was stalling on is freed. So, remove the stall first
1379 unstallPort();
1380
1381 for (int i = 0; i < len; ++i) {
1382 PacketPtr pkt = retries.front();
1383 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1384 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1385
1386 if (!sendTimingReq(pkt)) {
1387 stallPort(); // Stall port
1388 DPRINTF(GPUTLB, ": failed again\n");
1389 break;
1390 } else {
1391 DPRINTF(GPUTLB, ": successful\n");
1392 retries.pop_front();
1393 }
1394 }
1395}
1396
1397void
1398ComputeUnit::regStats()
1399{
|