1/*
2 * Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Author: John Kalamatianos, Anthony Gutierrez
34 */
35#include "gpu-compute/compute_unit.hh"
36
37#include <limits>
38
39#include "base/output.hh"
40#include "debug/GPUDisp.hh"
41#include "debug/GPUExec.hh"
42#include "debug/GPUFetch.hh"
43#include "debug/GPUMem.hh"
44#include "debug/GPUPort.hh"
45#include "debug/GPUPrefetch.hh"
46#include "debug/GPUSync.hh"
47#include "debug/GPUTLB.hh"
48#include "gpu-compute/dispatcher.hh"
49#include "gpu-compute/gpu_dyn_inst.hh"
50#include "gpu-compute/gpu_static_inst.hh"
51#include "gpu-compute/ndrange.hh"
52#include "gpu-compute/shader.hh"
53#include "gpu-compute/simple_pool_manager.hh"
54#include "gpu-compute/vector_register_file.hh"
55#include "gpu-compute/wavefront.hh"
56#include "mem/page_table.hh"
57#include "sim/process.hh"
58
59ComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p),
60 scoreboardCheckStage(p), scheduleStage(p), execStage(p),
61 globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
62 cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
63 spBypassPipeLength(p->spbypass_pipe_length),
64 dpBypassPipeLength(p->dpbypass_pipe_length),
65 issuePeriod(p->issue_period),
66 numGlbMemUnits(p->num_global_mem_pipes),
67 numLocMemUnits(p->num_shared_mem_pipes),
68 perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
69 prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
70 xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
71 functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
72 countPages(p->countPages), barrier_id(0),
73 vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
74 coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
75 req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
76 resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
77 _masterId(p->system->getMasterId(name() + ".ComputeUnit")),
78 lds(*p->localDataStore), globalSeqNum(0), wavefrontSize(p->wfSize),
79 kernelLaunchInst(new KernelLaunchStaticInst())
80{
81 /**
82 * This check is necessary because std::bitset only provides conversion
83 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
84 * there are * a few places in the code where to_ullong() is used, however
85 * if VSZ is larger than a value the host can support then bitset will
86 * throw a runtime exception. we should remove all use of to_long() or
87 * to_ullong() so we can have VSZ greater than 64b, however until that is
88 * done this assert is required.
89 */
90 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
91 p->wfSize <= 0,
92 "WF size is larger than the host can support");
93 fatal_if(!isPowerOf2(wavefrontSize),
94 "Wavefront size should be a power of 2");
95 // calculate how many cycles a vector load or store will need to transfer
96 // its data over the corresponding buses
97 numCyclesPerStoreTransfer =
98 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
99 (double)vrfToCoalescerBusWidth);
100
101 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
102 / coalescerToVrfBusWidth;
103
104 lastVaddrWF.resize(numSIMDs);
105 wfList.resize(numSIMDs);
106
107 for (int j = 0; j < numSIMDs; ++j) {
108 lastVaddrWF[j].resize(p->n_wf);
109
110 for (int i = 0; i < p->n_wf; ++i) {
111 lastVaddrWF[j][i].resize(wfSize());
112
113 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
114 wfList[j][i]->setParent(this);
115
116 for (int k = 0; k < wfSize(); ++k) {
117 lastVaddrWF[j][i][k] = 0;
118 }
119 }
120 }
121
122 lastVaddrSimd.resize(numSIMDs);
123
124 for (int i = 0; i < numSIMDs; ++i) {
125 lastVaddrSimd[i].resize(wfSize(), 0);
126 }
127
128 lastVaddrCU.resize(wfSize());
129
130 lds.setParent(this);
131
132 if (p->execPolicy == "OLDEST-FIRST") {
133 exec_policy = EXEC_POLICY::OLDEST;
134 } else if (p->execPolicy == "ROUND-ROBIN") {
135 exec_policy = EXEC_POLICY::RR;
136 } else {
137 fatal("Invalid WF execution policy (CU)\n");
138 }
139
140 memPort.resize(wfSize());
141
142 // resize the tlbPort vectorArray
143 int tlbPort_width = perLaneTLB ? wfSize() : 1;
144 tlbPort.resize(tlbPort_width);
145
146 cuExitCallback = new CUExitCallback(this);
147 registerExitCallback(cuExitCallback);
148
149 xactCasLoadMap.clear();
150 lastExecCycle.resize(numSIMDs, 0);
151
152 for (int i = 0; i < vrf.size(); ++i) {
153 vrf[i]->setParent(this);
154 }
155
156 numVecRegsPerSimd = vrf[0]->numRegs();
157}
158
159ComputeUnit::~ComputeUnit()
160{
161 // Delete wavefront slots
162 for (int j = 0; j < numSIMDs; ++j) {
163 for (int i = 0; i < shader->n_wf; ++i) {
164 delete wfList[j][i];
165 }
166 lastVaddrSimd[j].clear();
167 }
168 lastVaddrCU.clear();
169 readyList.clear();
170 waveStatusList.clear();
171 dispatchList.clear();
172 vectorAluInstAvail.clear();
173 delete cuExitCallback;
174 delete ldsPort;
175}
176
177void
178ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
179{
180 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
181
182 w->workGroupSz[0] = ndr->q.wgSize[0];
183 w->workGroupSz[1] = ndr->q.wgSize[1];
184 w->workGroupSz[2] = ndr->q.wgSize[2];
185 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
186 w->gridSz[0] = ndr->q.gdSize[0];
187 w->gridSz[1] = ndr->q.gdSize[1];
188 w->gridSz[2] = ndr->q.gdSize[2];
189 w->kernelArgs = ndr->q.args;
190 w->privSizePerItem = ndr->q.privMemPerItem;
191 w->spillSizePerItem = ndr->q.spillMemPerItem;
192 w->roBase = ndr->q.roMemStart;
193 w->roSize = ndr->q.roMemTotal;
194 w->computeActualWgSz(ndr);
195}
196
197void
198ComputeUnit::updateEvents() {
199
200 if (!timestampVec.empty()) {
201 uint32_t vecSize = timestampVec.size();
202 uint32_t i = 0;
203 while (i < vecSize) {
204 if (timestampVec[i] <= shader->tick_cnt) {
205 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
206 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
207 statusVec[i]);
208 timestampVec.erase(timestampVec.begin() + i);
209 regIdxVec.erase(regIdxVec.begin() + i);
210 statusVec.erase(statusVec.begin() + i);
211 --vecSize;
212 --i;
213 }
214 ++i;
215 }
216 }
217
218 for (int i = 0; i< numSIMDs; ++i) {
219 vrf[i]->updateEvents();
220 }
221}
222
223
224void
225ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
226 NDRange *ndr)
227{
228 static int _n_wave = 0;
229
230 VectorMask init_mask;
231 init_mask.reset();
232
233 for (int k = 0; k < wfSize(); ++k) {
234 if (k + waveId * wfSize() < w->actualWgSzTotal)
235 init_mask[k] = 1;
236 }
237
238 w->kernId = ndr->dispatchId;
239 w->wfId = waveId;
240 w->initMask = init_mask.to_ullong();
241
242 for (int k = 0; k < wfSize(); ++k) {
243 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
244 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
245 w->actualWgSz[1];
246 w->workItemId[2][k] = (k + waveId * wfSize()) /
247 (w->actualWgSz[0] * w->actualWgSz[1]);
248
249 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
250 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
251 w->workItemId[0][k];
252 }
253
254 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
255
256 w->barCnt.resize(wfSize(), 0);
257
258 w->maxBarCnt = 0;
259 w->oldBarrierCnt = 0;
260 w->barrierCnt = 0;
261
262 w->privBase = ndr->q.privMemStart;
263 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
264
265 w->spillBase = ndr->q.spillMemStart;
266 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
267
268 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
269
270 // WG state
271 w->wgId = ndr->globalWgId;
272 w->dispatchId = ndr->dispatchId;
273 w->workGroupId[0] = w->wgId % ndr->numWg[0];
274 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
275 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
276
277 w->barrierId = barrier_id;
278 w->stalledAtBarrier = false;
279
280 // set the wavefront context to have a pointer to this section of the LDS
281 w->ldsChunk = ldsChunk;
282
283 int32_t refCount M5_VAR_USED =
284 lds.increaseRefCounter(w->dispatchId, w->wgId);
285 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
286 cu_id, w->wgId, refCount);
287
288 w->instructionBuffer.clear();
289
290 if (w->pendingFetch)
291 w->dropFetch = true;
292
293 // is this the last wavefront in the workgroup
294 // if set the spillWidth to be the remaining work-items
295 // so that the vector access is correct
296 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
297 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
298 } else {
299 w->spillWidth = wfSize();
300 }
301
302 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
303 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
304
305 w->start(++_n_wave, ndr->q.code_ptr);
306}
307
308void
309ComputeUnit::StartWorkgroup(NDRange *ndr)
310{
311 // reserve the LDS capacity allocated to the work group
312 // disambiguated by the dispatch ID and workgroup ID, which should be
313 // globally unique
314 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
315 ndr->q.ldsSize);
316
317 // Send L1 cache acquire
318 // isKernel + isAcquire = Kernel Begin
319 if (shader->impl_kern_boundary_sync) {
320 GPUDynInstPtr gpuDynInst =
321 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
322 getAndIncSeqNum());
323
324 gpuDynInst->useContinuation = false;
325 injectGlobalMemFence(gpuDynInst, true);
326 }
327
328 // calculate the number of 32-bit vector registers required by wavefront
329 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
330 int wave_id = 0;
331
332 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
333 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
334 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
335 // Check if this wavefront slot is available:
336 // It must be stopped and not waiting
337 // for a release to complete S_RETURNING
338 if (w->status == Wavefront::S_STOPPED) {
339 fillKernelState(w, ndr);
340 // if we have scheduled all work items then stop
341 // scheduling wavefronts
342 if (wave_id * wfSize() >= w->actualWgSzTotal)
343 break;
344
345 // reserve vector registers for the scheduled wavefront
346 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
347 uint32_t normSize = 0;
348
349 w->startVgprIndex = vrf[m % numSIMDs]->manager->
350 allocateRegion(vregDemand, &normSize);
351
352 w->reservedVectorRegs = normSize;
353 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
354
355 startWavefront(w, wave_id, ldsChunk, ndr);
356 ++wave_id;
357 }
358 }
359 ++barrier_id;
360}
361
362int
363ComputeUnit::ReadyWorkgroup(NDRange *ndr)
364{
365 // Get true size of workgroup (after clamping to grid size)
366 int trueWgSize[3];
367 int trueWgSizeTotal = 1;
368
369 for (int d = 0; d < 3; ++d) {
370 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
371 ndr->wgId[d] * ndr->q.wgSize[d]);
372
373 trueWgSizeTotal *= trueWgSize[d];
374 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
375 }
376
377 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
378
379 // calculate the number of 32-bit vector registers required by each
380 // work item of the work group
381 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
382 bool vregAvail = true;
383 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
384 int freeWfSlots = 0;
385 // check if the total number of VGPRs required by all WFs of the WG
386 // fit in the VRFs of all SIMD units
387 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
388 int numMappedWfs = 0;
389 std::vector<int> numWfsPerSimd;
390 numWfsPerSimd.resize(numSIMDs, 0);
391 // find how many free WF slots we have across all SIMDs
392 for (int j = 0; j < shader->n_wf; ++j) {
393 for (int i = 0; i < numSIMDs; ++i) {
394 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
395 // count the number of free WF slots
396 ++freeWfSlots;
397 if (numMappedWfs < numWfs) {
398 // count the WFs to be assigned per SIMD
399 numWfsPerSimd[i]++;
400 }
401 numMappedWfs++;
402 }
403 }
404 }
405
406 // if there are enough free WF slots then find if there are enough
407 // free VGPRs per SIMD based on the WF->SIMD mapping
408 if (freeWfSlots >= numWfs) {
409 for (int j = 0; j < numSIMDs; ++j) {
410 // find if there are enough free VGPR regions in the SIMD's VRF
411 // to accommodate the WFs of the new WG that would be mapped to
412 // this SIMD unit
413 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
414 vregDemandPerWI);
415
416 // stop searching if there is at least one SIMD
417 // whose VRF does not have enough free VGPR pools.
418 // This is because a WG is scheduled only if ALL
419 // of its WFs can be scheduled
420 if (!vregAvail)
421 break;
422 }
423 }
424
425 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
426 freeWfSlots, vregAvail);
427
428 if (!vregAvail) {
429 ++numTimesWgBlockedDueVgprAlloc;
430 }
431
432 // Return true if enough WF slots to submit workgroup and if there are
433 // enough VGPRs to schedule all WFs to their SIMD units
434 if (!lds.canReserve(ndr->q.ldsSize)) {
435 wgBlockedDueLdsAllocation++;
436 }
437
438 // Return true if (a) there are enough free WF slots to submit
439 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
440 // SIMD units and (c) if there is enough space in LDS
441 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
442}
443
444int
445ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
446{
447 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
448 int ccnt = 0;
449
450 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
451 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
452 Wavefront *w = wfList[i_simd][i_wf];
453
454 if (w->status == Wavefront::S_RUNNING) {
455 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
456
457 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
458 w->barrierId, _barrier_id);
459
460 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
461 w->barrierCnt, bcnt);
462 }
463
464 if (w->status == Wavefront::S_RUNNING &&
465 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
466 !w->outstandingReqs) {
467 ++ccnt;
468
469 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
470 "%d\n", i_simd, i_wf, ccnt);
471 }
472 }
473 }
474
475 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
476 cu_id, ccnt, bslots);
477
478 return ccnt == bslots;
479}
480
481// Check if the current wavefront is blocked on additional resources.
482bool
483ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
484{
485 bool cede = false;
486
487 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
488 // magic instructions will impact the scheduling of wavefronts
489 if (xact_cas_mode) {
490 /*
491 * When a wavefront calls xact_cas_ld, it adds itself to a per address
492 * queue. All per address queues are managed by the xactCasLoadMap.
493 *
494 * A wavefront is not blocked if: it is not in ANY per address queue or
495 * if it is at the head of a per address queue.
496 */
497 for (auto itMap : xactCasLoadMap) {
498 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
499
500 if (!curWaveIDQueue.empty()) {
501 for (auto it : curWaveIDQueue) {
502 waveIdentifier cur_wave = it;
503
504 if (cur_wave.simdId == simdId &&
505 cur_wave.wfSlotId == wfSlotId) {
506 // 2 possibilities
507 // 1: this WF has a green light
508 // 2: another WF has a green light
509 waveIdentifier owner_wave = curWaveIDQueue.front();
510
511 if (owner_wave.simdId != cur_wave.simdId ||
512 owner_wave.wfSlotId != cur_wave.wfSlotId) {
513 // possibility 2
514 cede = true;
515 break;
516 } else {
517 // possibility 1
518 break;
519 }
520 }
521 }
522 }
523 }
524 }
525
526 return cede;
527}
528
529// Execute one clock worth of work on the ComputeUnit.
530void
531ComputeUnit::exec()
532{
533 updateEvents();
534 // Execute pipeline stages in reverse order to simulate
535 // the pipeline latency
536 globalMemoryPipe.exec();
537 localMemoryPipe.exec();
538 execStage.exec();
539 scheduleStage.exec();
540 scoreboardCheckStage.exec();
541 fetchStage.exec();
542
543 totalCycles++;
544}
545
546void
547ComputeUnit::init()
548{
549 // Initialize CU Bus models
550 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
551 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
552 nextGlbMemBus = 0;
553 nextLocMemBus = 0;
554 fatal_if(numGlbMemUnits > 1,
555 "No support for multiple Global Memory Pipelines exists!!!");
556 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
557 for (int j = 0; j < numGlbMemUnits; ++j) {
558 vrfToGlobalMemPipeBus[j] = WaitClass();
559 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
560 }
561
562 fatal_if(numLocMemUnits > 1,
563 "No support for multiple Local Memory Pipelines exists!!!");
564 vrfToLocalMemPipeBus.resize(numLocMemUnits);
565 for (int j = 0; j < numLocMemUnits; ++j) {
566 vrfToLocalMemPipeBus[j] = WaitClass();
567 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
568 }
569 vectorRegsReserved.resize(numSIMDs, 0);
570 aluPipe.resize(numSIMDs);
571 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
572
573 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
574 wfWait[i] = WaitClass();
575 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
576 }
577
578 for (int i = 0; i < numSIMDs; ++i) {
579 aluPipe[i] = WaitClass();
580 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
581 }
582
583 // Setup space for call args
584 for (int j = 0; j < numSIMDs; ++j) {
585 for (int i = 0; i < shader->n_wf; ++i) {
586 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
587 }
588 }
589
590 // Initializing pipeline resources
591 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
592 waveStatusList.resize(numSIMDs);
593
594 for (int j = 0; j < numSIMDs; ++j) {
595 for (int i = 0; i < shader->n_wf; ++i) {
596 waveStatusList[j].push_back(
597 std::make_pair(wfList[j][i], BLOCKED));
598 }
599 }
600
601 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
602 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
603 }
604
605 fetchStage.init(this);
606 scoreboardCheckStage.init(this);
607 scheduleStage.init(this);
608 execStage.init(this);
609 globalMemoryPipe.init(this);
610 localMemoryPipe.init(this);
611 // initialize state for statistics calculation
612 vectorAluInstAvail.resize(numSIMDs, false);
613 shrMemInstAvail = 0;
614 glbMemInstAvail = 0;
615}
616
617bool
618ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
619{
620 // Ruby has completed the memory op. Schedule the mem_resp_event at the
621 // appropriate cycle to process the timing memory response
622 // This delay represents the pipeline delay
623 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
624 int index = sender_state->port_index;
625 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
626
627 // Is the packet returned a Kernel End or Barrier
628 if (pkt->req->isKernel() && pkt->req->isRelease()) {
629 Wavefront *w =
630 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
631
632 // Check if we are waiting on Kernel End Release
633 if (w->status == Wavefront::S_RETURNING) {
634 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
635 computeUnit->cu_id, w->simdId, w->wfSlotId,
636 w->wfDynId, w->kernId);
637
638 computeUnit->shader->dispatcher->notifyWgCompl(w);
639 w->status = Wavefront::S_STOPPED;
640 } else {
641 w->outstandingReqs--;
642 }
643
644 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
645 computeUnit->cu_id, gpuDynInst->simdId,
646 gpuDynInst->wfSlotId, w->barrierCnt);
647
648 if (gpuDynInst->useContinuation) {
649 assert(!gpuDynInst->isNoScope());
650 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
651 gpuDynInst);
652 }
653
654 delete pkt->senderState;
655 delete pkt->req;
656 delete pkt;
657 return true;
658 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
659 if (gpuDynInst->useContinuation) {
660 assert(!gpuDynInst->isNoScope());
661 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
662 gpuDynInst);
663 }
664
665 delete pkt->senderState;
666 delete pkt->req;
667 delete pkt;
668 return true;
669 }
670
671 ComputeUnit::DataPort::MemRespEvent *mem_resp_event =
672 new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index],
673 pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
751 pkt->req->getFlags(), pkt->req->masterId(),
752 pkt->req->getPC());
753
754 // figure out the type of the request to set read/write
755 BaseTLB::Mode TLB_mode;
756 assert(pkt->isRead() || pkt->isWrite());
757
758 // Check write before read for atomic operations
759 // since atomic operations should use BaseTLB::Write
760 if (pkt->isWrite()){
761 TLB_mode = BaseTLB::Write;
762 } else if (pkt->isRead()) {
763 TLB_mode = BaseTLB::Read;
764 } else {
765 fatal("pkt is not a read nor a write\n");
766 }
767
768 tlbCycles -= curTick();
769 ++tlbRequests;
770
771 int tlbPort_index = perLaneTLB ? index : 0;
772
773 if (shader->timingSim) {
774 if (debugSegFault) {
775 Process *p = shader->gpuTc->getProcessPtr();
776 Addr vaddr = pkt->req->getVaddr();
777 unsigned size = pkt->getSize();
778
779 if ((vaddr + size - 1) % 64 < vaddr % 64) {
780 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
781 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
782 }
783
784 Addr paddr;
785
786 if (!p->pTable->translate(vaddr, paddr)) {
787 if (!p->fixupStackFault(vaddr)) {
788 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
789 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
790 vaddr);
791 }
792 }
793 }
794
795 // This is the SenderState needed upon return
796 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
797
798 // This is the senderState needed by the TLB hierarchy to function
799 TheISA::GpuTLB::TranslationState *translation_state =
800 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
801 pkt->senderState);
802
803 pkt->senderState = translation_state;
804
805 if (functionalTLB) {
806 tlbPort[tlbPort_index]->sendFunctional(pkt);
807
808 // update the hitLevel distribution
809 int hit_level = translation_state->hitLevel;
810 assert(hit_level != -1);
811 hitsPerTLBLevel[hit_level]++;
812
813 // New SenderState for the memory access
814 X86ISA::GpuTLB::TranslationState *sender_state =
815 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
816
817 delete sender_state->tlbEntry;
818 delete sender_state->saved;
819 delete sender_state;
820
821 assert(pkt->req->hasPaddr());
822 assert(pkt->req->hasSize());
823
824 uint8_t *tmpData = pkt->getPtr<uint8_t>();
825
826 // this is necessary because the GPU TLB receives packets instead
827 // of requests. when the translation is complete, all relevent
828 // fields in the request will be populated, but not in the packet.
829 // here we create the new packet so we can set the size, addr,
830 // and proper flags.
831 PacketPtr oldPkt = pkt;
832 pkt = new Packet(oldPkt->req, oldPkt->cmd);
833 delete oldPkt;
834 pkt->dataStatic(tmpData);
835
836
837 // New SenderState for the memory access
838 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
839 index, nullptr);
840
841 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
842 gpuDynInst->tlbHitLevel[index] = hit_level;
843
844
845 // translation is done. Schedule the mem_req_event at the
846 // appropriate cycle to send the timing memory request to ruby
847 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
848 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
849
850 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
851 "scheduled\n", cu_id, gpuDynInst->simdId,
852 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
853
854 schedule(mem_req_event, curTick() + req_tick_latency);
855 } else if (tlbPort[tlbPort_index]->isStalled()) {
856 assert(tlbPort[tlbPort_index]->retries.size() > 0);
857
858 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
859 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
860 tmp_vaddr);
861
862 tlbPort[tlbPort_index]->retries.push_back(pkt);
863 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
864 // Stall the data port;
865 // No more packet will be issued till
866 // ruby indicates resources are freed by
867 // a recvReqRetry() call back on this port.
868 tlbPort[tlbPort_index]->stallPort();
869
870 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
871 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
872 tmp_vaddr);
873
874 tlbPort[tlbPort_index]->retries.push_back(pkt);
875 } else {
876 DPRINTF(GPUTLB,
877 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
878 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
879 }
880 } else {
881 if (pkt->cmd == MemCmd::MemFenceReq) {
882 gpuDynInst->statusBitVector = VectorMask(0);
883 } else {
884 gpuDynInst->statusBitVector &= (~(1ll << index));
885 }
886
887 // New SenderState for the memory access
888 delete pkt->senderState;
889
890 // Because it's atomic operation, only need TLB translation state
891 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
892 shader->gpuTc);
893
894 tlbPort[tlbPort_index]->sendFunctional(pkt);
895
896 // the addr of the packet is not modified, so we need to create a new
897 // packet, or otherwise the memory access will have the old virtual
898 // address sent in the translation packet, instead of the physical
899 // address returned by the translation.
900 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
901 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
902
903 // Translation is done. It is safe to send the packet to memory.
904 memPort[0]->sendFunctional(new_pkt);
905
906 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
907 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
908 new_pkt->req->getPaddr());
909
910 // safe_cast the senderState
911 TheISA::GpuTLB::TranslationState *sender_state =
912 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
913
914 delete sender_state->tlbEntry;
915 delete new_pkt;
916 delete pkt->senderState;
917 delete pkt->req;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
926 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 Request* req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
948 }
949 req->setPaddr(0);
950 if (kernelLaunch) {
951 req->setFlags(Request::KERNEL);
952 }
953
954 // for non-kernel MemFence operations, memorder flags are set depending
955 // on which type of request is currently being sent, so this
956 // should be set by the caller (e.g. if an inst has acq-rel
957 // semantics, it will send one acquire req an one release req)
958 gpuDynInst->setRequestFlags(req, kernelLaunch);
959
960 // a mem fence must correspond to an acquire/release request
961 assert(req->isAcquire() || req->isRelease());
962
963 // create packet
964 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
965
966 // set packet's sender state
967 pkt->senderState =
968 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
969
970 // send the packet
971 sendSyncRequest(gpuDynInst, 0, pkt);
972}
973
974const char*
975ComputeUnit::DataPort::MemRespEvent::description() const
976{
977 return "ComputeUnit memory response event";
978}
979
980void
981ComputeUnit::DataPort::MemRespEvent::process()
982{
983 DataPort::SenderState *sender_state =
984 safe_cast<DataPort::SenderState*>(pkt->senderState);
985
986 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
987 ComputeUnit *compute_unit = dataPort->computeUnit;
988
989 assert(gpuDynInst);
990
991 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
992 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
993 pkt->req->getPaddr(), dataPort->index);
994
995 Addr paddr = pkt->req->getPaddr();
996
997 if (pkt->cmd != MemCmd::MemFenceResp) {
998 int index = gpuDynInst->memStatusVector[paddr].back();
999
1000 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
1001 pkt->req->getPaddr(), index);
1002
1003 gpuDynInst->memStatusVector[paddr].pop_back();
1004 gpuDynInst->pAddr = pkt->req->getPaddr();
1005
1006 if (pkt->isRead() || pkt->isWrite()) {
1007
1008 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1009 gpuDynInst->statusBitVector &= (~(1ULL << index));
1010 } else {
1011 assert(gpuDynInst->statusVector[index] > 0);
1012 gpuDynInst->statusVector[index]--;
1013
1014 if (!gpuDynInst->statusVector[index])
1015 gpuDynInst->statusBitVector &= (~(1ULL << index));
1016 }
1017
1018 DPRINTF(GPUMem, "bitvector is now %#x\n",
1019 gpuDynInst->statusBitVector);
1020
1021 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1022 auto iter = gpuDynInst->memStatusVector.begin();
1023 auto end = gpuDynInst->memStatusVector.end();
1024
1025 while (iter != end) {
1026 assert(iter->second.empty());
1027 ++iter;
1028 }
1029
1030 gpuDynInst->memStatusVector.clear();
1031
1032 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1033 gpuDynInst->statusVector.clear();
1034
1035 if (gpuDynInst->isLoad() || gpuDynInst->isAtomic()) {
1036 assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy());
1037
1038 compute_unit->globalMemoryPipe.getGMLdRespFIFO()
1039 .push(gpuDynInst);
1040 } else {
1041 assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy());
1042
1043 compute_unit->globalMemoryPipe.getGMStRespFIFO()
1044 .push(gpuDynInst);
1045 }
1046
1047 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1048 compute_unit->cu_id, gpuDynInst->simdId,
1049 gpuDynInst->wfSlotId);
1050
1051 // after clearing the status vectors,
1052 // see if there is a continuation to perform
1053 // the continuation may generate more work for
1054 // this memory request
1055 if (gpuDynInst->useContinuation) {
1056 assert(!gpuDynInst->isNoScope());
1057 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1058 gpuDynInst);
1059 }
1060 }
1061 }
1062 } else {
1063 gpuDynInst->statusBitVector = VectorMask(0);
1064
1065 if (gpuDynInst->useContinuation) {
1066 assert(!gpuDynInst->isNoScope());
1067 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1068 gpuDynInst);
1069 }
1070 }
1071
1072 delete pkt->senderState;
1073 delete pkt->req;
1074 delete pkt;
1075}
1076
1077ComputeUnit*
1078ComputeUnitParams::create()
1079{
1080 return new ComputeUnit(this);
1081}
1082
1083bool
1084ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1085{
1086 Addr line = pkt->req->getPaddr();
1087
1088 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1089 pkt->req->getVaddr(), line);
1090
1091 assert(pkt->senderState);
1092 computeUnit->tlbCycles += curTick();
1093
1094 // pop off the TLB translation state
1095 TheISA::GpuTLB::TranslationState *translation_state =
1096 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1097
1098 // no PageFaults are permitted for data accesses
1099 if (!translation_state->tlbEntry->valid) {
1100 DTLBPort::SenderState *sender_state =
1101 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1102
1103 Wavefront *w M5_VAR_USED =
1104 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1105 [sender_state->_gpuDynInst->wfSlotId];
1106
1107 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1108 pkt->req->getVaddr());
1109 }
1110
1111 assert(translation_state->tlbEntry->valid);
1112
1113 // update the hitLevel distribution
1114 int hit_level = translation_state->hitLevel;
1115 computeUnit->hitsPerTLBLevel[hit_level]++;
1116
1117 delete translation_state->tlbEntry;
1118 assert(!translation_state->ports.size());
1119 pkt->senderState = translation_state->saved;
1120
1121 // for prefetch pkt
1122 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1123
1124 delete translation_state;
1125
1126 // use the original sender state to know how to close this transaction
1127 DTLBPort::SenderState *sender_state =
1128 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1129
1130 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1131 int mp_index = sender_state->portIndex;
1132 Addr vaddr = pkt->req->getVaddr();
1133 gpuDynInst->memStatusVector[line].push_back(mp_index);
1134 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1135
1136 MemCmd requestCmd;
1137
1138 if (pkt->cmd == MemCmd::ReadResp) {
1139 requestCmd = MemCmd::ReadReq;
1140 } else if (pkt->cmd == MemCmd::WriteResp) {
1141 requestCmd = MemCmd::WriteReq;
1142 } else if (pkt->cmd == MemCmd::SwapResp) {
1143 requestCmd = MemCmd::SwapReq;
1144 } else {
1145 panic("unsupported response to request conversion %s\n",
1146 pkt->cmd.toString());
1147 }
1148
1149 if (computeUnit->prefetchDepth) {
1150 int simdId = gpuDynInst->simdId;
1151 int wfSlotId = gpuDynInst->wfSlotId;
1152 Addr last = 0;
1153
1154 switch(computeUnit->prefetchType) {
1155 case Enums::PF_CU:
1156 last = computeUnit->lastVaddrCU[mp_index];
1157 break;
1158 case Enums::PF_PHASE:
1159 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1160 break;
1161 case Enums::PF_WF:
1162 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1163 default:
1164 break;
1165 }
1166
1167 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1168 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1169
1170 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1171 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1172 : 0;
1173
1174 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1175
1176 computeUnit->lastVaddrCU[mp_index] = vaddr;
1177 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1178 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1179
1180 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1181 computeUnit->prefetchStride: stride;
1182
1183 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1184 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1185
1186 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1187
1188 // Prefetch Next few pages atomically
1189 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1190 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1191 vaddr+stride*pf*TheISA::PageBytes);
1192
1193 if (!stride)
1194 break;
1195
1196 Request *prefetch_req = new Request(0, vaddr + stride * pf *
1197 TheISA::PageBytes,
1198 sizeof(uint8_t), 0,
1199 computeUnit->masterId(),
1200 0, 0, 0);
1201
1202 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1203 uint8_t foo = 0;
1204 prefetch_pkt->dataStatic(&foo);
1205
1206 // Because it's atomic operation, only need TLB translation state
1207 prefetch_pkt->senderState =
1208 new TheISA::GpuTLB::TranslationState(TLB_mode,
1209 computeUnit->shader->gpuTc,
1210 true);
1211
1212 // Currently prefetches are zero-latency, hence the sendFunctional
1213 sendFunctional(prefetch_pkt);
1214
1215 /* safe_cast the senderState */
1216 TheISA::GpuTLB::TranslationState *tlb_state =
1217 safe_cast<TheISA::GpuTLB::TranslationState*>(
1218 prefetch_pkt->senderState);
1219
1220
1221 delete tlb_state->tlbEntry;
1222 delete tlb_state;
1223 delete prefetch_pkt->req;
1224 delete prefetch_pkt;
1225 }
1226 }
1227
1228 // First we must convert the response cmd back to a request cmd so that
1229 // the request can be sent through the cu's master port
1230 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1231 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1232 delete pkt->senderState;
1233 delete pkt;
1234
1235 // New SenderState for the memory access
1236 new_pkt->senderState =
1237 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1238 nullptr);
1239
1240 // translation is done. Schedule the mem_req_event at the appropriate
1241 // cycle to send the timing memory request to ruby
1242 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
1243 new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index],
1244 new_pkt);
1245
1246 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1247 computeUnit->cu_id, gpuDynInst->simdId,
1248 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1249
1250 computeUnit->schedule(mem_req_event, curTick() +
1251 computeUnit->req_tick_latency);
1252
1253 return true;
1254}
1255
1256const char*
1257ComputeUnit::DataPort::MemReqEvent::description() const
1258{
1259 return "ComputeUnit memory request event";
1260}
1261
1262void
1263ComputeUnit::DataPort::MemReqEvent::process()
1264{
1265 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1266 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1267 ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit;
1268
1269 if (!(dataPort->sendTimingReq(pkt))) {
1270 dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst));
1271
1272 DPRINTF(GPUPort,
1273 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1274 compute_unit->cu_id, gpuDynInst->simdId,
1275 gpuDynInst->wfSlotId, dataPort->index,
1276 pkt->req->getPaddr());
1277 } else {
1278 DPRINTF(GPUPort,
1279 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1280 compute_unit->cu_id, gpuDynInst->simdId,
1281 gpuDynInst->wfSlotId, dataPort->index,
1282 pkt->req->getPaddr());
1283 }
1284}
1285
1286/*
1287 * The initial translation request could have been rejected,
1288 * if <retries> queue is not Retry sending the translation
1289 * request. sendRetry() is called from the peer port whenever
1290 * a translation completes.
1291 */
1292void
1293ComputeUnit::DTLBPort::recvReqRetry()
1294{
1295 int len = retries.size();
1296
1297 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1298 computeUnit->cu_id, len);
1299
1300 assert(len > 0);
1301 assert(isStalled());
1302 // recvReqRetry is an indication that the resource on which this
1303 // port was stalling on is freed. So, remove the stall first
1304 unstallPort();
1305
1306 for (int i = 0; i < len; ++i) {
1307 PacketPtr pkt = retries.front();
1308 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1309 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1310
1311 if (!sendTimingReq(pkt)) {
1312 // Stall port
1313 stallPort();
1314 DPRINTF(GPUTLB, ": failed again\n");
1315 break;
1316 } else {
1317 DPRINTF(GPUTLB, ": successful\n");
1318 retries.pop_front();
1319 }
1320 }
1321}
1322
1323bool
1324ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1325{
1326 Addr line M5_VAR_USED = pkt->req->getPaddr();
1327 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1328 computeUnit->cu_id, pkt->req->getVaddr(), line);
1329
1330 assert(pkt->senderState);
1331
1332 // pop off the TLB translation state
1333 TheISA::GpuTLB::TranslationState *translation_state =
1334 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1335
1336 bool success = translation_state->tlbEntry->valid;
1337 delete translation_state->tlbEntry;
1338 assert(!translation_state->ports.size());
1339 pkt->senderState = translation_state->saved;
1340 delete translation_state;
1341
1342 // use the original sender state to know how to close this transaction
1343 ITLBPort::SenderState *sender_state =
1344 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1345
1346 // get the wavefront associated with this translation request
1347 Wavefront *wavefront = sender_state->wavefront;
1348 delete pkt->senderState;
1349
1350 if (success) {
1351 // pkt is reused in fetch(), don't delete it here. However, we must
1352 // reset the command to be a request so that it can be sent through
1353 // the cu's master port
1354 assert(pkt->cmd == MemCmd::ReadResp);
1355 pkt->cmd = MemCmd::ReadReq;
1356
1357 computeUnit->fetchStage.fetch(pkt, wavefront);
1358 } else {
1359 if (wavefront->dropFetch) {
1360 assert(wavefront->instructionBuffer.empty());
1361 wavefront->dropFetch = false;
1362 }
1363
1364 wavefront->pendingFetch = 0;
1365 }
1366
1367 return true;
1368}
1369
1370/*
1371 * The initial translation request could have been rejected, if
1372 * <retries> queue is not empty. Retry sending the translation
1373 * request. sendRetry() is called from the peer port whenever
1374 * a translation completes.
1375 */
1376void
1377ComputeUnit::ITLBPort::recvReqRetry()
1378{
1379
1380 int len = retries.size();
1381 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1382
1383 assert(len > 0);
1384 assert(isStalled());
1385
1386 // recvReqRetry is an indication that the resource on which this
1387 // port was stalling on is freed. So, remove the stall first
1388 unstallPort();
1389
1390 for (int i = 0; i < len; ++i) {
1391 PacketPtr pkt = retries.front();
1392 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1393 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1394
1395 if (!sendTimingReq(pkt)) {
1396 stallPort(); // Stall port
1397 DPRINTF(GPUTLB, ": failed again\n");
1398 break;
1399 } else {
1400 DPRINTF(GPUTLB, ": successful\n");
1401 retries.pop_front();
1402 }
1403 }
1404}
1405
1406void
1407ComputeUnit::regStats()
1408{
1409 MemObject::regStats();
1410
| 1/*
2 * Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Author: John Kalamatianos, Anthony Gutierrez
34 */
35#include "gpu-compute/compute_unit.hh"
36
37#include <limits>
38
39#include "base/output.hh"
40#include "debug/GPUDisp.hh"
41#include "debug/GPUExec.hh"
42#include "debug/GPUFetch.hh"
43#include "debug/GPUMem.hh"
44#include "debug/GPUPort.hh"
45#include "debug/GPUPrefetch.hh"
46#include "debug/GPUSync.hh"
47#include "debug/GPUTLB.hh"
48#include "gpu-compute/dispatcher.hh"
49#include "gpu-compute/gpu_dyn_inst.hh"
50#include "gpu-compute/gpu_static_inst.hh"
51#include "gpu-compute/ndrange.hh"
52#include "gpu-compute/shader.hh"
53#include "gpu-compute/simple_pool_manager.hh"
54#include "gpu-compute/vector_register_file.hh"
55#include "gpu-compute/wavefront.hh"
56#include "mem/page_table.hh"
57#include "sim/process.hh"
58
59ComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p),
60 scoreboardCheckStage(p), scheduleStage(p), execStage(p),
61 globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
62 cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
63 spBypassPipeLength(p->spbypass_pipe_length),
64 dpBypassPipeLength(p->dpbypass_pipe_length),
65 issuePeriod(p->issue_period),
66 numGlbMemUnits(p->num_global_mem_pipes),
67 numLocMemUnits(p->num_shared_mem_pipes),
68 perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
69 prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
70 xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
71 functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
72 countPages(p->countPages), barrier_id(0),
73 vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
74 coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
75 req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
76 resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
77 _masterId(p->system->getMasterId(name() + ".ComputeUnit")),
78 lds(*p->localDataStore), globalSeqNum(0), wavefrontSize(p->wfSize),
79 kernelLaunchInst(new KernelLaunchStaticInst())
80{
81 /**
82 * This check is necessary because std::bitset only provides conversion
83 * to unsigned long or unsigned long long via to_ulong() or to_ullong().
84 * there are * a few places in the code where to_ullong() is used, however
85 * if VSZ is larger than a value the host can support then bitset will
86 * throw a runtime exception. we should remove all use of to_long() or
87 * to_ullong() so we can have VSZ greater than 64b, however until that is
88 * done this assert is required.
89 */
90 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
91 p->wfSize <= 0,
92 "WF size is larger than the host can support");
93 fatal_if(!isPowerOf2(wavefrontSize),
94 "Wavefront size should be a power of 2");
95 // calculate how many cycles a vector load or store will need to transfer
96 // its data over the corresponding buses
97 numCyclesPerStoreTransfer =
98 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
99 (double)vrfToCoalescerBusWidth);
100
101 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
102 / coalescerToVrfBusWidth;
103
104 lastVaddrWF.resize(numSIMDs);
105 wfList.resize(numSIMDs);
106
107 for (int j = 0; j < numSIMDs; ++j) {
108 lastVaddrWF[j].resize(p->n_wf);
109
110 for (int i = 0; i < p->n_wf; ++i) {
111 lastVaddrWF[j][i].resize(wfSize());
112
113 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
114 wfList[j][i]->setParent(this);
115
116 for (int k = 0; k < wfSize(); ++k) {
117 lastVaddrWF[j][i][k] = 0;
118 }
119 }
120 }
121
122 lastVaddrSimd.resize(numSIMDs);
123
124 for (int i = 0; i < numSIMDs; ++i) {
125 lastVaddrSimd[i].resize(wfSize(), 0);
126 }
127
128 lastVaddrCU.resize(wfSize());
129
130 lds.setParent(this);
131
132 if (p->execPolicy == "OLDEST-FIRST") {
133 exec_policy = EXEC_POLICY::OLDEST;
134 } else if (p->execPolicy == "ROUND-ROBIN") {
135 exec_policy = EXEC_POLICY::RR;
136 } else {
137 fatal("Invalid WF execution policy (CU)\n");
138 }
139
140 memPort.resize(wfSize());
141
142 // resize the tlbPort vectorArray
143 int tlbPort_width = perLaneTLB ? wfSize() : 1;
144 tlbPort.resize(tlbPort_width);
145
146 cuExitCallback = new CUExitCallback(this);
147 registerExitCallback(cuExitCallback);
148
149 xactCasLoadMap.clear();
150 lastExecCycle.resize(numSIMDs, 0);
151
152 for (int i = 0; i < vrf.size(); ++i) {
153 vrf[i]->setParent(this);
154 }
155
156 numVecRegsPerSimd = vrf[0]->numRegs();
157}
158
159ComputeUnit::~ComputeUnit()
160{
161 // Delete wavefront slots
162 for (int j = 0; j < numSIMDs; ++j) {
163 for (int i = 0; i < shader->n_wf; ++i) {
164 delete wfList[j][i];
165 }
166 lastVaddrSimd[j].clear();
167 }
168 lastVaddrCU.clear();
169 readyList.clear();
170 waveStatusList.clear();
171 dispatchList.clear();
172 vectorAluInstAvail.clear();
173 delete cuExitCallback;
174 delete ldsPort;
175}
176
177void
178ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
179{
180 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
181
182 w->workGroupSz[0] = ndr->q.wgSize[0];
183 w->workGroupSz[1] = ndr->q.wgSize[1];
184 w->workGroupSz[2] = ndr->q.wgSize[2];
185 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
186 w->gridSz[0] = ndr->q.gdSize[0];
187 w->gridSz[1] = ndr->q.gdSize[1];
188 w->gridSz[2] = ndr->q.gdSize[2];
189 w->kernelArgs = ndr->q.args;
190 w->privSizePerItem = ndr->q.privMemPerItem;
191 w->spillSizePerItem = ndr->q.spillMemPerItem;
192 w->roBase = ndr->q.roMemStart;
193 w->roSize = ndr->q.roMemTotal;
194 w->computeActualWgSz(ndr);
195}
196
197void
198ComputeUnit::updateEvents() {
199
200 if (!timestampVec.empty()) {
201 uint32_t vecSize = timestampVec.size();
202 uint32_t i = 0;
203 while (i < vecSize) {
204 if (timestampVec[i] <= shader->tick_cnt) {
205 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
206 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
207 statusVec[i]);
208 timestampVec.erase(timestampVec.begin() + i);
209 regIdxVec.erase(regIdxVec.begin() + i);
210 statusVec.erase(statusVec.begin() + i);
211 --vecSize;
212 --i;
213 }
214 ++i;
215 }
216 }
217
218 for (int i = 0; i< numSIMDs; ++i) {
219 vrf[i]->updateEvents();
220 }
221}
222
223
224void
225ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
226 NDRange *ndr)
227{
228 static int _n_wave = 0;
229
230 VectorMask init_mask;
231 init_mask.reset();
232
233 for (int k = 0; k < wfSize(); ++k) {
234 if (k + waveId * wfSize() < w->actualWgSzTotal)
235 init_mask[k] = 1;
236 }
237
238 w->kernId = ndr->dispatchId;
239 w->wfId = waveId;
240 w->initMask = init_mask.to_ullong();
241
242 for (int k = 0; k < wfSize(); ++k) {
243 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
244 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
245 w->actualWgSz[1];
246 w->workItemId[2][k] = (k + waveId * wfSize()) /
247 (w->actualWgSz[0] * w->actualWgSz[1]);
248
249 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
250 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
251 w->workItemId[0][k];
252 }
253
254 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
255
256 w->barCnt.resize(wfSize(), 0);
257
258 w->maxBarCnt = 0;
259 w->oldBarrierCnt = 0;
260 w->barrierCnt = 0;
261
262 w->privBase = ndr->q.privMemStart;
263 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
264
265 w->spillBase = ndr->q.spillMemStart;
266 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
267
268 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
269
270 // WG state
271 w->wgId = ndr->globalWgId;
272 w->dispatchId = ndr->dispatchId;
273 w->workGroupId[0] = w->wgId % ndr->numWg[0];
274 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
275 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
276
277 w->barrierId = barrier_id;
278 w->stalledAtBarrier = false;
279
280 // set the wavefront context to have a pointer to this section of the LDS
281 w->ldsChunk = ldsChunk;
282
283 int32_t refCount M5_VAR_USED =
284 lds.increaseRefCounter(w->dispatchId, w->wgId);
285 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
286 cu_id, w->wgId, refCount);
287
288 w->instructionBuffer.clear();
289
290 if (w->pendingFetch)
291 w->dropFetch = true;
292
293 // is this the last wavefront in the workgroup
294 // if set the spillWidth to be the remaining work-items
295 // so that the vector access is correct
296 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
297 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
298 } else {
299 w->spillWidth = wfSize();
300 }
301
302 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
303 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
304
305 w->start(++_n_wave, ndr->q.code_ptr);
306}
307
308void
309ComputeUnit::StartWorkgroup(NDRange *ndr)
310{
311 // reserve the LDS capacity allocated to the work group
312 // disambiguated by the dispatch ID and workgroup ID, which should be
313 // globally unique
314 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
315 ndr->q.ldsSize);
316
317 // Send L1 cache acquire
318 // isKernel + isAcquire = Kernel Begin
319 if (shader->impl_kern_boundary_sync) {
320 GPUDynInstPtr gpuDynInst =
321 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
322 getAndIncSeqNum());
323
324 gpuDynInst->useContinuation = false;
325 injectGlobalMemFence(gpuDynInst, true);
326 }
327
328 // calculate the number of 32-bit vector registers required by wavefront
329 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
330 int wave_id = 0;
331
332 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
333 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
334 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
335 // Check if this wavefront slot is available:
336 // It must be stopped and not waiting
337 // for a release to complete S_RETURNING
338 if (w->status == Wavefront::S_STOPPED) {
339 fillKernelState(w, ndr);
340 // if we have scheduled all work items then stop
341 // scheduling wavefronts
342 if (wave_id * wfSize() >= w->actualWgSzTotal)
343 break;
344
345 // reserve vector registers for the scheduled wavefront
346 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
347 uint32_t normSize = 0;
348
349 w->startVgprIndex = vrf[m % numSIMDs]->manager->
350 allocateRegion(vregDemand, &normSize);
351
352 w->reservedVectorRegs = normSize;
353 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
354
355 startWavefront(w, wave_id, ldsChunk, ndr);
356 ++wave_id;
357 }
358 }
359 ++barrier_id;
360}
361
362int
363ComputeUnit::ReadyWorkgroup(NDRange *ndr)
364{
365 // Get true size of workgroup (after clamping to grid size)
366 int trueWgSize[3];
367 int trueWgSizeTotal = 1;
368
369 for (int d = 0; d < 3; ++d) {
370 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
371 ndr->wgId[d] * ndr->q.wgSize[d]);
372
373 trueWgSizeTotal *= trueWgSize[d];
374 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]);
375 }
376
377 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal);
378
379 // calculate the number of 32-bit vector registers required by each
380 // work item of the work group
381 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
382 bool vregAvail = true;
383 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
384 int freeWfSlots = 0;
385 // check if the total number of VGPRs required by all WFs of the WG
386 // fit in the VRFs of all SIMD units
387 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
388 int numMappedWfs = 0;
389 std::vector<int> numWfsPerSimd;
390 numWfsPerSimd.resize(numSIMDs, 0);
391 // find how many free WF slots we have across all SIMDs
392 for (int j = 0; j < shader->n_wf; ++j) {
393 for (int i = 0; i < numSIMDs; ++i) {
394 if (wfList[i][j]->status == Wavefront::S_STOPPED) {
395 // count the number of free WF slots
396 ++freeWfSlots;
397 if (numMappedWfs < numWfs) {
398 // count the WFs to be assigned per SIMD
399 numWfsPerSimd[i]++;
400 }
401 numMappedWfs++;
402 }
403 }
404 }
405
406 // if there are enough free WF slots then find if there are enough
407 // free VGPRs per SIMD based on the WF->SIMD mapping
408 if (freeWfSlots >= numWfs) {
409 for (int j = 0; j < numSIMDs; ++j) {
410 // find if there are enough free VGPR regions in the SIMD's VRF
411 // to accommodate the WFs of the new WG that would be mapped to
412 // this SIMD unit
413 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
414 vregDemandPerWI);
415
416 // stop searching if there is at least one SIMD
417 // whose VRF does not have enough free VGPR pools.
418 // This is because a WG is scheduled only if ALL
419 // of its WFs can be scheduled
420 if (!vregAvail)
421 break;
422 }
423 }
424
425 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n",
426 freeWfSlots, vregAvail);
427
428 if (!vregAvail) {
429 ++numTimesWgBlockedDueVgprAlloc;
430 }
431
432 // Return true if enough WF slots to submit workgroup and if there are
433 // enough VGPRs to schedule all WFs to their SIMD units
434 if (!lds.canReserve(ndr->q.ldsSize)) {
435 wgBlockedDueLdsAllocation++;
436 }
437
438 // Return true if (a) there are enough free WF slots to submit
439 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
440 // SIMD units and (c) if there is enough space in LDS
441 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
442}
443
444int
445ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
446{
447 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
448 int ccnt = 0;
449
450 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
451 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
452 Wavefront *w = wfList[i_simd][i_wf];
453
454 if (w->status == Wavefront::S_RUNNING) {
455 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
456
457 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
458 w->barrierId, _barrier_id);
459
460 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
461 w->barrierCnt, bcnt);
462 }
463
464 if (w->status == Wavefront::S_RUNNING &&
465 w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
466 !w->outstandingReqs) {
467 ++ccnt;
468
469 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
470 "%d\n", i_simd, i_wf, ccnt);
471 }
472 }
473 }
474
475 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
476 cu_id, ccnt, bslots);
477
478 return ccnt == bslots;
479}
480
481// Check if the current wavefront is blocked on additional resources.
482bool
483ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
484{
485 bool cede = false;
486
487 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
488 // magic instructions will impact the scheduling of wavefronts
489 if (xact_cas_mode) {
490 /*
491 * When a wavefront calls xact_cas_ld, it adds itself to a per address
492 * queue. All per address queues are managed by the xactCasLoadMap.
493 *
494 * A wavefront is not blocked if: it is not in ANY per address queue or
495 * if it is at the head of a per address queue.
496 */
497 for (auto itMap : xactCasLoadMap) {
498 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
499
500 if (!curWaveIDQueue.empty()) {
501 for (auto it : curWaveIDQueue) {
502 waveIdentifier cur_wave = it;
503
504 if (cur_wave.simdId == simdId &&
505 cur_wave.wfSlotId == wfSlotId) {
506 // 2 possibilities
507 // 1: this WF has a green light
508 // 2: another WF has a green light
509 waveIdentifier owner_wave = curWaveIDQueue.front();
510
511 if (owner_wave.simdId != cur_wave.simdId ||
512 owner_wave.wfSlotId != cur_wave.wfSlotId) {
513 // possibility 2
514 cede = true;
515 break;
516 } else {
517 // possibility 1
518 break;
519 }
520 }
521 }
522 }
523 }
524 }
525
526 return cede;
527}
528
529// Execute one clock worth of work on the ComputeUnit.
530void
531ComputeUnit::exec()
532{
533 updateEvents();
534 // Execute pipeline stages in reverse order to simulate
535 // the pipeline latency
536 globalMemoryPipe.exec();
537 localMemoryPipe.exec();
538 execStage.exec();
539 scheduleStage.exec();
540 scoreboardCheckStage.exec();
541 fetchStage.exec();
542
543 totalCycles++;
544}
545
546void
547ComputeUnit::init()
548{
549 // Initialize CU Bus models
550 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
551 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
552 nextGlbMemBus = 0;
553 nextLocMemBus = 0;
554 fatal_if(numGlbMemUnits > 1,
555 "No support for multiple Global Memory Pipelines exists!!!");
556 vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
557 for (int j = 0; j < numGlbMemUnits; ++j) {
558 vrfToGlobalMemPipeBus[j] = WaitClass();
559 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
560 }
561
562 fatal_if(numLocMemUnits > 1,
563 "No support for multiple Local Memory Pipelines exists!!!");
564 vrfToLocalMemPipeBus.resize(numLocMemUnits);
565 for (int j = 0; j < numLocMemUnits; ++j) {
566 vrfToLocalMemPipeBus[j] = WaitClass();
567 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
568 }
569 vectorRegsReserved.resize(numSIMDs, 0);
570 aluPipe.resize(numSIMDs);
571 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
572
573 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
574 wfWait[i] = WaitClass();
575 wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
576 }
577
578 for (int i = 0; i < numSIMDs; ++i) {
579 aluPipe[i] = WaitClass();
580 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
581 }
582
583 // Setup space for call args
584 for (int j = 0; j < numSIMDs; ++j) {
585 for (int i = 0; i < shader->n_wf; ++i) {
586 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
587 }
588 }
589
590 // Initializing pipeline resources
591 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
592 waveStatusList.resize(numSIMDs);
593
594 for (int j = 0; j < numSIMDs; ++j) {
595 for (int i = 0; i < shader->n_wf; ++i) {
596 waveStatusList[j].push_back(
597 std::make_pair(wfList[j][i], BLOCKED));
598 }
599 }
600
601 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
602 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
603 }
604
605 fetchStage.init(this);
606 scoreboardCheckStage.init(this);
607 scheduleStage.init(this);
608 execStage.init(this);
609 globalMemoryPipe.init(this);
610 localMemoryPipe.init(this);
611 // initialize state for statistics calculation
612 vectorAluInstAvail.resize(numSIMDs, false);
613 shrMemInstAvail = 0;
614 glbMemInstAvail = 0;
615}
616
617bool
618ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
619{
620 // Ruby has completed the memory op. Schedule the mem_resp_event at the
621 // appropriate cycle to process the timing memory response
622 // This delay represents the pipeline delay
623 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
624 int index = sender_state->port_index;
625 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
626
627 // Is the packet returned a Kernel End or Barrier
628 if (pkt->req->isKernel() && pkt->req->isRelease()) {
629 Wavefront *w =
630 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
631
632 // Check if we are waiting on Kernel End Release
633 if (w->status == Wavefront::S_RETURNING) {
634 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
635 computeUnit->cu_id, w->simdId, w->wfSlotId,
636 w->wfDynId, w->kernId);
637
638 computeUnit->shader->dispatcher->notifyWgCompl(w);
639 w->status = Wavefront::S_STOPPED;
640 } else {
641 w->outstandingReqs--;
642 }
643
644 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
645 computeUnit->cu_id, gpuDynInst->simdId,
646 gpuDynInst->wfSlotId, w->barrierCnt);
647
648 if (gpuDynInst->useContinuation) {
649 assert(!gpuDynInst->isNoScope());
650 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
651 gpuDynInst);
652 }
653
654 delete pkt->senderState;
655 delete pkt->req;
656 delete pkt;
657 return true;
658 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
659 if (gpuDynInst->useContinuation) {
660 assert(!gpuDynInst->isNoScope());
661 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
662 gpuDynInst);
663 }
664
665 delete pkt->senderState;
666 delete pkt->req;
667 delete pkt;
668 return true;
669 }
670
671 ComputeUnit::DataPort::MemRespEvent *mem_resp_event =
672 new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index],
673 pkt);
674
675 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
676 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
677 index, pkt->req->getPaddr());
678
679 computeUnit->schedule(mem_resp_event,
680 curTick() + computeUnit->resp_tick_latency);
681 return true;
682}
683
684void
685ComputeUnit::DataPort::recvReqRetry()
686{
687 int len = retries.size();
688
689 assert(len > 0);
690
691 for (int i = 0; i < len; ++i) {
692 PacketPtr pkt = retries.front().first;
693 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
694 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
695 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
696 pkt->req->getPaddr());
697
698 /** Currently Ruby can return false due to conflicts for the particular
699 * cache block or address. Thus other requests should be allowed to
700 * pass and the data port should expect multiple retries. */
701 if (!sendTimingReq(pkt)) {
702 DPRINTF(GPUMem, "failed again!\n");
703 break;
704 } else {
705 DPRINTF(GPUMem, "successful!\n");
706 retries.pop_front();
707 }
708 }
709}
710
711bool
712ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
713{
714 computeUnit->fetchStage.processFetchReturn(pkt);
715
716 return true;
717}
718
719void
720ComputeUnit::SQCPort::recvReqRetry()
721{
722 int len = retries.size();
723
724 assert(len > 0);
725
726 for (int i = 0; i < len; ++i) {
727 PacketPtr pkt = retries.front().first;
728 Wavefront *wavefront M5_VAR_USED = retries.front().second;
729 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
730 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
731 pkt->req->getPaddr());
732 if (!sendTimingReq(pkt)) {
733 DPRINTF(GPUFetch, "failed again!\n");
734 break;
735 } else {
736 DPRINTF(GPUFetch, "successful!\n");
737 retries.pop_front();
738 }
739 }
740}
741
742void
743ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
744{
745 // There must be a way around this check to do the globalMemStart...
746 Addr tmp_vaddr = pkt->req->getVaddr();
747
748 updatePageDivergenceDist(tmp_vaddr);
749
750 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
751 pkt->req->getFlags(), pkt->req->masterId(),
752 pkt->req->getPC());
753
754 // figure out the type of the request to set read/write
755 BaseTLB::Mode TLB_mode;
756 assert(pkt->isRead() || pkt->isWrite());
757
758 // Check write before read for atomic operations
759 // since atomic operations should use BaseTLB::Write
760 if (pkt->isWrite()){
761 TLB_mode = BaseTLB::Write;
762 } else if (pkt->isRead()) {
763 TLB_mode = BaseTLB::Read;
764 } else {
765 fatal("pkt is not a read nor a write\n");
766 }
767
768 tlbCycles -= curTick();
769 ++tlbRequests;
770
771 int tlbPort_index = perLaneTLB ? index : 0;
772
773 if (shader->timingSim) {
774 if (debugSegFault) {
775 Process *p = shader->gpuTc->getProcessPtr();
776 Addr vaddr = pkt->req->getVaddr();
777 unsigned size = pkt->getSize();
778
779 if ((vaddr + size - 1) % 64 < vaddr % 64) {
780 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
781 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
782 }
783
784 Addr paddr;
785
786 if (!p->pTable->translate(vaddr, paddr)) {
787 if (!p->fixupStackFault(vaddr)) {
788 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
789 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
790 vaddr);
791 }
792 }
793 }
794
795 // This is the SenderState needed upon return
796 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
797
798 // This is the senderState needed by the TLB hierarchy to function
799 TheISA::GpuTLB::TranslationState *translation_state =
800 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
801 pkt->senderState);
802
803 pkt->senderState = translation_state;
804
805 if (functionalTLB) {
806 tlbPort[tlbPort_index]->sendFunctional(pkt);
807
808 // update the hitLevel distribution
809 int hit_level = translation_state->hitLevel;
810 assert(hit_level != -1);
811 hitsPerTLBLevel[hit_level]++;
812
813 // New SenderState for the memory access
814 X86ISA::GpuTLB::TranslationState *sender_state =
815 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
816
817 delete sender_state->tlbEntry;
818 delete sender_state->saved;
819 delete sender_state;
820
821 assert(pkt->req->hasPaddr());
822 assert(pkt->req->hasSize());
823
824 uint8_t *tmpData = pkt->getPtr<uint8_t>();
825
826 // this is necessary because the GPU TLB receives packets instead
827 // of requests. when the translation is complete, all relevent
828 // fields in the request will be populated, but not in the packet.
829 // here we create the new packet so we can set the size, addr,
830 // and proper flags.
831 PacketPtr oldPkt = pkt;
832 pkt = new Packet(oldPkt->req, oldPkt->cmd);
833 delete oldPkt;
834 pkt->dataStatic(tmpData);
835
836
837 // New SenderState for the memory access
838 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
839 index, nullptr);
840
841 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
842 gpuDynInst->tlbHitLevel[index] = hit_level;
843
844
845 // translation is done. Schedule the mem_req_event at the
846 // appropriate cycle to send the timing memory request to ruby
847 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
848 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
849
850 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
851 "scheduled\n", cu_id, gpuDynInst->simdId,
852 gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
853
854 schedule(mem_req_event, curTick() + req_tick_latency);
855 } else if (tlbPort[tlbPort_index]->isStalled()) {
856 assert(tlbPort[tlbPort_index]->retries.size() > 0);
857
858 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
859 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
860 tmp_vaddr);
861
862 tlbPort[tlbPort_index]->retries.push_back(pkt);
863 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
864 // Stall the data port;
865 // No more packet will be issued till
866 // ruby indicates resources are freed by
867 // a recvReqRetry() call back on this port.
868 tlbPort[tlbPort_index]->stallPort();
869
870 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
871 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
872 tmp_vaddr);
873
874 tlbPort[tlbPort_index]->retries.push_back(pkt);
875 } else {
876 DPRINTF(GPUTLB,
877 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
878 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
879 }
880 } else {
881 if (pkt->cmd == MemCmd::MemFenceReq) {
882 gpuDynInst->statusBitVector = VectorMask(0);
883 } else {
884 gpuDynInst->statusBitVector &= (~(1ll << index));
885 }
886
887 // New SenderState for the memory access
888 delete pkt->senderState;
889
890 // Because it's atomic operation, only need TLB translation state
891 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
892 shader->gpuTc);
893
894 tlbPort[tlbPort_index]->sendFunctional(pkt);
895
896 // the addr of the packet is not modified, so we need to create a new
897 // packet, or otherwise the memory access will have the old virtual
898 // address sent in the translation packet, instead of the physical
899 // address returned by the translation.
900 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
901 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
902
903 // Translation is done. It is safe to send the packet to memory.
904 memPort[0]->sendFunctional(new_pkt);
905
906 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
907 gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
908 new_pkt->req->getPaddr());
909
910 // safe_cast the senderState
911 TheISA::GpuTLB::TranslationState *sender_state =
912 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
913
914 delete sender_state->tlbEntry;
915 delete new_pkt;
916 delete pkt->senderState;
917 delete pkt->req;
918 delete pkt;
919 }
920}
921
922void
923ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
924{
925 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
926 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
927
928
929 // New SenderState for the memory access
930 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
931 nullptr);
932
933 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
935 pkt->req->getPaddr());
936
937 schedule(mem_req_event, curTick() + req_tick_latency);
938}
939
940void
941ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
942 Request* req)
943{
944 assert(gpuDynInst->isGlobalSeg());
945
946 if (!req) {
947 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
948 }
949 req->setPaddr(0);
950 if (kernelLaunch) {
951 req->setFlags(Request::KERNEL);
952 }
953
954 // for non-kernel MemFence operations, memorder flags are set depending
955 // on which type of request is currently being sent, so this
956 // should be set by the caller (e.g. if an inst has acq-rel
957 // semantics, it will send one acquire req an one release req)
958 gpuDynInst->setRequestFlags(req, kernelLaunch);
959
960 // a mem fence must correspond to an acquire/release request
961 assert(req->isAcquire() || req->isRelease());
962
963 // create packet
964 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
965
966 // set packet's sender state
967 pkt->senderState =
968 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
969
970 // send the packet
971 sendSyncRequest(gpuDynInst, 0, pkt);
972}
973
974const char*
975ComputeUnit::DataPort::MemRespEvent::description() const
976{
977 return "ComputeUnit memory response event";
978}
979
980void
981ComputeUnit::DataPort::MemRespEvent::process()
982{
983 DataPort::SenderState *sender_state =
984 safe_cast<DataPort::SenderState*>(pkt->senderState);
985
986 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
987 ComputeUnit *compute_unit = dataPort->computeUnit;
988
989 assert(gpuDynInst);
990
991 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
992 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
993 pkt->req->getPaddr(), dataPort->index);
994
995 Addr paddr = pkt->req->getPaddr();
996
997 if (pkt->cmd != MemCmd::MemFenceResp) {
998 int index = gpuDynInst->memStatusVector[paddr].back();
999
1000 DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
1001 pkt->req->getPaddr(), index);
1002
1003 gpuDynInst->memStatusVector[paddr].pop_back();
1004 gpuDynInst->pAddr = pkt->req->getPaddr();
1005
1006 if (pkt->isRead() || pkt->isWrite()) {
1007
1008 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1009 gpuDynInst->statusBitVector &= (~(1ULL << index));
1010 } else {
1011 assert(gpuDynInst->statusVector[index] > 0);
1012 gpuDynInst->statusVector[index]--;
1013
1014 if (!gpuDynInst->statusVector[index])
1015 gpuDynInst->statusBitVector &= (~(1ULL << index));
1016 }
1017
1018 DPRINTF(GPUMem, "bitvector is now %#x\n",
1019 gpuDynInst->statusBitVector);
1020
1021 if (gpuDynInst->statusBitVector == VectorMask(0)) {
1022 auto iter = gpuDynInst->memStatusVector.begin();
1023 auto end = gpuDynInst->memStatusVector.end();
1024
1025 while (iter != end) {
1026 assert(iter->second.empty());
1027 ++iter;
1028 }
1029
1030 gpuDynInst->memStatusVector.clear();
1031
1032 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1033 gpuDynInst->statusVector.clear();
1034
1035 if (gpuDynInst->isLoad() || gpuDynInst->isAtomic()) {
1036 assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy());
1037
1038 compute_unit->globalMemoryPipe.getGMLdRespFIFO()
1039 .push(gpuDynInst);
1040 } else {
1041 assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy());
1042
1043 compute_unit->globalMemoryPipe.getGMStRespFIFO()
1044 .push(gpuDynInst);
1045 }
1046
1047 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1048 compute_unit->cu_id, gpuDynInst->simdId,
1049 gpuDynInst->wfSlotId);
1050
1051 // after clearing the status vectors,
1052 // see if there is a continuation to perform
1053 // the continuation may generate more work for
1054 // this memory request
1055 if (gpuDynInst->useContinuation) {
1056 assert(!gpuDynInst->isNoScope());
1057 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1058 gpuDynInst);
1059 }
1060 }
1061 }
1062 } else {
1063 gpuDynInst->statusBitVector = VectorMask(0);
1064
1065 if (gpuDynInst->useContinuation) {
1066 assert(!gpuDynInst->isNoScope());
1067 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1068 gpuDynInst);
1069 }
1070 }
1071
1072 delete pkt->senderState;
1073 delete pkt->req;
1074 delete pkt;
1075}
1076
1077ComputeUnit*
1078ComputeUnitParams::create()
1079{
1080 return new ComputeUnit(this);
1081}
1082
1083bool
1084ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1085{
1086 Addr line = pkt->req->getPaddr();
1087
1088 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1089 pkt->req->getVaddr(), line);
1090
1091 assert(pkt->senderState);
1092 computeUnit->tlbCycles += curTick();
1093
1094 // pop off the TLB translation state
1095 TheISA::GpuTLB::TranslationState *translation_state =
1096 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1097
1098 // no PageFaults are permitted for data accesses
1099 if (!translation_state->tlbEntry->valid) {
1100 DTLBPort::SenderState *sender_state =
1101 safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1102
1103 Wavefront *w M5_VAR_USED =
1104 computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1105 [sender_state->_gpuDynInst->wfSlotId];
1106
1107 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1108 pkt->req->getVaddr());
1109 }
1110
1111 assert(translation_state->tlbEntry->valid);
1112
1113 // update the hitLevel distribution
1114 int hit_level = translation_state->hitLevel;
1115 computeUnit->hitsPerTLBLevel[hit_level]++;
1116
1117 delete translation_state->tlbEntry;
1118 assert(!translation_state->ports.size());
1119 pkt->senderState = translation_state->saved;
1120
1121 // for prefetch pkt
1122 BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1123
1124 delete translation_state;
1125
1126 // use the original sender state to know how to close this transaction
1127 DTLBPort::SenderState *sender_state =
1128 safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1129
1130 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1131 int mp_index = sender_state->portIndex;
1132 Addr vaddr = pkt->req->getVaddr();
1133 gpuDynInst->memStatusVector[line].push_back(mp_index);
1134 gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1135
1136 MemCmd requestCmd;
1137
1138 if (pkt->cmd == MemCmd::ReadResp) {
1139 requestCmd = MemCmd::ReadReq;
1140 } else if (pkt->cmd == MemCmd::WriteResp) {
1141 requestCmd = MemCmd::WriteReq;
1142 } else if (pkt->cmd == MemCmd::SwapResp) {
1143 requestCmd = MemCmd::SwapReq;
1144 } else {
1145 panic("unsupported response to request conversion %s\n",
1146 pkt->cmd.toString());
1147 }
1148
1149 if (computeUnit->prefetchDepth) {
1150 int simdId = gpuDynInst->simdId;
1151 int wfSlotId = gpuDynInst->wfSlotId;
1152 Addr last = 0;
1153
1154 switch(computeUnit->prefetchType) {
1155 case Enums::PF_CU:
1156 last = computeUnit->lastVaddrCU[mp_index];
1157 break;
1158 case Enums::PF_PHASE:
1159 last = computeUnit->lastVaddrSimd[simdId][mp_index];
1160 break;
1161 case Enums::PF_WF:
1162 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1163 default:
1164 break;
1165 }
1166
1167 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1168 computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1169
1170 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1171 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1172 : 0;
1173
1174 DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1175
1176 computeUnit->lastVaddrCU[mp_index] = vaddr;
1177 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1178 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1179
1180 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1181 computeUnit->prefetchStride: stride;
1182
1183 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1184 computeUnit->cu_id, simdId, wfSlotId, mp_index);
1185
1186 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1187
1188 // Prefetch Next few pages atomically
1189 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1190 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1191 vaddr+stride*pf*TheISA::PageBytes);
1192
1193 if (!stride)
1194 break;
1195
1196 Request *prefetch_req = new Request(0, vaddr + stride * pf *
1197 TheISA::PageBytes,
1198 sizeof(uint8_t), 0,
1199 computeUnit->masterId(),
1200 0, 0, 0);
1201
1202 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1203 uint8_t foo = 0;
1204 prefetch_pkt->dataStatic(&foo);
1205
1206 // Because it's atomic operation, only need TLB translation state
1207 prefetch_pkt->senderState =
1208 new TheISA::GpuTLB::TranslationState(TLB_mode,
1209 computeUnit->shader->gpuTc,
1210 true);
1211
1212 // Currently prefetches are zero-latency, hence the sendFunctional
1213 sendFunctional(prefetch_pkt);
1214
1215 /* safe_cast the senderState */
1216 TheISA::GpuTLB::TranslationState *tlb_state =
1217 safe_cast<TheISA::GpuTLB::TranslationState*>(
1218 prefetch_pkt->senderState);
1219
1220
1221 delete tlb_state->tlbEntry;
1222 delete tlb_state;
1223 delete prefetch_pkt->req;
1224 delete prefetch_pkt;
1225 }
1226 }
1227
1228 // First we must convert the response cmd back to a request cmd so that
1229 // the request can be sent through the cu's master port
1230 PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1231 new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1232 delete pkt->senderState;
1233 delete pkt;
1234
1235 // New SenderState for the memory access
1236 new_pkt->senderState =
1237 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1238 nullptr);
1239
1240 // translation is done. Schedule the mem_req_event at the appropriate
1241 // cycle to send the timing memory request to ruby
1242 ComputeUnit::DataPort::MemReqEvent *mem_req_event =
1243 new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index],
1244 new_pkt);
1245
1246 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1247 computeUnit->cu_id, gpuDynInst->simdId,
1248 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1249
1250 computeUnit->schedule(mem_req_event, curTick() +
1251 computeUnit->req_tick_latency);
1252
1253 return true;
1254}
1255
1256const char*
1257ComputeUnit::DataPort::MemReqEvent::description() const
1258{
1259 return "ComputeUnit memory request event";
1260}
1261
1262void
1263ComputeUnit::DataPort::MemReqEvent::process()
1264{
1265 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1266 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1267 ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit;
1268
1269 if (!(dataPort->sendTimingReq(pkt))) {
1270 dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst));
1271
1272 DPRINTF(GPUPort,
1273 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1274 compute_unit->cu_id, gpuDynInst->simdId,
1275 gpuDynInst->wfSlotId, dataPort->index,
1276 pkt->req->getPaddr());
1277 } else {
1278 DPRINTF(GPUPort,
1279 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1280 compute_unit->cu_id, gpuDynInst->simdId,
1281 gpuDynInst->wfSlotId, dataPort->index,
1282 pkt->req->getPaddr());
1283 }
1284}
1285
1286/*
1287 * The initial translation request could have been rejected,
1288 * if <retries> queue is not Retry sending the translation
1289 * request. sendRetry() is called from the peer port whenever
1290 * a translation completes.
1291 */
1292void
1293ComputeUnit::DTLBPort::recvReqRetry()
1294{
1295 int len = retries.size();
1296
1297 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1298 computeUnit->cu_id, len);
1299
1300 assert(len > 0);
1301 assert(isStalled());
1302 // recvReqRetry is an indication that the resource on which this
1303 // port was stalling on is freed. So, remove the stall first
1304 unstallPort();
1305
1306 for (int i = 0; i < len; ++i) {
1307 PacketPtr pkt = retries.front();
1308 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1309 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1310
1311 if (!sendTimingReq(pkt)) {
1312 // Stall port
1313 stallPort();
1314 DPRINTF(GPUTLB, ": failed again\n");
1315 break;
1316 } else {
1317 DPRINTF(GPUTLB, ": successful\n");
1318 retries.pop_front();
1319 }
1320 }
1321}
1322
1323bool
1324ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1325{
1326 Addr line M5_VAR_USED = pkt->req->getPaddr();
1327 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1328 computeUnit->cu_id, pkt->req->getVaddr(), line);
1329
1330 assert(pkt->senderState);
1331
1332 // pop off the TLB translation state
1333 TheISA::GpuTLB::TranslationState *translation_state =
1334 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1335
1336 bool success = translation_state->tlbEntry->valid;
1337 delete translation_state->tlbEntry;
1338 assert(!translation_state->ports.size());
1339 pkt->senderState = translation_state->saved;
1340 delete translation_state;
1341
1342 // use the original sender state to know how to close this transaction
1343 ITLBPort::SenderState *sender_state =
1344 safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1345
1346 // get the wavefront associated with this translation request
1347 Wavefront *wavefront = sender_state->wavefront;
1348 delete pkt->senderState;
1349
1350 if (success) {
1351 // pkt is reused in fetch(), don't delete it here. However, we must
1352 // reset the command to be a request so that it can be sent through
1353 // the cu's master port
1354 assert(pkt->cmd == MemCmd::ReadResp);
1355 pkt->cmd = MemCmd::ReadReq;
1356
1357 computeUnit->fetchStage.fetch(pkt, wavefront);
1358 } else {
1359 if (wavefront->dropFetch) {
1360 assert(wavefront->instructionBuffer.empty());
1361 wavefront->dropFetch = false;
1362 }
1363
1364 wavefront->pendingFetch = 0;
1365 }
1366
1367 return true;
1368}
1369
1370/*
1371 * The initial translation request could have been rejected, if
1372 * <retries> queue is not empty. Retry sending the translation
1373 * request. sendRetry() is called from the peer port whenever
1374 * a translation completes.
1375 */
1376void
1377ComputeUnit::ITLBPort::recvReqRetry()
1378{
1379
1380 int len = retries.size();
1381 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1382
1383 assert(len > 0);
1384 assert(isStalled());
1385
1386 // recvReqRetry is an indication that the resource on which this
1387 // port was stalling on is freed. So, remove the stall first
1388 unstallPort();
1389
1390 for (int i = 0; i < len; ++i) {
1391 PacketPtr pkt = retries.front();
1392 Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1393 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1394
1395 if (!sendTimingReq(pkt)) {
1396 stallPort(); // Stall port
1397 DPRINTF(GPUTLB, ": failed again\n");
1398 break;
1399 } else {
1400 DPRINTF(GPUTLB, ": successful\n");
1401 retries.pop_front();
1402 }
1403 }
1404}
1405
1406void
1407ComputeUnit::regStats()
1408{
1409 MemObject::regStats();
1410
|