compute_unit.cc revision 11698:d1ad31187fa5 
1/*
2 * Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Author: John Kalamatianos, Anthony Gutierrez
34 */
35#include "gpu-compute/compute_unit.hh"
36
37#include <limits>
38
39#include "base/output.hh"
40#include "debug/GPUDisp.hh"
41#include "debug/GPUExec.hh"
42#include "debug/GPUFetch.hh"
43#include "debug/GPUMem.hh"
44#include "debug/GPUPort.hh"
45#include "debug/GPUPrefetch.hh"
46#include "debug/GPUSync.hh"
47#include "debug/GPUTLB.hh"
48#include "gpu-compute/dispatcher.hh"
49#include "gpu-compute/gpu_dyn_inst.hh"
50#include "gpu-compute/gpu_static_inst.hh"
51#include "gpu-compute/ndrange.hh"
52#include "gpu-compute/shader.hh"
53#include "gpu-compute/simple_pool_manager.hh"
54#include "gpu-compute/vector_register_file.hh"
55#include "gpu-compute/wavefront.hh"
56#include "mem/page_table.hh"
57#include "sim/process.hh"
58
59ComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p),
60    scoreboardCheckStage(p), scheduleStage(p), execStage(p),
61    globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
62    cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
63    spBypassPipeLength(p->spbypass_pipe_length),
64    dpBypassPipeLength(p->dpbypass_pipe_length),
65    issuePeriod(p->issue_period),
66    numGlbMemUnits(p->num_global_mem_pipes),
67    numLocMemUnits(p->num_shared_mem_pipes),
68    perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
69    prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
70    xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
71    functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
72    countPages(p->countPages), barrier_id(0),
73    vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
74    coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
75    req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
76    resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
77    _masterId(p->system->getMasterId(name() + ".ComputeUnit")),
78    lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
79    globalSeqNum(0), wavefrontSize(p->wfSize),
80    kernelLaunchInst(new KernelLaunchStaticInst())
81{
82    /**
83     * This check is necessary because std::bitset only provides conversion
84     * to unsigned long or unsigned long long via to_ulong() or to_ullong().
85     * there are * a few places in the code where to_ullong() is used, however
86     * if VSZ is larger than a value the host can support then bitset will
87     * throw a runtime exception. we should remove all use of to_long() or
88     * to_ullong() so we can have VSZ greater than 64b, however until that is
89     * done this assert is required.
90     */
91    fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
92             p->wfSize <= 0,
93             "WF size is larger than the host can support");
94    fatal_if(!isPowerOf2(wavefrontSize),
95             "Wavefront size should be a power of 2");
96    // calculate how many cycles a vector load or store will need to transfer
97    // its data over the corresponding buses
98    numCyclesPerStoreTransfer =
99        (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
100                (double)vrfToCoalescerBusWidth);
101
102    numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
103                               / coalescerToVrfBusWidth;
104
105    lastVaddrWF.resize(numSIMDs);
106    wfList.resize(numSIMDs);
107
108    for (int j = 0; j < numSIMDs; ++j) {
109        lastVaddrWF[j].resize(p->n_wf);
110
111        for (int i = 0; i < p->n_wf; ++i) {
112            lastVaddrWF[j][i].resize(wfSize());
113
114            wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
115            wfList[j][i]->setParent(this);
116
117            for (int k = 0; k < wfSize(); ++k) {
118                lastVaddrWF[j][i][k] = 0;
119            }
120        }
121    }
122
123    lastVaddrSimd.resize(numSIMDs);
124
125    for (int i = 0; i < numSIMDs; ++i) {
126        lastVaddrSimd[i].resize(wfSize(), 0);
127    }
128
129    lastVaddrCU.resize(wfSize());
130
131    lds.setParent(this);
132
133    if (p->execPolicy == "OLDEST-FIRST") {
134        exec_policy = EXEC_POLICY::OLDEST;
135    } else if (p->execPolicy == "ROUND-ROBIN") {
136        exec_policy = EXEC_POLICY::RR;
137    } else {
138        fatal("Invalid WF execution policy (CU)\n");
139    }
140
141    memPort.resize(wfSize());
142
143    // resize the tlbPort vectorArray
144    int tlbPort_width = perLaneTLB ? wfSize() : 1;
145    tlbPort.resize(tlbPort_width);
146
147    cuExitCallback = new CUExitCallback(this);
148    registerExitCallback(cuExitCallback);
149
150    xactCasLoadMap.clear();
151    lastExecCycle.resize(numSIMDs, 0);
152
153    for (int i = 0; i < vrf.size(); ++i) {
154        vrf[i]->setParent(this);
155    }
156
157    numVecRegsPerSimd = vrf[0]->numRegs();
158}
159
160ComputeUnit::~ComputeUnit()
161{
162    // Delete wavefront slots
163    for (int j = 0; j < numSIMDs; ++j) {
164        for (int i = 0; i < shader->n_wf; ++i) {
165            delete wfList[j][i];
166        }
167        lastVaddrSimd[j].clear();
168    }
169    lastVaddrCU.clear();
170    readyList.clear();
171    waveStatusList.clear();
172    dispatchList.clear();
173    vectorAluInstAvail.clear();
174    delete cuExitCallback;
175    delete ldsPort;
176}
177
178void
179ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
180{
181    w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
182
183    w->workGroupSz[0] = ndr->q.wgSize[0];
184    w->workGroupSz[1] = ndr->q.wgSize[1];
185    w->workGroupSz[2] = ndr->q.wgSize[2];
186    w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
187    w->gridSz[0] = ndr->q.gdSize[0];
188    w->gridSz[1] = ndr->q.gdSize[1];
189    w->gridSz[2] = ndr->q.gdSize[2];
190    w->kernelArgs = ndr->q.args;
191    w->privSizePerItem = ndr->q.privMemPerItem;
192    w->spillSizePerItem = ndr->q.spillMemPerItem;
193    w->roBase = ndr->q.roMemStart;
194    w->roSize = ndr->q.roMemTotal;
195    w->computeActualWgSz(ndr);
196}
197
198void
199ComputeUnit::updateEvents() {
200
201    if (!timestampVec.empty()) {
202        uint32_t vecSize = timestampVec.size();
203        uint32_t i = 0;
204        while (i < vecSize) {
205            if (timestampVec[i] <= shader->tick_cnt) {
206                std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
207                vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
208                                            statusVec[i]);
209                timestampVec.erase(timestampVec.begin() + i);
210                regIdxVec.erase(regIdxVec.begin() + i);
211                statusVec.erase(statusVec.begin() + i);
212                --vecSize;
213                --i;
214            }
215            ++i;
216        }
217    }
218
219    for (int i = 0; i< numSIMDs; ++i) {
220        vrf[i]->updateEvents();
221    }
222}
223
224
225void
226ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
227                            NDRange *ndr)
228{
229    static int _n_wave = 0;
230
231    VectorMask init_mask;
232    init_mask.reset();
233
234    for (int k = 0; k < wfSize(); ++k) {
235        if (k + waveId * wfSize() < w->actualWgSzTotal)
236            init_mask[k] = 1;
237    }
238
239    w->kernId = ndr->dispatchId;
240    w->wfId = waveId;
241    w->initMask = init_mask.to_ullong();
242
243    for (int k = 0; k < wfSize(); ++k) {
244        w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
245        w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
246                             w->actualWgSz[1];
247        w->workItemId[2][k] = (k + waveId * wfSize()) /
248                              (w->actualWgSz[0] * w->actualWgSz[1]);
249
250        w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
251            w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
252            w->workItemId[0][k];
253    }
254
255    w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
256
257    w->barCnt.resize(wfSize(), 0);
258
259    w->maxBarCnt = 0;
260    w->oldBarrierCnt = 0;
261    w->barrierCnt = 0;
262
263    w->privBase = ndr->q.privMemStart;
264    ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
265
266    w->spillBase = ndr->q.spillMemStart;
267    ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
268
269    w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
270
271    // WG state
272    w->wgId = ndr->globalWgId;
273    w->dispatchId = ndr->dispatchId;
274    w->workGroupId[0] = w->wgId % ndr->numWg[0];
275    w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
276    w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
277
278    w->barrierId = barrier_id;
279    w->stalledAtBarrier = false;
280
281    // set the wavefront context to have a pointer to this section of the LDS
282    w->ldsChunk = ldsChunk;
283
284    int32_t refCount M5_VAR_USED =
285                    lds.increaseRefCounter(w->dispatchId, w->wgId);
286    DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
287                    cu_id, w->wgId, refCount);
288
289    w->instructionBuffer.clear();
290
291    if (w->pendingFetch)
292        w->dropFetch = true;
293
294    // is this the last wavefront in the workgroup
295    // if set the spillWidth to be the remaining work-items
296    // so that the vector access is correct
297    if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
298        w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
299    } else {
300        w->spillWidth = wfSize();
301    }
302
303    DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
304            "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
305
306    w->start(++_n_wave, ndr->q.code_ptr);
307}
308
309void
310ComputeUnit::StartWorkgroup(NDRange *ndr)
311{
312    // reserve the LDS capacity allocated to the work group
313    // disambiguated by the dispatch ID and workgroup ID, which should be
314    // globally unique
315    LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
316                                          ndr->q.ldsSize);
317
318    // Send L1 cache acquire
319    // isKernel + isAcquire = Kernel Begin
320    if (shader->impl_kern_boundary_sync) {
321        GPUDynInstPtr gpuDynInst =
322            std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
323                                         getAndIncSeqNum());
324
325        gpuDynInst->useContinuation = false;
326        injectGlobalMemFence(gpuDynInst, true);
327    }
328
329    // calculate the number of 32-bit vector registers required by wavefront
330    int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
331    int wave_id = 0;
332
333    // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
334    for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
335        Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
336        // Check if this wavefront slot is available:
337        // It must be stopped and not waiting
338        // for a release to complete S_RETURNING
339        if (w->status == Wavefront::S_STOPPED) {
340            fillKernelState(w, ndr);
341            // if we have scheduled all work items then stop
342            // scheduling wavefronts
343            if (wave_id * wfSize() >= w->actualWgSzTotal)
344                break;
345
346            // reserve vector registers for the scheduled wavefront
347            assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
348            uint32_t normSize = 0;
349
350            w->startVgprIndex = vrf[m % numSIMDs]->manager->
351                                    allocateRegion(vregDemand, &normSize);
352
353            w->reservedVectorRegs = normSize;
354            vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
355
356            startWavefront(w, wave_id, ldsChunk, ndr);
357            ++wave_id;
358        }
359    }
360    ++barrier_id;
361}
362
363int
364ComputeUnit::ReadyWorkgroup(NDRange *ndr)
365{
366    // Get true size of workgroup (after clamping to grid size)
367    int trueWgSize[3];
368    int trueWgSizeTotal = 1;
369
370    for (int d = 0; d < 3; ++d) {
371        trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
372                                 ndr->wgId[d] * ndr->q.wgSize[d]);
373
374        trueWgSizeTotal *= trueWgSize[d];
375        DPRINTF(GPUDisp, "trueWgSize[%d] =  %d\n", d, trueWgSize[d]);
376    }
377
378    DPRINTF(GPUDisp, "trueWgSizeTotal =  %d\n", trueWgSizeTotal);
379
380    // calculate the number of 32-bit vector registers required by each
381    // work item of the work group
382    int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
383    bool vregAvail = true;
384    int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
385    int freeWfSlots = 0;
386    // check if the total number of VGPRs required by all WFs of the WG
387    // fit in the VRFs of all SIMD units
388    assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
389    int numMappedWfs = 0;
390    std::vector<int> numWfsPerSimd;
391    numWfsPerSimd.resize(numSIMDs, 0);
392    // find how many free WF slots we have across all SIMDs
393    for (int j = 0; j < shader->n_wf; ++j) {
394        for (int i = 0; i < numSIMDs; ++i) {
395            if (wfList[i][j]->status == Wavefront::S_STOPPED) {
396                // count the number of free WF slots
397                ++freeWfSlots;
398                if (numMappedWfs < numWfs) {
399                    // count the WFs to be assigned per SIMD
400                    numWfsPerSimd[i]++;
401                }
402                numMappedWfs++;
403            }
404        }
405    }
406
407    // if there are enough free WF slots then find if there are enough
408    // free VGPRs per SIMD based on the WF->SIMD mapping
409    if (freeWfSlots >= numWfs) {
410        for (int j = 0; j < numSIMDs; ++j) {
411            // find if there are enough free VGPR regions in the SIMD's VRF
412            // to accommodate the WFs of the new WG that would be mapped to
413            // this SIMD unit
414            vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
415                                                     vregDemandPerWI);
416
417            // stop searching if there is at least one SIMD
418            // whose VRF does not have enough free VGPR pools.
419            // This is because a WG is scheduled only if ALL
420            // of its WFs can be scheduled
421            if (!vregAvail)
422                break;
423        }
424    }
425
426    DPRINTF(GPUDisp, "Free WF slots =  %d, VGPR Availability = %d\n",
427            freeWfSlots, vregAvail);
428
429    if (!vregAvail) {
430        ++numTimesWgBlockedDueVgprAlloc;
431    }
432
433    // Return true if enough WF slots to submit workgroup and if there are
434    // enough VGPRs to schedule all WFs to their SIMD units
435    if (!lds.canReserve(ndr->q.ldsSize)) {
436        wgBlockedDueLdsAllocation++;
437    }
438
439    // Return true if (a) there are enough free WF slots to submit
440    // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
441    // SIMD units and (c) if there is enough space in LDS
442    return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
443}
444
445int
446ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
447{
448    DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
449    int ccnt = 0;
450
451    for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
452        for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
453            Wavefront *w = wfList[i_simd][i_wf];
454
455            if (w->status == Wavefront::S_RUNNING) {
456                DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
457
458                DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
459                        w->barrierId, _barrier_id);
460
461                DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
462                        w->barrierCnt, bcnt);
463            }
464
465            if (w->status == Wavefront::S_RUNNING &&
466                w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
467                !w->outstandingReqs) {
468                ++ccnt;
469
470                DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
471                        "%d\n", i_simd, i_wf, ccnt);
472            }
473        }
474    }
475
476    DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
477            cu_id, ccnt, bslots);
478
479    return ccnt == bslots;
480}
481
482//  Check if the current wavefront is blocked on additional resources.
483bool
484ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
485{
486    bool cede = false;
487
488    // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
489    // magic instructions will impact the scheduling of wavefronts
490    if (xact_cas_mode) {
491        /*
492         * When a wavefront calls xact_cas_ld, it adds itself to a per address
493         * queue. All per address queues are managed by the xactCasLoadMap.
494         *
495         * A wavefront is not blocked if: it is not in ANY per address queue or
496         * if it is at the head of a per address queue.
497         */
498        for (auto itMap : xactCasLoadMap) {
499            std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
500
501            if (!curWaveIDQueue.empty()) {
502                for (auto it : curWaveIDQueue) {
503                    waveIdentifier cur_wave = it;
504
505                    if (cur_wave.simdId == simdId &&
506                        cur_wave.wfSlotId == wfSlotId) {
507                        // 2 possibilities
508                        // 1: this WF has a green light
509                        // 2: another WF has a green light
510                        waveIdentifier owner_wave = curWaveIDQueue.front();
511
512                        if (owner_wave.simdId != cur_wave.simdId ||
513                            owner_wave.wfSlotId != cur_wave.wfSlotId) {
514                            // possibility 2
515                            cede = true;
516                            break;
517                        } else {
518                            // possibility 1
519                            break;
520                        }
521                    }
522                }
523            }
524        }
525    }
526
527    return cede;
528}
529
530// Execute one clock worth of work on the ComputeUnit.
531void
532ComputeUnit::exec()
533{
534    updateEvents();
535    // Execute pipeline stages in reverse order to simulate
536    // the pipeline latency
537    globalMemoryPipe.exec();
538    localMemoryPipe.exec();
539    execStage.exec();
540    scheduleStage.exec();
541    scoreboardCheckStage.exec();
542    fetchStage.exec();
543
544    totalCycles++;
545}
546
547void
548ComputeUnit::init()
549{
550    // Initialize CU Bus models
551    glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
552    locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
553    nextGlbMemBus = 0;
554    nextLocMemBus = 0;
555    fatal_if(numGlbMemUnits > 1,
556             "No support for multiple Global Memory Pipelines exists!!!");
557    vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
558    for (int j = 0; j < numGlbMemUnits; ++j) {
559        vrfToGlobalMemPipeBus[j] = WaitClass();
560        vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
561    }
562
563    fatal_if(numLocMemUnits > 1,
564             "No support for multiple Local Memory Pipelines exists!!!");
565    vrfToLocalMemPipeBus.resize(numLocMemUnits);
566    for (int j = 0; j < numLocMemUnits; ++j) {
567        vrfToLocalMemPipeBus[j] = WaitClass();
568        vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
569    }
570    vectorRegsReserved.resize(numSIMDs, 0);
571    aluPipe.resize(numSIMDs);
572    wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
573
574    for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
575        wfWait[i] = WaitClass();
576        wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
577    }
578
579    for (int i = 0; i < numSIMDs; ++i) {
580        aluPipe[i] = WaitClass();
581        aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
582    }
583
584    // Setup space for call args
585    for (int j = 0; j < numSIMDs; ++j) {
586        for (int i = 0; i < shader->n_wf; ++i) {
587            wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
588        }
589    }
590
591    // Initializing pipeline resources
592    readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
593    waveStatusList.resize(numSIMDs);
594
595    for (int j = 0; j < numSIMDs; ++j) {
596        for (int i = 0; i < shader->n_wf; ++i) {
597            waveStatusList[j].push_back(
598                std::make_pair(wfList[j][i], BLOCKED));
599        }
600    }
601
602    for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
603        dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
604    }
605
606    fetchStage.init(this);
607    scoreboardCheckStage.init(this);
608    scheduleStage.init(this);
609    execStage.init(this);
610    globalMemoryPipe.init(this);
611    localMemoryPipe.init(this);
612    // initialize state for statistics calculation
613    vectorAluInstAvail.resize(numSIMDs, false);
614    shrMemInstAvail = 0;
615    glbMemInstAvail = 0;
616}
617
618bool
619ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
620{
621    // Ruby has completed the memory op. Schedule the mem_resp_event at the
622    // appropriate cycle to process the timing memory response
623    // This delay represents the pipeline delay
624    SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
625    int index = sender_state->port_index;
626    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
627
628    // Is the packet returned a Kernel End or Barrier
629    if (pkt->req->isKernel() && pkt->req->isRelease()) {
630        Wavefront *w =
631            computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
632
633        // Check if we are waiting on Kernel End Release
634        if (w->status == Wavefront::S_RETURNING) {
635            DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
636                    computeUnit->cu_id, w->simdId, w->wfSlotId,
637                    w->wfDynId, w->kernId);
638
639            computeUnit->shader->dispatcher->notifyWgCompl(w);
640            w->status = Wavefront::S_STOPPED;
641        } else {
642            w->outstandingReqs--;
643        }
644
645        DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
646                computeUnit->cu_id, gpuDynInst->simdId,
647                gpuDynInst->wfSlotId, w->barrierCnt);
648
649        if (gpuDynInst->useContinuation) {
650            assert(!gpuDynInst->isNoScope());
651            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
652                                           gpuDynInst);
653        }
654
655        delete pkt->senderState;
656        delete pkt->req;
657        delete pkt;
658        return true;
659    } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
660        if (gpuDynInst->useContinuation) {
661            assert(!gpuDynInst->isNoScope());
662            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
663                                           gpuDynInst);
664        }
665
666        delete pkt->senderState;
667        delete pkt->req;
668        delete pkt;
669        return true;
670    }
671
672    ComputeUnit::DataPort::MemRespEvent *mem_resp_event =
673        new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index],
674                                                pkt);
675
676    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
677            computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
678            index, pkt->req->getPaddr());
679
680    computeUnit->schedule(mem_resp_event,
681                          curTick() + computeUnit->resp_tick_latency);
682    return true;
683}
684
685void
686ComputeUnit::DataPort::recvReqRetry()
687{
688    int len = retries.size();
689
690    assert(len > 0);
691
692    for (int i = 0; i < len; ++i) {
693        PacketPtr pkt = retries.front().first;
694        GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
695        DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
696                computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
697                pkt->req->getPaddr());
698
699        /** Currently Ruby can return false due to conflicts for the particular
700         *  cache block or address.  Thus other requests should be allowed to
701         *  pass and the data port should expect multiple retries. */
702        if (!sendTimingReq(pkt)) {
703            DPRINTF(GPUMem, "failed again!\n");
704            break;
705        } else {
706            DPRINTF(GPUMem, "successful!\n");
707            retries.pop_front();
708        }
709    }
710}
711
712bool
713ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
714{
715    computeUnit->fetchStage.processFetchReturn(pkt);
716
717    return true;
718}
719
720void
721ComputeUnit::SQCPort::recvReqRetry()
722{
723    int len = retries.size();
724
725    assert(len > 0);
726
727    for (int i = 0; i < len; ++i) {
728        PacketPtr pkt = retries.front().first;
729        Wavefront *wavefront M5_VAR_USED = retries.front().second;
730        DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
731                computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
732                pkt->req->getPaddr());
733        if (!sendTimingReq(pkt)) {
734            DPRINTF(GPUFetch, "failed again!\n");
735            break;
736        } else {
737            DPRINTF(GPUFetch, "successful!\n");
738            retries.pop_front();
739        }
740    }
741}
742
743void
744ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
745{
746    // There must be a way around this check to do the globalMemStart...
747    Addr tmp_vaddr = pkt->req->getVaddr();
748
749    updatePageDivergenceDist(tmp_vaddr);
750
751    pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
752                      pkt->req->getFlags(), pkt->req->masterId(),
753                      pkt->req->getPC());
754
755    // figure out the type of the request to set read/write
756    BaseTLB::Mode TLB_mode;
757    assert(pkt->isRead() || pkt->isWrite());
758
759    // Check write before read for atomic operations
760    // since atomic operations should use BaseTLB::Write
761    if (pkt->isWrite()){
762        TLB_mode = BaseTLB::Write;
763    } else if (pkt->isRead()) {
764        TLB_mode = BaseTLB::Read;
765    } else {
766        fatal("pkt is not a read nor a write\n");
767    }
768
769    tlbCycles -= curTick();
770    ++tlbRequests;
771
772    int tlbPort_index = perLaneTLB ? index : 0;
773
774    if (shader->timingSim) {
775        if (debugSegFault) {
776            Process *p = shader->gpuTc->getProcessPtr();
777            Addr vaddr = pkt->req->getVaddr();
778            unsigned size = pkt->getSize();
779
780            if ((vaddr + size - 1) % 64 < vaddr % 64) {
781                panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
782                      cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
783            }
784
785            Addr paddr;
786
787            if (!p->pTable->translate(vaddr, paddr)) {
788                if (!p->fixupStackFault(vaddr)) {
789                    panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
790                          cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
791                          vaddr);
792                }
793            }
794        }
795
796        // This is the SenderState needed upon return
797        pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
798
799        // This is the senderState needed by the TLB hierarchy to function
800        TheISA::GpuTLB::TranslationState *translation_state =
801          new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
802                                               pkt->senderState);
803
804        pkt->senderState = translation_state;
805
806        if (functionalTLB) {
807            tlbPort[tlbPort_index]->sendFunctional(pkt);
808
809            // update the hitLevel distribution
810            int hit_level = translation_state->hitLevel;
811            assert(hit_level != -1);
812            hitsPerTLBLevel[hit_level]++;
813
814            // New SenderState for the memory access
815            X86ISA::GpuTLB::TranslationState *sender_state =
816                safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
817
818            delete sender_state->tlbEntry;
819            delete sender_state->saved;
820            delete sender_state;
821
822            assert(pkt->req->hasPaddr());
823            assert(pkt->req->hasSize());
824
825            uint8_t *tmpData = pkt->getPtr<uint8_t>();
826
827            // this is necessary because the GPU TLB receives packets instead
828            // of requests. when the translation is complete, all relevent
829            // fields in the request will be populated, but not in the packet.
830            // here we create the new packet so we can set the size, addr,
831            // and proper flags.
832            PacketPtr oldPkt = pkt;
833            pkt = new Packet(oldPkt->req, oldPkt->cmd);
834            delete oldPkt;
835            pkt->dataStatic(tmpData);
836
837
838            // New SenderState for the memory access
839            pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
840                                                             index, nullptr);
841
842            gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
843            gpuDynInst->tlbHitLevel[index] = hit_level;
844
845
846            // translation is done. Schedule the mem_req_event at the
847            // appropriate cycle to send the timing memory request to ruby
848            ComputeUnit::DataPort::MemReqEvent *mem_req_event =
849                new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
850
851            DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
852                    "scheduled\n", cu_id, gpuDynInst->simdId,
853                    gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
854
855            schedule(mem_req_event, curTick() + req_tick_latency);
856        } else if (tlbPort[tlbPort_index]->isStalled()) {
857            assert(tlbPort[tlbPort_index]->retries.size() > 0);
858
859            DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
860                    "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
861                    tmp_vaddr);
862
863            tlbPort[tlbPort_index]->retries.push_back(pkt);
864        } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
865            // Stall the data port;
866            // No more packet will be issued till
867            // ruby indicates resources are freed by
868            // a recvReqRetry() call back on this port.
869            tlbPort[tlbPort_index]->stallPort();
870
871            DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
872                    "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
873                    tmp_vaddr);
874
875            tlbPort[tlbPort_index]->retries.push_back(pkt);
876        } else {
877           DPRINTF(GPUTLB,
878                   "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
879                   cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
880        }
881    } else {
882        if (pkt->cmd == MemCmd::MemFenceReq) {
883            gpuDynInst->statusBitVector = VectorMask(0);
884        } else {
885            gpuDynInst->statusBitVector &= (~(1ll << index));
886        }
887
888        // New SenderState for the memory access
889        delete pkt->senderState;
890
891        // Because it's atomic operation, only need TLB translation state
892        pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
893                                                                shader->gpuTc);
894
895        tlbPort[tlbPort_index]->sendFunctional(pkt);
896
897        // the addr of the packet is not modified, so we need to create a new
898        // packet, or otherwise the memory access will have the old virtual
899        // address sent in the translation packet, instead of the physical
900        // address returned by the translation.
901        PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
902        new_pkt->dataStatic(pkt->getPtr<uint8_t>());
903
904        // Translation is done. It is safe to send the packet to memory.
905        memPort[0]->sendFunctional(new_pkt);
906
907        DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
908                gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
909                new_pkt->req->getPaddr());
910
911        // safe_cast the senderState
912        TheISA::GpuTLB::TranslationState *sender_state =
913             safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
914
915        delete sender_state->tlbEntry;
916        delete new_pkt;
917        delete pkt->senderState;
918        delete pkt->req;
919        delete pkt;
920    }
921}
922
923void
924ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
925{
926    ComputeUnit::DataPort::MemReqEvent *mem_req_event =
927        new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt);
928
929
930    // New SenderState for the memory access
931    pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
932                                                              nullptr);
933
934    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
935            cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
936            pkt->req->getPaddr());
937
938    schedule(mem_req_event, curTick() + req_tick_latency);
939}
940
941void
942ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
943                                  Request* req)
944{
945    assert(gpuDynInst->isGlobalSeg());
946
947    if (!req) {
948        req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
949    }
950    req->setPaddr(0);
951    if (kernelLaunch) {
952        req->setFlags(Request::KERNEL);
953    }
954
955    // for non-kernel MemFence operations, memorder flags are set depending
956    // on which type of request is currently being sent, so this
957    // should be set by the caller (e.g. if an inst has acq-rel
958    // semantics, it will send one acquire req an one release req)
959    gpuDynInst->setRequestFlags(req, kernelLaunch);
960
961    // a mem fence must correspond to an acquire/release request
962    assert(req->isAcquire() || req->isRelease());
963
964    // create packet
965    PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
966
967    // set packet's sender state
968    pkt->senderState =
969        new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
970
971    // send the packet
972    sendSyncRequest(gpuDynInst, 0, pkt);
973}
974
975const char*
976ComputeUnit::DataPort::MemRespEvent::description() const
977{
978    return "ComputeUnit memory response event";
979}
980
981void
982ComputeUnit::DataPort::MemRespEvent::process()
983{
984    DataPort::SenderState *sender_state =
985        safe_cast<DataPort::SenderState*>(pkt->senderState);
986
987    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
988    ComputeUnit *compute_unit = dataPort->computeUnit;
989
990    assert(gpuDynInst);
991
992    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
993            compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
994            pkt->req->getPaddr(), dataPort->index);
995
996    Addr paddr = pkt->req->getPaddr();
997
998    if (pkt->cmd != MemCmd::MemFenceResp) {
999        int index = gpuDynInst->memStatusVector[paddr].back();
1000
1001        DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
1002                pkt->req->getPaddr(), index);
1003
1004        gpuDynInst->memStatusVector[paddr].pop_back();
1005        gpuDynInst->pAddr = pkt->req->getPaddr();
1006
1007        if (pkt->isRead() || pkt->isWrite()) {
1008
1009            if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1010                gpuDynInst->statusBitVector &= (~(1ULL << index));
1011            } else {
1012                assert(gpuDynInst->statusVector[index] > 0);
1013                gpuDynInst->statusVector[index]--;
1014
1015                if (!gpuDynInst->statusVector[index])
1016                    gpuDynInst->statusBitVector &= (~(1ULL << index));
1017            }
1018
1019            DPRINTF(GPUMem, "bitvector is now %#x\n",
1020                    gpuDynInst->statusBitVector);
1021
1022            if (gpuDynInst->statusBitVector == VectorMask(0)) {
1023                auto iter = gpuDynInst->memStatusVector.begin();
1024                auto end = gpuDynInst->memStatusVector.end();
1025
1026                while (iter != end) {
1027                    assert(iter->second.empty());
1028                    ++iter;
1029                }
1030
1031                gpuDynInst->memStatusVector.clear();
1032
1033                if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1034                    gpuDynInst->statusVector.clear();
1035
1036                if (gpuDynInst->isLoad() || gpuDynInst->isAtomic()) {
1037                    assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy());
1038
1039                    compute_unit->globalMemoryPipe.getGMLdRespFIFO()
1040                        .push(gpuDynInst);
1041                } else {
1042                    assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy());
1043
1044                    compute_unit->globalMemoryPipe.getGMStRespFIFO()
1045                        .push(gpuDynInst);
1046                }
1047
1048                DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1049                        compute_unit->cu_id, gpuDynInst->simdId,
1050                        gpuDynInst->wfSlotId);
1051
1052                // after clearing the status vectors,
1053                // see if there is a continuation to perform
1054                // the continuation may generate more work for
1055                // this memory request
1056                if (gpuDynInst->useContinuation) {
1057                    assert(!gpuDynInst->isNoScope());
1058                    gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1059                                                 gpuDynInst);
1060                }
1061            }
1062        }
1063    } else {
1064        gpuDynInst->statusBitVector = VectorMask(0);
1065
1066        if (gpuDynInst->useContinuation) {
1067            assert(!gpuDynInst->isNoScope());
1068            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1069                                         gpuDynInst);
1070        }
1071    }
1072
1073    delete pkt->senderState;
1074    delete pkt->req;
1075    delete pkt;
1076}
1077
1078ComputeUnit*
1079ComputeUnitParams::create()
1080{
1081    return new ComputeUnit(this);
1082}
1083
1084bool
1085ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1086{
1087    Addr line = pkt->req->getPaddr();
1088
1089    DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1090            pkt->req->getVaddr(), line);
1091
1092    assert(pkt->senderState);
1093    computeUnit->tlbCycles += curTick();
1094
1095    // pop off the TLB translation state
1096    TheISA::GpuTLB::TranslationState *translation_state =
1097               safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1098
1099    // no PageFaults are permitted for data accesses
1100    if (!translation_state->tlbEntry->valid) {
1101        DTLBPort::SenderState *sender_state =
1102            safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1103
1104        Wavefront *w M5_VAR_USED =
1105            computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1106            [sender_state->_gpuDynInst->wfSlotId];
1107
1108        DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1109                 pkt->req->getVaddr());
1110    }
1111
1112    assert(translation_state->tlbEntry->valid);
1113
1114    // update the hitLevel distribution
1115    int hit_level = translation_state->hitLevel;
1116    computeUnit->hitsPerTLBLevel[hit_level]++;
1117
1118    delete translation_state->tlbEntry;
1119    assert(!translation_state->ports.size());
1120    pkt->senderState = translation_state->saved;
1121
1122    // for prefetch pkt
1123    BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1124
1125    delete translation_state;
1126
1127    // use the original sender state to know how to close this transaction
1128    DTLBPort::SenderState *sender_state =
1129        safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1130
1131    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1132    int mp_index = sender_state->portIndex;
1133    Addr vaddr = pkt->req->getVaddr();
1134    gpuDynInst->memStatusVector[line].push_back(mp_index);
1135    gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1136
1137    MemCmd requestCmd;
1138
1139    if (pkt->cmd == MemCmd::ReadResp) {
1140        requestCmd = MemCmd::ReadReq;
1141    } else if (pkt->cmd == MemCmd::WriteResp) {
1142        requestCmd = MemCmd::WriteReq;
1143    } else if (pkt->cmd == MemCmd::SwapResp) {
1144        requestCmd = MemCmd::SwapReq;
1145    } else {
1146        panic("unsupported response to request conversion %s\n",
1147              pkt->cmd.toString());
1148    }
1149
1150    if (computeUnit->prefetchDepth) {
1151        int simdId = gpuDynInst->simdId;
1152        int wfSlotId = gpuDynInst->wfSlotId;
1153        Addr last = 0;
1154
1155        switch(computeUnit->prefetchType) {
1156        case Enums::PF_CU:
1157            last = computeUnit->lastVaddrCU[mp_index];
1158            break;
1159        case Enums::PF_PHASE:
1160            last = computeUnit->lastVaddrSimd[simdId][mp_index];
1161            break;
1162        case Enums::PF_WF:
1163            last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1164        default:
1165            break;
1166        }
1167
1168        DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1169                computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1170
1171        int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1172                     roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1173                     : 0;
1174
1175        DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1176
1177        computeUnit->lastVaddrCU[mp_index] = vaddr;
1178        computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1179        computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1180
1181        stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1182            computeUnit->prefetchStride: stride;
1183
1184        DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1185                computeUnit->cu_id, simdId, wfSlotId, mp_index);
1186
1187        DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1188
1189        // Prefetch Next few pages atomically
1190        for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1191            DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1192                    vaddr+stride*pf*TheISA::PageBytes);
1193
1194            if (!stride)
1195                break;
1196
1197            Request *prefetch_req = new Request(0, vaddr + stride * pf *
1198                                                TheISA::PageBytes,
1199                                                sizeof(uint8_t), 0,
1200                                                computeUnit->masterId(),
1201                                                0, 0, 0);
1202
1203            PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1204            uint8_t foo = 0;
1205            prefetch_pkt->dataStatic(&foo);
1206
1207            // Because it's atomic operation, only need TLB translation state
1208            prefetch_pkt->senderState =
1209                new TheISA::GpuTLB::TranslationState(TLB_mode,
1210                                                     computeUnit->shader->gpuTc,
1211                                                     true);
1212
1213            // Currently prefetches are zero-latency, hence the sendFunctional
1214            sendFunctional(prefetch_pkt);
1215
1216            /* safe_cast the senderState */
1217            TheISA::GpuTLB::TranslationState *tlb_state =
1218                 safe_cast<TheISA::GpuTLB::TranslationState*>(
1219                         prefetch_pkt->senderState);
1220
1221
1222            delete tlb_state->tlbEntry;
1223            delete tlb_state;
1224            delete prefetch_pkt->req;
1225            delete prefetch_pkt;
1226        }
1227    }
1228
1229    // First we must convert the response cmd back to a request cmd so that
1230    // the request can be sent through the cu's master port
1231    PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1232    new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1233    delete pkt->senderState;
1234    delete pkt;
1235
1236    // New SenderState for the memory access
1237    new_pkt->senderState =
1238            new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1239                                                   nullptr);
1240
1241    // translation is done. Schedule the mem_req_event at the appropriate
1242    // cycle to send the timing memory request to ruby
1243    ComputeUnit::DataPort::MemReqEvent *mem_req_event =
1244        new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index],
1245                                               new_pkt);
1246
1247    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1248            computeUnit->cu_id, gpuDynInst->simdId,
1249            gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1250
1251    computeUnit->schedule(mem_req_event, curTick() +
1252                          computeUnit->req_tick_latency);
1253
1254    return true;
1255}
1256
1257const char*
1258ComputeUnit::DataPort::MemReqEvent::description() const
1259{
1260    return "ComputeUnit memory request event";
1261}
1262
1263void
1264ComputeUnit::DataPort::MemReqEvent::process()
1265{
1266    SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1267    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1268    ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit;
1269
1270    if (!(dataPort->sendTimingReq(pkt))) {
1271        dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst));
1272
1273        DPRINTF(GPUPort,
1274                "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1275                compute_unit->cu_id, gpuDynInst->simdId,
1276                gpuDynInst->wfSlotId, dataPort->index,
1277                pkt->req->getPaddr());
1278    } else {
1279        DPRINTF(GPUPort,
1280                "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1281                compute_unit->cu_id, gpuDynInst->simdId,
1282                gpuDynInst->wfSlotId, dataPort->index,
1283                pkt->req->getPaddr());
1284    }
1285}
1286
1287/*
1288 * The initial translation request could have been rejected,
1289 * if <retries> queue is not Retry sending the translation
1290 * request. sendRetry() is called from the peer port whenever
1291 * a translation completes.
1292 */
1293void
1294ComputeUnit::DTLBPort::recvReqRetry()
1295{
1296    int len = retries.size();
1297
1298    DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1299            computeUnit->cu_id, len);
1300
1301    assert(len > 0);
1302    assert(isStalled());
1303    // recvReqRetry is an indication that the resource on which this
1304    // port was stalling on is freed. So, remove the stall first
1305    unstallPort();
1306
1307    for (int i = 0; i < len; ++i) {
1308        PacketPtr pkt = retries.front();
1309        Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1310        DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1311
1312        if (!sendTimingReq(pkt)) {
1313            // Stall port
1314            stallPort();
1315            DPRINTF(GPUTLB, ": failed again\n");
1316            break;
1317        } else {
1318            DPRINTF(GPUTLB, ": successful\n");
1319            retries.pop_front();
1320        }
1321    }
1322}
1323
1324bool
1325ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1326{
1327    Addr line M5_VAR_USED = pkt->req->getPaddr();
1328    DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1329            computeUnit->cu_id, pkt->req->getVaddr(), line);
1330
1331    assert(pkt->senderState);
1332
1333    // pop off the TLB translation state
1334    TheISA::GpuTLB::TranslationState *translation_state =
1335                 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1336
1337    bool success = translation_state->tlbEntry->valid;
1338    delete translation_state->tlbEntry;
1339    assert(!translation_state->ports.size());
1340    pkt->senderState = translation_state->saved;
1341    delete translation_state;
1342
1343    // use the original sender state to know how to close this transaction
1344    ITLBPort::SenderState *sender_state =
1345        safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1346
1347    // get the wavefront associated with this translation request
1348    Wavefront *wavefront = sender_state->wavefront;
1349    delete pkt->senderState;
1350
1351    if (success) {
1352        // pkt is reused in fetch(), don't delete it here.  However, we must
1353        // reset the command to be a request so that it can be sent through
1354        // the cu's master port
1355        assert(pkt->cmd == MemCmd::ReadResp);
1356        pkt->cmd = MemCmd::ReadReq;
1357
1358        computeUnit->fetchStage.fetch(pkt, wavefront);
1359    } else {
1360        if (wavefront->dropFetch) {
1361            assert(wavefront->instructionBuffer.empty());
1362            wavefront->dropFetch = false;
1363        }
1364
1365        wavefront->pendingFetch = 0;
1366    }
1367
1368    return true;
1369}
1370
1371/*
1372 * The initial translation request could have been rejected, if
1373 * <retries> queue is not empty. Retry sending the translation
1374 * request. sendRetry() is called from the peer port whenever
1375 * a translation completes.
1376 */
1377void
1378ComputeUnit::ITLBPort::recvReqRetry()
1379{
1380
1381    int len = retries.size();
1382    DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1383
1384    assert(len > 0);
1385    assert(isStalled());
1386
1387    // recvReqRetry is an indication that the resource on which this
1388    // port was stalling on is freed. So, remove the stall first
1389    unstallPort();
1390
1391    for (int i = 0; i < len; ++i) {
1392        PacketPtr pkt = retries.front();
1393        Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1394        DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1395
1396        if (!sendTimingReq(pkt)) {
1397            stallPort(); // Stall port
1398            DPRINTF(GPUTLB, ": failed again\n");
1399            break;
1400        } else {
1401            DPRINTF(GPUTLB, ": successful\n");
1402            retries.pop_front();
1403        }
1404    }
1405}
1406
1407void
1408ComputeUnit::regStats()
1409{
1410    MemObject::regStats();
1411
1412    vALUInsts
1413        .name(name() + ".valu_insts")
1414        .desc("Number of vector ALU insts issued.")
1415        ;
1416    vALUInstsPerWF
1417        .name(name() + ".valu_insts_per_wf")
1418        .desc("The avg. number of vector ALU insts issued per-wavefront.")
1419        ;
1420    sALUInsts
1421        .name(name() + ".salu_insts")
1422        .desc("Number of scalar ALU insts issued.")
1423        ;
1424    sALUInstsPerWF
1425        .name(name() + ".salu_insts_per_wf")
1426        .desc("The avg. number of scalar ALU insts issued per-wavefront.")
1427        ;
1428    instCyclesVALU
1429        .name(name() + ".inst_cycles_valu")
1430        .desc("Number of cycles needed to execute VALU insts.")
1431        ;
1432    instCyclesSALU
1433        .name(name() + ".inst_cycles_salu")
1434        .desc("Number of cycles needed to execute SALU insts.")
1435        ;
1436    threadCyclesVALU
1437        .name(name() + ".thread_cycles_valu")
1438        .desc("Number of thread cycles used to execute vector ALU ops. "
1439              "Similar to instCyclesVALU but multiplied by the number of "
1440              "active threads.")
1441        ;
1442    vALUUtilization
1443        .name(name() + ".valu_utilization")
1444        .desc("Percentage of active vector ALU threads in a wave.")
1445        ;
1446    ldsNoFlatInsts
1447        .name(name() + ".lds_no_flat_insts")
1448        .desc("Number of LDS insts issued, not including FLAT "
1449              "accesses that resolve to LDS.")
1450        ;
1451    ldsNoFlatInstsPerWF
1452        .name(name() + ".lds_no_flat_insts_per_wf")
1453        .desc("The avg. number of LDS insts (not including FLAT "
1454              "accesses that resolve to LDS) per-wavefront.")
1455        ;
1456    flatVMemInsts
1457        .name(name() + ".flat_vmem_insts")
1458        .desc("The number of FLAT insts that resolve to vmem issued.")
1459        ;
1460    flatVMemInstsPerWF
1461        .name(name() + ".flat_vmem_insts_per_wf")
1462        .desc("The average number of FLAT insts that resolve to vmem "
1463              "issued per-wavefront.")
1464        ;
1465    flatLDSInsts
1466        .name(name() + ".flat_lds_insts")
1467        .desc("The number of FLAT insts that resolve to LDS issued.")
1468        ;
1469    flatLDSInstsPerWF
1470        .name(name() + ".flat_lds_insts_per_wf")
1471        .desc("The average number of FLAT insts that resolve to LDS "
1472              "issued per-wavefront.")
1473        ;
1474    vectorMemWrites
1475        .name(name() + ".vector_mem_writes")
1476        .desc("Number of vector mem write insts (excluding FLAT insts).")
1477        ;
1478    vectorMemWritesPerWF
1479        .name(name() + ".vector_mem_writes_per_wf")
1480        .desc("The average number of vector mem write insts "
1481              "(excluding FLAT insts) per-wavefront.")
1482        ;
1483    vectorMemReads
1484        .name(name() + ".vector_mem_reads")
1485        .desc("Number of vector mem read insts (excluding FLAT insts).")
1486        ;
1487    vectorMemReadsPerWF
1488        .name(name() + ".vector_mem_reads_per_wf")
1489        .desc("The avg. number of vector mem read insts (excluding "
1490              "FLAT insts) per-wavefront.")
1491        ;
1492    scalarMemWrites
1493        .name(name() + ".scalar_mem_writes")
1494        .desc("Number of scalar mem write insts.")
1495        ;
1496    scalarMemWritesPerWF
1497        .name(name() + ".scalar_mem_writes_per_wf")
1498        .desc("The average number of scalar mem write insts per-wavefront.")
1499        ;
1500    scalarMemReads
1501        .name(name() + ".scalar_mem_reads")
1502        .desc("Number of scalar mem read insts.")
1503        ;
1504    scalarMemReadsPerWF
1505        .name(name() + ".scalar_mem_reads_per_wf")
1506        .desc("The average number of scalar mem read insts per-wavefront.")
1507        ;
1508
1509    vALUInstsPerWF = vALUInsts / completedWfs;
1510    sALUInstsPerWF = sALUInsts / completedWfs;
1511    vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100;
1512    ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs;
1513    flatVMemInstsPerWF = flatVMemInsts / completedWfs;
1514    flatLDSInstsPerWF = flatLDSInsts / completedWfs;
1515    vectorMemWritesPerWF = vectorMemWrites / completedWfs;
1516    vectorMemReadsPerWF = vectorMemReads / completedWfs;
1517    scalarMemWritesPerWF = scalarMemWrites / completedWfs;
1518    scalarMemReadsPerWF = scalarMemReads / completedWfs;
1519
1520    tlbCycles
1521        .name(name() + ".tlb_cycles")
1522        .desc("total number of cycles for all uncoalesced requests")
1523        ;
1524
1525    tlbRequests
1526        .name(name() + ".tlb_requests")
1527        .desc("number of uncoalesced requests")
1528        ;
1529
1530    tlbLatency
1531        .name(name() + ".avg_translation_latency")
1532        .desc("Avg. translation latency for data translations")
1533        ;
1534
1535    tlbLatency = tlbCycles / tlbRequests;
1536
1537    hitsPerTLBLevel
1538       .init(4)
1539       .name(name() + ".TLB_hits_distribution")
1540       .desc("TLB hits distribution (0 for page table, x for Lx-TLB")
1541       ;
1542
1543    // fixed number of TLB levels
1544    for (int i = 0; i < 4; ++i) {
1545        if (!i)
1546            hitsPerTLBLevel.subname(i,"page_table");
1547        else
1548            hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i));
1549    }
1550
1551    execRateDist
1552        .init(0, 10, 2)
1553        .name(name() + ".inst_exec_rate")
1554        .desc("Instruction Execution Rate: Number of executed vector "
1555              "instructions per cycle")
1556        ;
1557
1558    ldsBankConflictDist
1559       .init(0, wfSize(), 2)
1560       .name(name() + ".lds_bank_conflicts")
1561       .desc("Number of bank conflicts per LDS memory packet")
1562       ;
1563
1564    ldsBankAccesses
1565        .name(name() + ".lds_bank_access_cnt")
1566        .desc("Total number of LDS bank accesses")
1567        ;
1568
1569    pageDivergenceDist
1570        // A wavefront can touch up to N pages per memory instruction where
1571        // N is equal to the wavefront size
1572        // The number of pages per bin can be configured (here it's 4).
1573       .init(1, wfSize(), 4)
1574       .name(name() + ".page_divergence_dist")
1575       .desc("pages touched per wf (over all mem. instr.)")
1576       ;
1577
1578    controlFlowDivergenceDist
1579        .init(1, wfSize(), 4)
1580        .name(name() + ".warp_execution_dist")
1581        .desc("number of lanes active per instruction (oval all instructions)")
1582        ;
1583
1584    activeLanesPerGMemInstrDist
1585        .init(1, wfSize(), 4)
1586        .name(name() + ".gmem_lanes_execution_dist")
1587        .desc("number of active lanes per global memory instruction")
1588        ;
1589
1590    activeLanesPerLMemInstrDist
1591        .init(1, wfSize(), 4)
1592        .name(name() + ".lmem_lanes_execution_dist")
1593        .desc("number of active lanes per local memory instruction")
1594        ;
1595
1596    numInstrExecuted
1597        .name(name() + ".num_instr_executed")
1598        .desc("number of instructions executed")
1599        ;
1600
1601    numVecOpsExecuted
1602        .name(name() + ".num_vec_ops_executed")
1603        .desc("number of vec ops executed (e.g. WF size/inst)")
1604        ;
1605
1606    totalCycles
1607        .name(name() + ".num_total_cycles")
1608        .desc("number of cycles the CU ran for")
1609        ;
1610
1611    ipc
1612        .name(name() + ".ipc")
1613        .desc("Instructions per cycle (this CU only)")
1614        ;
1615
1616    vpc
1617        .name(name() + ".vpc")
1618        .desc("Vector Operations per cycle (this CU only)")
1619        ;
1620
1621    numALUInstsExecuted
1622        .name(name() + ".num_alu_insts_executed")
1623        .desc("Number of dynamic non-GM memory insts executed")
1624        ;
1625
1626    wgBlockedDueLdsAllocation
1627        .name(name() + ".wg_blocked_due_lds_alloc")
1628        .desc("Workgroup blocked due to LDS capacity")
1629        ;
1630
1631    ipc = numInstrExecuted / totalCycles;
1632    vpc = numVecOpsExecuted / totalCycles;
1633
1634    numTimesWgBlockedDueVgprAlloc
1635        .name(name() + ".times_wg_blocked_due_vgpr_alloc")
1636        .desc("Number of times WGs are blocked due to VGPR allocation per SIMD")
1637        ;
1638
1639    dynamicGMemInstrCnt
1640        .name(name() + ".global_mem_instr_cnt")
1641        .desc("dynamic global memory instructions count")
1642        ;
1643
1644    dynamicLMemInstrCnt
1645        .name(name() + ".local_mem_instr_cnt")
1646        .desc("dynamic local memory intruction count")
1647        ;
1648
1649    numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt -
1650        dynamicLMemInstrCnt;
1651
1652    completedWfs
1653        .name(name() + ".num_completed_wfs")
1654        .desc("number of completed wavefronts")
1655        ;
1656
1657    numCASOps
1658        .name(name() + ".num_CAS_ops")
1659        .desc("number of compare and swap operations")
1660        ;
1661
1662    numFailedCASOps
1663        .name(name() + ".num_failed_CAS_ops")
1664        .desc("number of compare and swap operations that failed")
1665        ;
1666
1667    // register stats of pipeline stages
1668    fetchStage.regStats();
1669    scoreboardCheckStage.regStats();
1670    scheduleStage.regStats();
1671    execStage.regStats();
1672
1673    // register stats of memory pipeline
1674    globalMemoryPipe.regStats();
1675    localMemoryPipe.regStats();
1676}
1677
1678void
1679ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst)
1680{
1681    if (gpuDynInst->isScalar()) {
1682        if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) {
1683            sALUInsts++;
1684            instCyclesSALU++;
1685        } else if (gpuDynInst->isLoad()) {
1686            scalarMemReads++;
1687        } else if (gpuDynInst->isStore()) {
1688            scalarMemWrites++;
1689        }
1690    } else {
1691        if (gpuDynInst->isALU()) {
1692            vALUInsts++;
1693            instCyclesVALU++;
1694            threadCyclesVALU += gpuDynInst->wavefront()->execMask().count();
1695        } else if (gpuDynInst->isFlat()) {
1696            if (gpuDynInst->isLocalMem()) {
1697                flatLDSInsts++;
1698            } else {
1699                flatVMemInsts++;
1700            }
1701        } else if (gpuDynInst->isLocalMem()) {
1702            ldsNoFlatInsts++;
1703        } else if (gpuDynInst->isLoad()) {
1704            vectorMemReads++;
1705        } else if (gpuDynInst->isStore()) {
1706            vectorMemWrites++;
1707        }
1708    }
1709}
1710
1711void
1712ComputeUnit::updatePageDivergenceDist(Addr addr)
1713{
1714    Addr virt_page_addr = roundDown(addr, TheISA::PageBytes);
1715
1716    if (!pagesTouched.count(virt_page_addr))
1717        pagesTouched[virt_page_addr] = 1;
1718    else
1719        pagesTouched[virt_page_addr]++;
1720}
1721
1722void
1723ComputeUnit::CUExitCallback::process()
1724{
1725    if (computeUnit->countPages) {
1726        std::ostream *page_stat_file =
1727            simout.create(computeUnit->name().c_str())->stream();
1728
1729        *page_stat_file << "page, wavefront accesses, workitem accesses" <<
1730            std::endl;
1731
1732        for (auto iter : computeUnit->pageAccesses) {
1733            *page_stat_file << std::hex << iter.first << ",";
1734            *page_stat_file << std::dec << iter.second.first << ",";
1735            *page_stat_file << std::dec << iter.second.second << std::endl;
1736        }
1737    }
1738 }
1739
1740bool
1741ComputeUnit::isDone() const
1742{
1743    for (int i = 0; i < numSIMDs; ++i) {
1744        if (!isSimdDone(i)) {
1745            return false;
1746        }
1747    }
1748
1749    bool glbMemBusRdy = true;
1750    for (int j = 0; j < numGlbMemUnits; ++j) {
1751        glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy();
1752    }
1753    bool locMemBusRdy = true;
1754    for (int j = 0; j < numLocMemUnits; ++j) {
1755        locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy();
1756    }
1757
1758    if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() ||
1759        !globalMemoryPipe.isGMStRespFIFOWrRdy() ||
1760        !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy()
1761        || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() ||
1762        !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) {
1763        return false;
1764    }
1765
1766    return true;
1767}
1768
1769int32_t
1770ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const
1771{
1772    return lds.getRefCounter(dispatchId, wgId);
1773}
1774
1775bool
1776ComputeUnit::isSimdDone(uint32_t simdId) const
1777{
1778    assert(simdId < numSIMDs);
1779
1780    for (int i=0; i < numGlbMemUnits; ++i) {
1781        if (!vrfToGlobalMemPipeBus[i].rdy())
1782            return false;
1783    }
1784    for (int i=0; i < numLocMemUnits; ++i) {
1785        if (!vrfToLocalMemPipeBus[i].rdy())
1786            return false;
1787    }
1788    if (!aluPipe[simdId].rdy()) {
1789        return false;
1790    }
1791
1792    for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){
1793        if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) {
1794            return false;
1795        }
1796    }
1797
1798    return true;
1799}
1800
1801/**
1802 * send a general request to the LDS
1803 * make sure to look at the return value here as your request might be
1804 * NACK'd and returning false means that you have to have some backup plan
1805 */
1806bool
1807ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst)
1808{
1809    // this is just a request to carry the GPUDynInstPtr
1810    // back and forth
1811    Request *newRequest = new Request();
1812    newRequest->setPaddr(0x0);
1813
1814    // ReadReq is not evaluted by the LDS but the Packet ctor requires this
1815    PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq);
1816
1817    // This is the SenderState needed upon return
1818    newPacket->senderState = new LDSPort::SenderState(gpuDynInst);
1819
1820    return ldsPort->sendTimingReq(newPacket);
1821}
1822
1823/**
1824 * get the result of packets sent to the LDS when they return
1825 */
1826bool
1827ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet)
1828{
1829    const ComputeUnit::LDSPort::SenderState *senderState =
1830        dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState);
1831
1832    fatal_if(!senderState, "did not get the right sort of sender state");
1833
1834    GPUDynInstPtr gpuDynInst = senderState->getMemInst();
1835
1836    delete packet->senderState;
1837    delete packet->req;
1838    delete packet;
1839
1840    computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst);
1841    return true;
1842}
1843
1844/**
1845 * attempt to send this packet, either the port is already stalled, the request
1846 * is nack'd and must stall or the request goes through
1847 * when a request cannot be sent, add it to the retries queue
1848 */
1849bool
1850ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt)
1851{
1852    ComputeUnit::LDSPort::SenderState *sender_state =
1853            dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState);
1854    fatal_if(!sender_state, "packet without a valid sender state");
1855
1856    GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst();
1857
1858    if (isStalled()) {
1859        fatal_if(retries.empty(), "must have retries waiting to be stalled");
1860
1861        retries.push(pkt);
1862
1863        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n",
1864                        computeUnit->cu_id, gpuDynInst->simdId,
1865                        gpuDynInst->wfSlotId);
1866        return false;
1867    } else if (!MasterPort::sendTimingReq(pkt)) {
1868        // need to stall the LDS port until a recvReqRetry() is received
1869        // this indicates that there is more space
1870        stallPort();
1871        retries.push(pkt);
1872
1873        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n",
1874                computeUnit->cu_id, gpuDynInst->simdId,
1875                gpuDynInst->wfSlotId, pkt->req->getPaddr());
1876        return false;
1877    } else {
1878        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n",
1879                computeUnit->cu_id, gpuDynInst->simdId,
1880                gpuDynInst->wfSlotId, pkt->req->getPaddr());
1881        return true;
1882    }
1883}
1884
1885/**
1886 * the bus is telling the port that there is now space so retrying stalled
1887 * requests should work now
1888 * this allows the port to have a request be nack'd and then have the receiver
1889 * say when there is space, rather than simply retrying the send every cycle
1890 */
1891void
1892ComputeUnit::LDSPort::recvReqRetry()
1893{
1894    auto queueSize = retries.size();
1895
1896    DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n",
1897            computeUnit->cu_id, queueSize);
1898
1899    fatal_if(queueSize < 1,
1900             "why was there a recvReqRetry() with no pending reqs?");
1901    fatal_if(!isStalled(),
1902             "recvReqRetry() happened when the port was not stalled");
1903
1904    unstallPort();
1905
1906    while (!retries.empty()) {
1907        PacketPtr packet = retries.front();
1908
1909        DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id);
1910
1911        if (!MasterPort::sendTimingReq(packet)) {
1912            // Stall port
1913            stallPort();
1914            DPRINTF(GPUPort, ": LDS send failed again\n");
1915            break;
1916        } else {
1917            DPRINTF(GPUTLB, ": LDS send successful\n");
1918            retries.pop();
1919        }
1920    }
1921}
1922