compute_unit.cc revision 12697:cd71b966be1e 
1/*
2 * Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived from this
19 * software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Authors: John Kalamatianos,
34 *          Anthony Gutierrez
35 */
36
37#include "gpu-compute/compute_unit.hh"
38
39#include <limits>
40
41#include "base/output.hh"
42#include "debug/GPUDisp.hh"
43#include "debug/GPUExec.hh"
44#include "debug/GPUFetch.hh"
45#include "debug/GPUMem.hh"
46#include "debug/GPUPort.hh"
47#include "debug/GPUPrefetch.hh"
48#include "debug/GPUSync.hh"
49#include "debug/GPUTLB.hh"
50#include "gpu-compute/dispatcher.hh"
51#include "gpu-compute/gpu_dyn_inst.hh"
52#include "gpu-compute/gpu_static_inst.hh"
53#include "gpu-compute/ndrange.hh"
54#include "gpu-compute/shader.hh"
55#include "gpu-compute/simple_pool_manager.hh"
56#include "gpu-compute/vector_register_file.hh"
57#include "gpu-compute/wavefront.hh"
58#include "mem/page_table.hh"
59#include "sim/process.hh"
60
61ComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p),
62    scoreboardCheckStage(p), scheduleStage(p), execStage(p),
63    globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0),
64    cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs),
65    spBypassPipeLength(p->spbypass_pipe_length),
66    dpBypassPipeLength(p->dpbypass_pipe_length),
67    issuePeriod(p->issue_period),
68    numGlbMemUnits(p->num_global_mem_pipes),
69    numLocMemUnits(p->num_shared_mem_pipes),
70    perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth),
71    prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type),
72    xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault),
73    functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier),
74    countPages(p->countPages), barrier_id(0),
75    vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width),
76    coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width),
77    req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()),
78    resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()),
79    _masterId(p->system->getMasterId(this, "ComputeUnit")),
80    lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()),
81    globalSeqNum(0), wavefrontSize(p->wfSize),
82    kernelLaunchInst(new KernelLaunchStaticInst())
83{
84    /**
85     * This check is necessary because std::bitset only provides conversion
86     * to unsigned long or unsigned long long via to_ulong() or to_ullong().
87     * there are * a few places in the code where to_ullong() is used, however
88     * if VSZ is larger than a value the host can support then bitset will
89     * throw a runtime exception. we should remove all use of to_long() or
90     * to_ullong() so we can have VSZ greater than 64b, however until that is
91     * done this assert is required.
92     */
93    fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits ||
94             p->wfSize <= 0,
95             "WF size is larger than the host can support");
96    fatal_if(!isPowerOf2(wavefrontSize),
97             "Wavefront size should be a power of 2");
98    // calculate how many cycles a vector load or store will need to transfer
99    // its data over the corresponding buses
100    numCyclesPerStoreTransfer =
101        (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) /
102                (double)vrfToCoalescerBusWidth);
103
104    numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t))
105                               / coalescerToVrfBusWidth;
106
107    lastVaddrWF.resize(numSIMDs);
108    wfList.resize(numSIMDs);
109
110    for (int j = 0; j < numSIMDs; ++j) {
111        lastVaddrWF[j].resize(p->n_wf);
112
113        for (int i = 0; i < p->n_wf; ++i) {
114            lastVaddrWF[j][i].resize(wfSize());
115
116            wfList[j].push_back(p->wavefronts[j * p->n_wf + i]);
117            wfList[j][i]->setParent(this);
118
119            for (int k = 0; k < wfSize(); ++k) {
120                lastVaddrWF[j][i][k] = 0;
121            }
122        }
123    }
124
125    lastVaddrSimd.resize(numSIMDs);
126
127    for (int i = 0; i < numSIMDs; ++i) {
128        lastVaddrSimd[i].resize(wfSize(), 0);
129    }
130
131    lastVaddrCU.resize(wfSize());
132
133    lds.setParent(this);
134
135    if (p->execPolicy == "OLDEST-FIRST") {
136        exec_policy = EXEC_POLICY::OLDEST;
137    } else if (p->execPolicy == "ROUND-ROBIN") {
138        exec_policy = EXEC_POLICY::RR;
139    } else {
140        fatal("Invalid WF execution policy (CU)\n");
141    }
142
143    memPort.resize(wfSize());
144
145    // resize the tlbPort vectorArray
146    int tlbPort_width = perLaneTLB ? wfSize() : 1;
147    tlbPort.resize(tlbPort_width);
148
149    cuExitCallback = new CUExitCallback(this);
150    registerExitCallback(cuExitCallback);
151
152    xactCasLoadMap.clear();
153    lastExecCycle.resize(numSIMDs, 0);
154
155    for (int i = 0; i < vrf.size(); ++i) {
156        vrf[i]->setParent(this);
157    }
158
159    numVecRegsPerSimd = vrf[0]->numRegs();
160}
161
162ComputeUnit::~ComputeUnit()
163{
164    // Delete wavefront slots
165    for (int j = 0; j < numSIMDs; ++j) {
166        for (int i = 0; i < shader->n_wf; ++i) {
167            delete wfList[j][i];
168        }
169        lastVaddrSimd[j].clear();
170    }
171    lastVaddrCU.clear();
172    readyList.clear();
173    waveStatusList.clear();
174    dispatchList.clear();
175    vectorAluInstAvail.clear();
176    delete cuExitCallback;
177    delete ldsPort;
178}
179
180void
181ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr)
182{
183    w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount);
184
185    w->workGroupSz[0] = ndr->q.wgSize[0];
186    w->workGroupSz[1] = ndr->q.wgSize[1];
187    w->workGroupSz[2] = ndr->q.wgSize[2];
188    w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2];
189    w->gridSz[0] = ndr->q.gdSize[0];
190    w->gridSz[1] = ndr->q.gdSize[1];
191    w->gridSz[2] = ndr->q.gdSize[2];
192    w->kernelArgs = ndr->q.args;
193    w->privSizePerItem = ndr->q.privMemPerItem;
194    w->spillSizePerItem = ndr->q.spillMemPerItem;
195    w->roBase = ndr->q.roMemStart;
196    w->roSize = ndr->q.roMemTotal;
197    w->computeActualWgSz(ndr);
198}
199
200void
201ComputeUnit::updateEvents() {
202
203    if (!timestampVec.empty()) {
204        uint32_t vecSize = timestampVec.size();
205        uint32_t i = 0;
206        while (i < vecSize) {
207            if (timestampVec[i] <= shader->tick_cnt) {
208                std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i];
209                vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t),
210                                            statusVec[i]);
211                timestampVec.erase(timestampVec.begin() + i);
212                regIdxVec.erase(regIdxVec.begin() + i);
213                statusVec.erase(statusVec.begin() + i);
214                --vecSize;
215                --i;
216            }
217            ++i;
218        }
219    }
220
221    for (int i = 0; i< numSIMDs; ++i) {
222        vrf[i]->updateEvents();
223    }
224}
225
226
227void
228ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk,
229                            NDRange *ndr)
230{
231    static int _n_wave = 0;
232
233    VectorMask init_mask;
234    init_mask.reset();
235
236    for (int k = 0; k < wfSize(); ++k) {
237        if (k + waveId * wfSize() < w->actualWgSzTotal)
238            init_mask[k] = 1;
239    }
240
241    w->kernId = ndr->dispatchId;
242    w->wfId = waveId;
243    w->initMask = init_mask.to_ullong();
244
245    for (int k = 0; k < wfSize(); ++k) {
246        w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0];
247        w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) %
248                             w->actualWgSz[1];
249        w->workItemId[2][k] = (k + waveId * wfSize()) /
250                              (w->actualWgSz[0] * w->actualWgSz[1]);
251
252        w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] *
253            w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] +
254            w->workItemId[0][k];
255    }
256
257    w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize());
258
259    w->barCnt.resize(wfSize(), 0);
260
261    w->maxBarCnt = 0;
262    w->oldBarrierCnt = 0;
263    w->barrierCnt = 0;
264
265    w->privBase = ndr->q.privMemStart;
266    ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize();
267
268    w->spillBase = ndr->q.spillMemStart;
269    ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize();
270
271    w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong());
272
273    // WG state
274    w->wgId = ndr->globalWgId;
275    w->dispatchId = ndr->dispatchId;
276    w->workGroupId[0] = w->wgId % ndr->numWg[0];
277    w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1];
278    w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]);
279
280    w->barrierId = barrier_id;
281    w->stalledAtBarrier = false;
282
283    // set the wavefront context to have a pointer to this section of the LDS
284    w->ldsChunk = ldsChunk;
285
286    int32_t refCount M5_VAR_USED =
287                    lds.increaseRefCounter(w->dispatchId, w->wgId);
288    DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n",
289                    cu_id, w->wgId, refCount);
290
291    w->instructionBuffer.clear();
292
293    if (w->pendingFetch)
294        w->dropFetch = true;
295
296    // is this the last wavefront in the workgroup
297    // if set the spillWidth to be the remaining work-items
298    // so that the vector access is correct
299    if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) {
300        w->spillWidth = w->actualWgSzTotal - (waveId * wfSize());
301    } else {
302        w->spillWidth = wfSize();
303    }
304
305    DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: "
306            "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId);
307
308    w->start(++_n_wave, ndr->q.code_ptr);
309}
310
311void
312ComputeUnit::StartWorkgroup(NDRange *ndr)
313{
314    // reserve the LDS capacity allocated to the work group
315    // disambiguated by the dispatch ID and workgroup ID, which should be
316    // globally unique
317    LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId,
318                                          ndr->q.ldsSize);
319
320    // Send L1 cache acquire
321    // isKernel + isAcquire = Kernel Begin
322    if (shader->impl_kern_boundary_sync) {
323        GPUDynInstPtr gpuDynInst =
324            std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst,
325                                         getAndIncSeqNum());
326
327        gpuDynInst->useContinuation = false;
328        injectGlobalMemFence(gpuDynInst, true);
329    }
330
331    // calculate the number of 32-bit vector registers required by wavefront
332    int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
333    int wave_id = 0;
334
335    // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time
336    for (int m = 0; m < shader->n_wf * numSIMDs; ++m) {
337        Wavefront *w = wfList[m % numSIMDs][m / numSIMDs];
338        // Check if this wavefront slot is available:
339        // It must be stopped and not waiting
340        // for a release to complete S_RETURNING
341        if (w->status == Wavefront::S_STOPPED) {
342            fillKernelState(w, ndr);
343            // if we have scheduled all work items then stop
344            // scheduling wavefronts
345            if (wave_id * wfSize() >= w->actualWgSzTotal)
346                break;
347
348            // reserve vector registers for the scheduled wavefront
349            assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd);
350            uint32_t normSize = 0;
351
352            w->startVgprIndex = vrf[m % numSIMDs]->manager->
353                                    allocateRegion(vregDemand, &normSize);
354
355            w->reservedVectorRegs = normSize;
356            vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs;
357
358            startWavefront(w, wave_id, ldsChunk, ndr);
359            ++wave_id;
360        }
361    }
362    ++barrier_id;
363}
364
365int
366ComputeUnit::ReadyWorkgroup(NDRange *ndr)
367{
368    // Get true size of workgroup (after clamping to grid size)
369    int trueWgSize[3];
370    int trueWgSizeTotal = 1;
371
372    for (int d = 0; d < 3; ++d) {
373        trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] -
374                                 ndr->wgId[d] * ndr->q.wgSize[d]);
375
376        trueWgSizeTotal *= trueWgSize[d];
377        DPRINTF(GPUDisp, "trueWgSize[%d] =  %d\n", d, trueWgSize[d]);
378    }
379
380    DPRINTF(GPUDisp, "trueWgSizeTotal =  %d\n", trueWgSizeTotal);
381
382    // calculate the number of 32-bit vector registers required by each
383    // work item of the work group
384    int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount);
385    bool vregAvail = true;
386    int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize();
387    int freeWfSlots = 0;
388    // check if the total number of VGPRs required by all WFs of the WG
389    // fit in the VRFs of all SIMD units
390    assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd));
391    int numMappedWfs = 0;
392    std::vector<int> numWfsPerSimd;
393    numWfsPerSimd.resize(numSIMDs, 0);
394    // find how many free WF slots we have across all SIMDs
395    for (int j = 0; j < shader->n_wf; ++j) {
396        for (int i = 0; i < numSIMDs; ++i) {
397            if (wfList[i][j]->status == Wavefront::S_STOPPED) {
398                // count the number of free WF slots
399                ++freeWfSlots;
400                if (numMappedWfs < numWfs) {
401                    // count the WFs to be assigned per SIMD
402                    numWfsPerSimd[i]++;
403                }
404                numMappedWfs++;
405            }
406        }
407    }
408
409    // if there are enough free WF slots then find if there are enough
410    // free VGPRs per SIMD based on the WF->SIMD mapping
411    if (freeWfSlots >= numWfs) {
412        for (int j = 0; j < numSIMDs; ++j) {
413            // find if there are enough free VGPR regions in the SIMD's VRF
414            // to accommodate the WFs of the new WG that would be mapped to
415            // this SIMD unit
416            vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j],
417                                                     vregDemandPerWI);
418
419            // stop searching if there is at least one SIMD
420            // whose VRF does not have enough free VGPR pools.
421            // This is because a WG is scheduled only if ALL
422            // of its WFs can be scheduled
423            if (!vregAvail)
424                break;
425        }
426    }
427
428    DPRINTF(GPUDisp, "Free WF slots =  %d, VGPR Availability = %d\n",
429            freeWfSlots, vregAvail);
430
431    if (!vregAvail) {
432        ++numTimesWgBlockedDueVgprAlloc;
433    }
434
435    // Return true if enough WF slots to submit workgroup and if there are
436    // enough VGPRs to schedule all WFs to their SIMD units
437    if (!lds.canReserve(ndr->q.ldsSize)) {
438        wgBlockedDueLdsAllocation++;
439    }
440
441    // Return true if (a) there are enough free WF slots to submit
442    // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their
443    // SIMD units and (c) if there is enough space in LDS
444    return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize);
445}
446
447int
448ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots)
449{
450    DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id);
451    int ccnt = 0;
452
453    for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) {
454        for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) {
455            Wavefront *w = wfList[i_simd][i_wf];
456
457            if (w->status == Wavefront::S_RUNNING) {
458                DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf);
459
460                DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n",
461                        w->barrierId, _barrier_id);
462
463                DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n",
464                        w->barrierCnt, bcnt);
465            }
466
467            if (w->status == Wavefront::S_RUNNING &&
468                w->barrierId == _barrier_id && w->barrierCnt == bcnt &&
469                !w->outstandingReqs) {
470                ++ccnt;
471
472                DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to "
473                        "%d\n", i_simd, i_wf, ccnt);
474            }
475        }
476    }
477
478    DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n",
479            cu_id, ccnt, bslots);
480
481    return ccnt == bslots;
482}
483
484//  Check if the current wavefront is blocked on additional resources.
485bool
486ComputeUnit::cedeSIMD(int simdId, int wfSlotId)
487{
488    bool cede = false;
489
490    // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld
491    // magic instructions will impact the scheduling of wavefronts
492    if (xact_cas_mode) {
493        /*
494         * When a wavefront calls xact_cas_ld, it adds itself to a per address
495         * queue. All per address queues are managed by the xactCasLoadMap.
496         *
497         * A wavefront is not blocked if: it is not in ANY per address queue or
498         * if it is at the head of a per address queue.
499         */
500        for (auto itMap : xactCasLoadMap) {
501            std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue;
502
503            if (!curWaveIDQueue.empty()) {
504                for (auto it : curWaveIDQueue) {
505                    waveIdentifier cur_wave = it;
506
507                    if (cur_wave.simdId == simdId &&
508                        cur_wave.wfSlotId == wfSlotId) {
509                        // 2 possibilities
510                        // 1: this WF has a green light
511                        // 2: another WF has a green light
512                        waveIdentifier owner_wave = curWaveIDQueue.front();
513
514                        if (owner_wave.simdId != cur_wave.simdId ||
515                            owner_wave.wfSlotId != cur_wave.wfSlotId) {
516                            // possibility 2
517                            cede = true;
518                            break;
519                        } else {
520                            // possibility 1
521                            break;
522                        }
523                    }
524                }
525            }
526        }
527    }
528
529    return cede;
530}
531
532// Execute one clock worth of work on the ComputeUnit.
533void
534ComputeUnit::exec()
535{
536    updateEvents();
537    // Execute pipeline stages in reverse order to simulate
538    // the pipeline latency
539    globalMemoryPipe.exec();
540    localMemoryPipe.exec();
541    execStage.exec();
542    scheduleStage.exec();
543    scoreboardCheckStage.exec();
544    fetchStage.exec();
545
546    totalCycles++;
547}
548
549void
550ComputeUnit::init()
551{
552    // Initialize CU Bus models
553    glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
554    locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1));
555    nextGlbMemBus = 0;
556    nextLocMemBus = 0;
557    fatal_if(numGlbMemUnits > 1,
558             "No support for multiple Global Memory Pipelines exists!!!");
559    vrfToGlobalMemPipeBus.resize(numGlbMemUnits);
560    for (int j = 0; j < numGlbMemUnits; ++j) {
561        vrfToGlobalMemPipeBus[j] = WaitClass();
562        vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
563    }
564
565    fatal_if(numLocMemUnits > 1,
566             "No support for multiple Local Memory Pipelines exists!!!");
567    vrfToLocalMemPipeBus.resize(numLocMemUnits);
568    for (int j = 0; j < numLocMemUnits; ++j) {
569        vrfToLocalMemPipeBus[j] = WaitClass();
570        vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1));
571    }
572    vectorRegsReserved.resize(numSIMDs, 0);
573    aluPipe.resize(numSIMDs);
574    wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits);
575
576    for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) {
577        wfWait[i] = WaitClass();
578        wfWait[i].init(&shader->tick_cnt, shader->ticks(1));
579    }
580
581    for (int i = 0; i < numSIMDs; ++i) {
582        aluPipe[i] = WaitClass();
583        aluPipe[i].init(&shader->tick_cnt, shader->ticks(1));
584    }
585
586    // Setup space for call args
587    for (int j = 0; j < numSIMDs; ++j) {
588        for (int i = 0; i < shader->n_wf; ++i) {
589            wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize);
590        }
591    }
592
593    // Initializing pipeline resources
594    readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits);
595    waveStatusList.resize(numSIMDs);
596
597    for (int j = 0; j < numSIMDs; ++j) {
598        for (int i = 0; i < shader->n_wf; ++i) {
599            waveStatusList[j].push_back(
600                std::make_pair(wfList[j][i], BLOCKED));
601        }
602    }
603
604    for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) {
605        dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY));
606    }
607
608    fetchStage.init(this);
609    scoreboardCheckStage.init(this);
610    scheduleStage.init(this);
611    execStage.init(this);
612    globalMemoryPipe.init(this);
613    localMemoryPipe.init(this);
614    // initialize state for statistics calculation
615    vectorAluInstAvail.resize(numSIMDs, false);
616    shrMemInstAvail = 0;
617    glbMemInstAvail = 0;
618}
619
620bool
621ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt)
622{
623    // Ruby has completed the memory op. Schedule the mem_resp_event at the
624    // appropriate cycle to process the timing memory response
625    // This delay represents the pipeline delay
626    SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
627    int index = sender_state->port_index;
628    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
629
630    // Is the packet returned a Kernel End or Barrier
631    if (pkt->req->isKernel() && pkt->req->isRelease()) {
632        Wavefront *w =
633            computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId];
634
635        // Check if we are waiting on Kernel End Release
636        if (w->status == Wavefront::S_RETURNING) {
637            DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n",
638                    computeUnit->cu_id, w->simdId, w->wfSlotId,
639                    w->wfDynId, w->kernId);
640
641            computeUnit->shader->dispatcher->notifyWgCompl(w);
642            w->status = Wavefront::S_STOPPED;
643        } else {
644            w->outstandingReqs--;
645        }
646
647        DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n",
648                computeUnit->cu_id, gpuDynInst->simdId,
649                gpuDynInst->wfSlotId, w->barrierCnt);
650
651        if (gpuDynInst->useContinuation) {
652            assert(!gpuDynInst->isNoScope());
653            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
654                                           gpuDynInst);
655        }
656
657        delete pkt->senderState;
658        delete pkt->req;
659        delete pkt;
660        return true;
661    } else if (pkt->req->isKernel() && pkt->req->isAcquire()) {
662        if (gpuDynInst->useContinuation) {
663            assert(!gpuDynInst->isNoScope());
664            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
665                                           gpuDynInst);
666        }
667
668        delete pkt->senderState;
669        delete pkt->req;
670        delete pkt;
671        return true;
672    }
673
674    EventFunctionWrapper *mem_resp_event =
675        computeUnit->memPort[index]->createMemRespEvent(pkt);
676
677    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n",
678            computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
679            index, pkt->req->getPaddr());
680
681    computeUnit->schedule(mem_resp_event,
682                          curTick() + computeUnit->resp_tick_latency);
683    return true;
684}
685
686void
687ComputeUnit::DataPort::recvReqRetry()
688{
689    int len = retries.size();
690
691    assert(len > 0);
692
693    for (int i = 0; i < len; ++i) {
694        PacketPtr pkt = retries.front().first;
695        GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second;
696        DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n",
697                computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
698                pkt->req->getPaddr());
699
700        /** Currently Ruby can return false due to conflicts for the particular
701         *  cache block or address.  Thus other requests should be allowed to
702         *  pass and the data port should expect multiple retries. */
703        if (!sendTimingReq(pkt)) {
704            DPRINTF(GPUMem, "failed again!\n");
705            break;
706        } else {
707            DPRINTF(GPUMem, "successful!\n");
708            retries.pop_front();
709        }
710    }
711}
712
713bool
714ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt)
715{
716    computeUnit->fetchStage.processFetchReturn(pkt);
717
718    return true;
719}
720
721void
722ComputeUnit::SQCPort::recvReqRetry()
723{
724    int len = retries.size();
725
726    assert(len > 0);
727
728    for (int i = 0; i < len; ++i) {
729        PacketPtr pkt = retries.front().first;
730        Wavefront *wavefront M5_VAR_USED = retries.front().second;
731        DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n",
732                computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId,
733                pkt->req->getPaddr());
734        if (!sendTimingReq(pkt)) {
735            DPRINTF(GPUFetch, "failed again!\n");
736            break;
737        } else {
738            DPRINTF(GPUFetch, "successful!\n");
739            retries.pop_front();
740        }
741    }
742}
743
744void
745ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
746{
747    // There must be a way around this check to do the globalMemStart...
748    Addr tmp_vaddr = pkt->req->getVaddr();
749
750    updatePageDivergenceDist(tmp_vaddr);
751
752    pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(),
753                      pkt->req->getFlags(), pkt->req->masterId(),
754                      pkt->req->getPC());
755
756    // figure out the type of the request to set read/write
757    BaseTLB::Mode TLB_mode;
758    assert(pkt->isRead() || pkt->isWrite());
759
760    // Check write before read for atomic operations
761    // since atomic operations should use BaseTLB::Write
762    if (pkt->isWrite()){
763        TLB_mode = BaseTLB::Write;
764    } else if (pkt->isRead()) {
765        TLB_mode = BaseTLB::Read;
766    } else {
767        fatal("pkt is not a read nor a write\n");
768    }
769
770    tlbCycles -= curTick();
771    ++tlbRequests;
772
773    int tlbPort_index = perLaneTLB ? index : 0;
774
775    if (shader->timingSim) {
776        if (debugSegFault) {
777            Process *p = shader->gpuTc->getProcessPtr();
778            Addr vaddr = pkt->req->getVaddr();
779            unsigned size = pkt->getSize();
780
781            if ((vaddr + size - 1) % 64 < vaddr % 64) {
782                panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n",
783                      cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr);
784            }
785
786            Addr paddr;
787
788            if (!p->pTable->translate(vaddr, paddr)) {
789                if (!p->fixupStackFault(vaddr)) {
790                    panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n",
791                          cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
792                          vaddr);
793                }
794            }
795        }
796
797        // This is the SenderState needed upon return
798        pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index);
799
800        // This is the senderState needed by the TLB hierarchy to function
801        TheISA::GpuTLB::TranslationState *translation_state =
802          new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false,
803                                               pkt->senderState);
804
805        pkt->senderState = translation_state;
806
807        if (functionalTLB) {
808            tlbPort[tlbPort_index]->sendFunctional(pkt);
809
810            // update the hitLevel distribution
811            int hit_level = translation_state->hitLevel;
812            assert(hit_level != -1);
813            hitsPerTLBLevel[hit_level]++;
814
815            // New SenderState for the memory access
816            X86ISA::GpuTLB::TranslationState *sender_state =
817                safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState);
818
819            delete sender_state->tlbEntry;
820            delete sender_state->saved;
821            delete sender_state;
822
823            assert(pkt->req->hasPaddr());
824            assert(pkt->req->hasSize());
825
826            uint8_t *tmpData = pkt->getPtr<uint8_t>();
827
828            // this is necessary because the GPU TLB receives packets instead
829            // of requests. when the translation is complete, all relevent
830            // fields in the request will be populated, but not in the packet.
831            // here we create the new packet so we can set the size, addr,
832            // and proper flags.
833            PacketPtr oldPkt = pkt;
834            pkt = new Packet(oldPkt->req, oldPkt->cmd);
835            delete oldPkt;
836            pkt->dataStatic(tmpData);
837
838
839            // New SenderState for the memory access
840            pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst,
841                                                             index, nullptr);
842
843            gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index);
844            gpuDynInst->tlbHitLevel[index] = hit_level;
845
846
847            // translation is done. Schedule the mem_req_event at the
848            // appropriate cycle to send the timing memory request to ruby
849            EventFunctionWrapper *mem_req_event =
850                memPort[index]->createMemReqEvent(pkt);
851
852            DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data "
853                    "scheduled\n", cu_id, gpuDynInst->simdId,
854                    gpuDynInst->wfSlotId, index, pkt->req->getPaddr());
855
856            schedule(mem_req_event, curTick() + req_tick_latency);
857        } else if (tlbPort[tlbPort_index]->isStalled()) {
858            assert(tlbPort[tlbPort_index]->retries.size() > 0);
859
860            DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
861                    "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
862                    tmp_vaddr);
863
864            tlbPort[tlbPort_index]->retries.push_back(pkt);
865        } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) {
866            // Stall the data port;
867            // No more packet will be issued till
868            // ruby indicates resources are freed by
869            // a recvReqRetry() call back on this port.
870            tlbPort[tlbPort_index]->stallPort();
871
872            DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x "
873                    "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
874                    tmp_vaddr);
875
876            tlbPort[tlbPort_index]->retries.push_back(pkt);
877        } else {
878           DPRINTF(GPUTLB,
879                   "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n",
880                   cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr);
881        }
882    } else {
883        if (pkt->cmd == MemCmd::MemFenceReq) {
884            gpuDynInst->statusBitVector = VectorMask(0);
885        } else {
886            gpuDynInst->statusBitVector &= (~(1ll << index));
887        }
888
889        // New SenderState for the memory access
890        delete pkt->senderState;
891
892        // Because it's atomic operation, only need TLB translation state
893        pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode,
894                                                                shader->gpuTc);
895
896        tlbPort[tlbPort_index]->sendFunctional(pkt);
897
898        // the addr of the packet is not modified, so we need to create a new
899        // packet, or otherwise the memory access will have the old virtual
900        // address sent in the translation packet, instead of the physical
901        // address returned by the translation.
902        PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd);
903        new_pkt->dataStatic(pkt->getPtr<uint8_t>());
904
905        // Translation is done. It is safe to send the packet to memory.
906        memPort[0]->sendFunctional(new_pkt);
907
908        DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id,
909                gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
910                new_pkt->req->getPaddr());
911
912        // safe_cast the senderState
913        TheISA::GpuTLB::TranslationState *sender_state =
914             safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
915
916        delete sender_state->tlbEntry;
917        delete new_pkt;
918        delete pkt->senderState;
919        delete pkt->req;
920        delete pkt;
921    }
922}
923
924void
925ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt)
926{
927    EventFunctionWrapper *mem_req_event =
928        memPort[index]->createMemReqEvent(pkt);
929
930
931    // New SenderState for the memory access
932    pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index,
933                                                              nullptr);
934
935    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n",
936            cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index,
937            pkt->req->getPaddr());
938
939    schedule(mem_req_event, curTick() + req_tick_latency);
940}
941
942void
943ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch,
944                                  Request* req)
945{
946    assert(gpuDynInst->isGlobalSeg());
947
948    if (!req) {
949        req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId);
950    }
951    req->setPaddr(0);
952    if (kernelLaunch) {
953        req->setFlags(Request::KERNEL);
954    }
955
956    // for non-kernel MemFence operations, memorder flags are set depending
957    // on which type of request is currently being sent, so this
958    // should be set by the caller (e.g. if an inst has acq-rel
959    // semantics, it will send one acquire req an one release req)
960    gpuDynInst->setRequestFlags(req, kernelLaunch);
961
962    // a mem fence must correspond to an acquire/release request
963    assert(req->isAcquire() || req->isRelease());
964
965    // create packet
966    PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq);
967
968    // set packet's sender state
969    pkt->senderState =
970        new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr);
971
972    // send the packet
973    sendSyncRequest(gpuDynInst, 0, pkt);
974}
975
976void
977ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt)
978{
979    DataPort::SenderState *sender_state =
980        safe_cast<DataPort::SenderState*>(pkt->senderState);
981
982    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
983    ComputeUnit *compute_unit = computeUnit;
984
985    assert(gpuDynInst);
986
987    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n",
988            compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId,
989            pkt->req->getPaddr(), index);
990
991    Addr paddr = pkt->req->getPaddr();
992
993    if (pkt->cmd != MemCmd::MemFenceResp) {
994        int index = gpuDynInst->memStatusVector[paddr].back();
995
996        DPRINTF(GPUMem, "Response for addr %#x, index %d\n",
997                pkt->req->getPaddr(), index);
998
999        gpuDynInst->memStatusVector[paddr].pop_back();
1000        gpuDynInst->pAddr = pkt->req->getPaddr();
1001
1002        if (pkt->isRead() || pkt->isWrite()) {
1003
1004            if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) {
1005                gpuDynInst->statusBitVector &= (~(1ULL << index));
1006            } else {
1007                assert(gpuDynInst->statusVector[index] > 0);
1008                gpuDynInst->statusVector[index]--;
1009
1010                if (!gpuDynInst->statusVector[index])
1011                    gpuDynInst->statusBitVector &= (~(1ULL << index));
1012            }
1013
1014            DPRINTF(GPUMem, "bitvector is now %#x\n",
1015                    gpuDynInst->statusBitVector);
1016
1017            if (gpuDynInst->statusBitVector == VectorMask(0)) {
1018                auto iter = gpuDynInst->memStatusVector.begin();
1019                auto end = gpuDynInst->memStatusVector.end();
1020
1021                while (iter != end) {
1022                    assert(iter->second.empty());
1023                    ++iter;
1024                }
1025
1026                gpuDynInst->memStatusVector.clear();
1027
1028                if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST)
1029                    gpuDynInst->statusVector.clear();
1030
1031                compute_unit->globalMemoryPipe.handleResponse(gpuDynInst);
1032
1033                DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n",
1034                        compute_unit->cu_id, gpuDynInst->simdId,
1035                        gpuDynInst->wfSlotId);
1036
1037                // after clearing the status vectors,
1038                // see if there is a continuation to perform
1039                // the continuation may generate more work for
1040                // this memory request
1041                if (gpuDynInst->useContinuation) {
1042                    assert(!gpuDynInst->isNoScope());
1043                    gpuDynInst->execContinuation(
1044                        gpuDynInst->staticInstruction(),
1045                        gpuDynInst);
1046                }
1047            }
1048        }
1049    } else {
1050        gpuDynInst->statusBitVector = VectorMask(0);
1051
1052        if (gpuDynInst->useContinuation) {
1053            assert(!gpuDynInst->isNoScope());
1054            gpuDynInst->execContinuation(gpuDynInst->staticInstruction(),
1055                                         gpuDynInst);
1056        }
1057    }
1058
1059    delete pkt->senderState;
1060    delete pkt->req;
1061    delete pkt;
1062}
1063
1064ComputeUnit*
1065ComputeUnitParams::create()
1066{
1067    return new ComputeUnit(this);
1068}
1069
1070bool
1071ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt)
1072{
1073    Addr line = pkt->req->getPaddr();
1074
1075    DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id,
1076            pkt->req->getVaddr(), line);
1077
1078    assert(pkt->senderState);
1079    computeUnit->tlbCycles += curTick();
1080
1081    // pop off the TLB translation state
1082    TheISA::GpuTLB::TranslationState *translation_state =
1083               safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1084
1085    // no PageFaults are permitted for data accesses
1086    if (!translation_state->tlbEntry->valid) {
1087        DTLBPort::SenderState *sender_state =
1088            safe_cast<DTLBPort::SenderState*>(translation_state->saved);
1089
1090        Wavefront *w M5_VAR_USED =
1091            computeUnit->wfList[sender_state->_gpuDynInst->simdId]
1092            [sender_state->_gpuDynInst->wfSlotId];
1093
1094        DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId,
1095                 pkt->req->getVaddr());
1096    }
1097
1098    assert(translation_state->tlbEntry->valid);
1099
1100    // update the hitLevel distribution
1101    int hit_level = translation_state->hitLevel;
1102    computeUnit->hitsPerTLBLevel[hit_level]++;
1103
1104    delete translation_state->tlbEntry;
1105    assert(!translation_state->ports.size());
1106    pkt->senderState = translation_state->saved;
1107
1108    // for prefetch pkt
1109    BaseTLB::Mode TLB_mode = translation_state->tlbMode;
1110
1111    delete translation_state;
1112
1113    // use the original sender state to know how to close this transaction
1114    DTLBPort::SenderState *sender_state =
1115        safe_cast<DTLBPort::SenderState*>(pkt->senderState);
1116
1117    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1118    int mp_index = sender_state->portIndex;
1119    Addr vaddr = pkt->req->getVaddr();
1120    gpuDynInst->memStatusVector[line].push_back(mp_index);
1121    gpuDynInst->tlbHitLevel[mp_index] = hit_level;
1122
1123    MemCmd requestCmd;
1124
1125    if (pkt->cmd == MemCmd::ReadResp) {
1126        requestCmd = MemCmd::ReadReq;
1127    } else if (pkt->cmd == MemCmd::WriteResp) {
1128        requestCmd = MemCmd::WriteReq;
1129    } else if (pkt->cmd == MemCmd::SwapResp) {
1130        requestCmd = MemCmd::SwapReq;
1131    } else {
1132        panic("unsupported response to request conversion %s\n",
1133              pkt->cmd.toString());
1134    }
1135
1136    if (computeUnit->prefetchDepth) {
1137        int simdId = gpuDynInst->simdId;
1138        int wfSlotId = gpuDynInst->wfSlotId;
1139        Addr last = 0;
1140
1141        switch(computeUnit->prefetchType) {
1142        case Enums::PF_CU:
1143            last = computeUnit->lastVaddrCU[mp_index];
1144            break;
1145        case Enums::PF_PHASE:
1146            last = computeUnit->lastVaddrSimd[simdId][mp_index];
1147            break;
1148        case Enums::PF_WF:
1149            last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index];
1150        default:
1151            break;
1152        }
1153
1154        DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n",
1155                computeUnit->cu_id, simdId, wfSlotId, mp_index, last);
1156
1157        int stride = last ? (roundDown(vaddr, TheISA::PageBytes) -
1158                     roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift
1159                     : 0;
1160
1161        DPRINTF(GPUPrefetch, "Stride is %d\n", stride);
1162
1163        computeUnit->lastVaddrCU[mp_index] = vaddr;
1164        computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr;
1165        computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr;
1166
1167        stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ?
1168            computeUnit->prefetchStride: stride;
1169
1170        DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr,
1171                computeUnit->cu_id, simdId, wfSlotId, mp_index);
1172
1173        DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr);
1174
1175        // Prefetch Next few pages atomically
1176        for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) {
1177            DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride,
1178                    vaddr+stride*pf*TheISA::PageBytes);
1179
1180            if (!stride)
1181                break;
1182
1183            Request *prefetch_req = new Request(0, vaddr + stride * pf *
1184                                                TheISA::PageBytes,
1185                                                sizeof(uint8_t), 0,
1186                                                computeUnit->masterId(),
1187                                                0, 0, 0);
1188
1189            PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd);
1190            uint8_t foo = 0;
1191            prefetch_pkt->dataStatic(&foo);
1192
1193            // Because it's atomic operation, only need TLB translation state
1194            prefetch_pkt->senderState =
1195                new TheISA::GpuTLB::TranslationState(TLB_mode,
1196                                                     computeUnit->shader->gpuTc,
1197                                                     true);
1198
1199            // Currently prefetches are zero-latency, hence the sendFunctional
1200            sendFunctional(prefetch_pkt);
1201
1202            /* safe_cast the senderState */
1203            TheISA::GpuTLB::TranslationState *tlb_state =
1204                 safe_cast<TheISA::GpuTLB::TranslationState*>(
1205                         prefetch_pkt->senderState);
1206
1207
1208            delete tlb_state->tlbEntry;
1209            delete tlb_state;
1210            delete prefetch_pkt->req;
1211            delete prefetch_pkt;
1212        }
1213    }
1214
1215    // First we must convert the response cmd back to a request cmd so that
1216    // the request can be sent through the cu's master port
1217    PacketPtr new_pkt = new Packet(pkt->req, requestCmd);
1218    new_pkt->dataStatic(pkt->getPtr<uint8_t>());
1219    delete pkt->senderState;
1220    delete pkt;
1221
1222    // New SenderState for the memory access
1223    new_pkt->senderState =
1224            new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index,
1225                                                   nullptr);
1226
1227    // translation is done. Schedule the mem_req_event at the appropriate
1228    // cycle to send the timing memory request to ruby
1229    EventFunctionWrapper *mem_req_event =
1230        computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt);
1231
1232    DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n",
1233            computeUnit->cu_id, gpuDynInst->simdId,
1234            gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr());
1235
1236    computeUnit->schedule(mem_req_event, curTick() +
1237                          computeUnit->req_tick_latency);
1238
1239    return true;
1240}
1241
1242EventFunctionWrapper*
1243ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt)
1244{
1245    return new EventFunctionWrapper(
1246        [this, pkt]{ processMemReqEvent(pkt); },
1247        "ComputeUnit memory request event", true);
1248}
1249
1250EventFunctionWrapper*
1251ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt)
1252{
1253    return new EventFunctionWrapper(
1254        [this, pkt]{ processMemRespEvent(pkt); },
1255        "ComputeUnit memory response event", true);
1256}
1257
1258void
1259ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt)
1260{
1261    SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState);
1262    GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst;
1263    ComputeUnit *compute_unit M5_VAR_USED = computeUnit;
1264
1265    if (!(sendTimingReq(pkt))) {
1266        retries.push_back(std::make_pair(pkt, gpuDynInst));
1267
1268        DPRINTF(GPUPort,
1269                "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n",
1270                compute_unit->cu_id, gpuDynInst->simdId,
1271                gpuDynInst->wfSlotId, index,
1272                pkt->req->getPaddr());
1273    } else {
1274        DPRINTF(GPUPort,
1275                "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n",
1276                compute_unit->cu_id, gpuDynInst->simdId,
1277                gpuDynInst->wfSlotId, index,
1278                pkt->req->getPaddr());
1279    }
1280}
1281
1282/*
1283 * The initial translation request could have been rejected,
1284 * if <retries> queue is not Retry sending the translation
1285 * request. sendRetry() is called from the peer port whenever
1286 * a translation completes.
1287 */
1288void
1289ComputeUnit::DTLBPort::recvReqRetry()
1290{
1291    int len = retries.size();
1292
1293    DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n",
1294            computeUnit->cu_id, len);
1295
1296    assert(len > 0);
1297    assert(isStalled());
1298    // recvReqRetry is an indication that the resource on which this
1299    // port was stalling on is freed. So, remove the stall first
1300    unstallPort();
1301
1302    for (int i = 0; i < len; ++i) {
1303        PacketPtr pkt = retries.front();
1304        Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1305        DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr);
1306
1307        if (!sendTimingReq(pkt)) {
1308            // Stall port
1309            stallPort();
1310            DPRINTF(GPUTLB, ": failed again\n");
1311            break;
1312        } else {
1313            DPRINTF(GPUTLB, ": successful\n");
1314            retries.pop_front();
1315        }
1316    }
1317}
1318
1319bool
1320ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt)
1321{
1322    Addr line M5_VAR_USED = pkt->req->getPaddr();
1323    DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n",
1324            computeUnit->cu_id, pkt->req->getVaddr(), line);
1325
1326    assert(pkt->senderState);
1327
1328    // pop off the TLB translation state
1329    TheISA::GpuTLB::TranslationState *translation_state =
1330                 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
1331
1332    bool success = translation_state->tlbEntry->valid;
1333    delete translation_state->tlbEntry;
1334    assert(!translation_state->ports.size());
1335    pkt->senderState = translation_state->saved;
1336    delete translation_state;
1337
1338    // use the original sender state to know how to close this transaction
1339    ITLBPort::SenderState *sender_state =
1340        safe_cast<ITLBPort::SenderState*>(pkt->senderState);
1341
1342    // get the wavefront associated with this translation request
1343    Wavefront *wavefront = sender_state->wavefront;
1344    delete pkt->senderState;
1345
1346    if (success) {
1347        // pkt is reused in fetch(), don't delete it here.  However, we must
1348        // reset the command to be a request so that it can be sent through
1349        // the cu's master port
1350        assert(pkt->cmd == MemCmd::ReadResp);
1351        pkt->cmd = MemCmd::ReadReq;
1352
1353        computeUnit->fetchStage.fetch(pkt, wavefront);
1354    } else {
1355        if (wavefront->dropFetch) {
1356            assert(wavefront->instructionBuffer.empty());
1357            wavefront->dropFetch = false;
1358        }
1359
1360        wavefront->pendingFetch = 0;
1361    }
1362
1363    return true;
1364}
1365
1366/*
1367 * The initial translation request could have been rejected, if
1368 * <retries> queue is not empty. Retry sending the translation
1369 * request. sendRetry() is called from the peer port whenever
1370 * a translation completes.
1371 */
1372void
1373ComputeUnit::ITLBPort::recvReqRetry()
1374{
1375
1376    int len = retries.size();
1377    DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len);
1378
1379    assert(len > 0);
1380    assert(isStalled());
1381
1382    // recvReqRetry is an indication that the resource on which this
1383    // port was stalling on is freed. So, remove the stall first
1384    unstallPort();
1385
1386    for (int i = 0; i < len; ++i) {
1387        PacketPtr pkt = retries.front();
1388        Addr vaddr M5_VAR_USED = pkt->req->getVaddr();
1389        DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr);
1390
1391        if (!sendTimingReq(pkt)) {
1392            stallPort(); // Stall port
1393            DPRINTF(GPUTLB, ": failed again\n");
1394            break;
1395        } else {
1396            DPRINTF(GPUTLB, ": successful\n");
1397            retries.pop_front();
1398        }
1399    }
1400}
1401
1402void
1403ComputeUnit::regStats()
1404{
1405    MemObject::regStats();
1406
1407    vALUInsts
1408        .name(name() + ".valu_insts")
1409        .desc("Number of vector ALU insts issued.")
1410        ;
1411    vALUInstsPerWF
1412        .name(name() + ".valu_insts_per_wf")
1413        .desc("The avg. number of vector ALU insts issued per-wavefront.")
1414        ;
1415    sALUInsts
1416        .name(name() + ".salu_insts")
1417        .desc("Number of scalar ALU insts issued.")
1418        ;
1419    sALUInstsPerWF
1420        .name(name() + ".salu_insts_per_wf")
1421        .desc("The avg. number of scalar ALU insts issued per-wavefront.")
1422        ;
1423    instCyclesVALU
1424        .name(name() + ".inst_cycles_valu")
1425        .desc("Number of cycles needed to execute VALU insts.")
1426        ;
1427    instCyclesSALU
1428        .name(name() + ".inst_cycles_salu")
1429        .desc("Number of cycles needed to execute SALU insts.")
1430        ;
1431    threadCyclesVALU
1432        .name(name() + ".thread_cycles_valu")
1433        .desc("Number of thread cycles used to execute vector ALU ops. "
1434              "Similar to instCyclesVALU but multiplied by the number of "
1435              "active threads.")
1436        ;
1437    vALUUtilization
1438        .name(name() + ".valu_utilization")
1439        .desc("Percentage of active vector ALU threads in a wave.")
1440        ;
1441    ldsNoFlatInsts
1442        .name(name() + ".lds_no_flat_insts")
1443        .desc("Number of LDS insts issued, not including FLAT "
1444              "accesses that resolve to LDS.")
1445        ;
1446    ldsNoFlatInstsPerWF
1447        .name(name() + ".lds_no_flat_insts_per_wf")
1448        .desc("The avg. number of LDS insts (not including FLAT "
1449              "accesses that resolve to LDS) per-wavefront.")
1450        ;
1451    flatVMemInsts
1452        .name(name() + ".flat_vmem_insts")
1453        .desc("The number of FLAT insts that resolve to vmem issued.")
1454        ;
1455    flatVMemInstsPerWF
1456        .name(name() + ".flat_vmem_insts_per_wf")
1457        .desc("The average number of FLAT insts that resolve to vmem "
1458              "issued per-wavefront.")
1459        ;
1460    flatLDSInsts
1461        .name(name() + ".flat_lds_insts")
1462        .desc("The number of FLAT insts that resolve to LDS issued.")
1463        ;
1464    flatLDSInstsPerWF
1465        .name(name() + ".flat_lds_insts_per_wf")
1466        .desc("The average number of FLAT insts that resolve to LDS "
1467              "issued per-wavefront.")
1468        ;
1469    vectorMemWrites
1470        .name(name() + ".vector_mem_writes")
1471        .desc("Number of vector mem write insts (excluding FLAT insts).")
1472        ;
1473    vectorMemWritesPerWF
1474        .name(name() + ".vector_mem_writes_per_wf")
1475        .desc("The average number of vector mem write insts "
1476              "(excluding FLAT insts) per-wavefront.")
1477        ;
1478    vectorMemReads
1479        .name(name() + ".vector_mem_reads")
1480        .desc("Number of vector mem read insts (excluding FLAT insts).")
1481        ;
1482    vectorMemReadsPerWF
1483        .name(name() + ".vector_mem_reads_per_wf")
1484        .desc("The avg. number of vector mem read insts (excluding "
1485              "FLAT insts) per-wavefront.")
1486        ;
1487    scalarMemWrites
1488        .name(name() + ".scalar_mem_writes")
1489        .desc("Number of scalar mem write insts.")
1490        ;
1491    scalarMemWritesPerWF
1492        .name(name() + ".scalar_mem_writes_per_wf")
1493        .desc("The average number of scalar mem write insts per-wavefront.")
1494        ;
1495    scalarMemReads
1496        .name(name() + ".scalar_mem_reads")
1497        .desc("Number of scalar mem read insts.")
1498        ;
1499    scalarMemReadsPerWF
1500        .name(name() + ".scalar_mem_reads_per_wf")
1501        .desc("The average number of scalar mem read insts per-wavefront.")
1502        ;
1503
1504    vALUInstsPerWF = vALUInsts / completedWfs;
1505    sALUInstsPerWF = sALUInsts / completedWfs;
1506    vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100;
1507    ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs;
1508    flatVMemInstsPerWF = flatVMemInsts / completedWfs;
1509    flatLDSInstsPerWF = flatLDSInsts / completedWfs;
1510    vectorMemWritesPerWF = vectorMemWrites / completedWfs;
1511    vectorMemReadsPerWF = vectorMemReads / completedWfs;
1512    scalarMemWritesPerWF = scalarMemWrites / completedWfs;
1513    scalarMemReadsPerWF = scalarMemReads / completedWfs;
1514
1515    tlbCycles
1516        .name(name() + ".tlb_cycles")
1517        .desc("total number of cycles for all uncoalesced requests")
1518        ;
1519
1520    tlbRequests
1521        .name(name() + ".tlb_requests")
1522        .desc("number of uncoalesced requests")
1523        ;
1524
1525    tlbLatency
1526        .name(name() + ".avg_translation_latency")
1527        .desc("Avg. translation latency for data translations")
1528        ;
1529
1530    tlbLatency = tlbCycles / tlbRequests;
1531
1532    hitsPerTLBLevel
1533       .init(4)
1534       .name(name() + ".TLB_hits_distribution")
1535       .desc("TLB hits distribution (0 for page table, x for Lx-TLB")
1536       ;
1537
1538    // fixed number of TLB levels
1539    for (int i = 0; i < 4; ++i) {
1540        if (!i)
1541            hitsPerTLBLevel.subname(i,"page_table");
1542        else
1543            hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i));
1544    }
1545
1546    execRateDist
1547        .init(0, 10, 2)
1548        .name(name() + ".inst_exec_rate")
1549        .desc("Instruction Execution Rate: Number of executed vector "
1550              "instructions per cycle")
1551        ;
1552
1553    ldsBankConflictDist
1554       .init(0, wfSize(), 2)
1555       .name(name() + ".lds_bank_conflicts")
1556       .desc("Number of bank conflicts per LDS memory packet")
1557       ;
1558
1559    ldsBankAccesses
1560        .name(name() + ".lds_bank_access_cnt")
1561        .desc("Total number of LDS bank accesses")
1562        ;
1563
1564    pageDivergenceDist
1565        // A wavefront can touch up to N pages per memory instruction where
1566        // N is equal to the wavefront size
1567        // The number of pages per bin can be configured (here it's 4).
1568       .init(1, wfSize(), 4)
1569       .name(name() + ".page_divergence_dist")
1570       .desc("pages touched per wf (over all mem. instr.)")
1571       ;
1572
1573    controlFlowDivergenceDist
1574        .init(1, wfSize(), 4)
1575        .name(name() + ".warp_execution_dist")
1576        .desc("number of lanes active per instruction (oval all instructions)")
1577        ;
1578
1579    activeLanesPerGMemInstrDist
1580        .init(1, wfSize(), 4)
1581        .name(name() + ".gmem_lanes_execution_dist")
1582        .desc("number of active lanes per global memory instruction")
1583        ;
1584
1585    activeLanesPerLMemInstrDist
1586        .init(1, wfSize(), 4)
1587        .name(name() + ".lmem_lanes_execution_dist")
1588        .desc("number of active lanes per local memory instruction")
1589        ;
1590
1591    numInstrExecuted
1592        .name(name() + ".num_instr_executed")
1593        .desc("number of instructions executed")
1594        ;
1595
1596    numVecOpsExecuted
1597        .name(name() + ".num_vec_ops_executed")
1598        .desc("number of vec ops executed (e.g. WF size/inst)")
1599        ;
1600
1601    totalCycles
1602        .name(name() + ".num_total_cycles")
1603        .desc("number of cycles the CU ran for")
1604        ;
1605
1606    ipc
1607        .name(name() + ".ipc")
1608        .desc("Instructions per cycle (this CU only)")
1609        ;
1610
1611    vpc
1612        .name(name() + ".vpc")
1613        .desc("Vector Operations per cycle (this CU only)")
1614        ;
1615
1616    numALUInstsExecuted
1617        .name(name() + ".num_alu_insts_executed")
1618        .desc("Number of dynamic non-GM memory insts executed")
1619        ;
1620
1621    wgBlockedDueLdsAllocation
1622        .name(name() + ".wg_blocked_due_lds_alloc")
1623        .desc("Workgroup blocked due to LDS capacity")
1624        ;
1625
1626    ipc = numInstrExecuted / totalCycles;
1627    vpc = numVecOpsExecuted / totalCycles;
1628
1629    numTimesWgBlockedDueVgprAlloc
1630        .name(name() + ".times_wg_blocked_due_vgpr_alloc")
1631        .desc("Number of times WGs are blocked due to VGPR allocation per SIMD")
1632        ;
1633
1634    dynamicGMemInstrCnt
1635        .name(name() + ".global_mem_instr_cnt")
1636        .desc("dynamic global memory instructions count")
1637        ;
1638
1639    dynamicLMemInstrCnt
1640        .name(name() + ".local_mem_instr_cnt")
1641        .desc("dynamic local memory intruction count")
1642        ;
1643
1644    numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt -
1645        dynamicLMemInstrCnt;
1646
1647    completedWfs
1648        .name(name() + ".num_completed_wfs")
1649        .desc("number of completed wavefronts")
1650        ;
1651
1652    numCASOps
1653        .name(name() + ".num_CAS_ops")
1654        .desc("number of compare and swap operations")
1655        ;
1656
1657    numFailedCASOps
1658        .name(name() + ".num_failed_CAS_ops")
1659        .desc("number of compare and swap operations that failed")
1660        ;
1661
1662    // register stats of pipeline stages
1663    fetchStage.regStats();
1664    scoreboardCheckStage.regStats();
1665    scheduleStage.regStats();
1666    execStage.regStats();
1667
1668    // register stats of memory pipeline
1669    globalMemoryPipe.regStats();
1670    localMemoryPipe.regStats();
1671}
1672
1673void
1674ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst)
1675{
1676    if (gpuDynInst->isScalar()) {
1677        if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) {
1678            sALUInsts++;
1679            instCyclesSALU++;
1680        } else if (gpuDynInst->isLoad()) {
1681            scalarMemReads++;
1682        } else if (gpuDynInst->isStore()) {
1683            scalarMemWrites++;
1684        }
1685    } else {
1686        if (gpuDynInst->isALU()) {
1687            vALUInsts++;
1688            instCyclesVALU++;
1689            threadCyclesVALU += gpuDynInst->wavefront()->execMask().count();
1690        } else if (gpuDynInst->isFlat()) {
1691            if (gpuDynInst->isLocalMem()) {
1692                flatLDSInsts++;
1693            } else {
1694                flatVMemInsts++;
1695            }
1696        } else if (gpuDynInst->isLocalMem()) {
1697            ldsNoFlatInsts++;
1698        } else if (gpuDynInst->isLoad()) {
1699            vectorMemReads++;
1700        } else if (gpuDynInst->isStore()) {
1701            vectorMemWrites++;
1702        }
1703    }
1704}
1705
1706void
1707ComputeUnit::updatePageDivergenceDist(Addr addr)
1708{
1709    Addr virt_page_addr = roundDown(addr, TheISA::PageBytes);
1710
1711    if (!pagesTouched.count(virt_page_addr))
1712        pagesTouched[virt_page_addr] = 1;
1713    else
1714        pagesTouched[virt_page_addr]++;
1715}
1716
1717void
1718ComputeUnit::CUExitCallback::process()
1719{
1720    if (computeUnit->countPages) {
1721        std::ostream *page_stat_file =
1722            simout.create(computeUnit->name().c_str())->stream();
1723
1724        *page_stat_file << "page, wavefront accesses, workitem accesses" <<
1725            std::endl;
1726
1727        for (auto iter : computeUnit->pageAccesses) {
1728            *page_stat_file << std::hex << iter.first << ",";
1729            *page_stat_file << std::dec << iter.second.first << ",";
1730            *page_stat_file << std::dec << iter.second.second << std::endl;
1731        }
1732    }
1733 }
1734
1735bool
1736ComputeUnit::isDone() const
1737{
1738    for (int i = 0; i < numSIMDs; ++i) {
1739        if (!isSimdDone(i)) {
1740            return false;
1741        }
1742    }
1743
1744    bool glbMemBusRdy = true;
1745    for (int j = 0; j < numGlbMemUnits; ++j) {
1746        glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy();
1747    }
1748    bool locMemBusRdy = true;
1749    for (int j = 0; j < numLocMemUnits; ++j) {
1750        locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy();
1751    }
1752
1753    if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() ||
1754        !globalMemoryPipe.isGMStRespFIFOWrRdy() ||
1755        !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy()
1756        || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() ||
1757        !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) {
1758        return false;
1759    }
1760
1761    return true;
1762}
1763
1764int32_t
1765ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const
1766{
1767    return lds.getRefCounter(dispatchId, wgId);
1768}
1769
1770bool
1771ComputeUnit::isSimdDone(uint32_t simdId) const
1772{
1773    assert(simdId < numSIMDs);
1774
1775    for (int i=0; i < numGlbMemUnits; ++i) {
1776        if (!vrfToGlobalMemPipeBus[i].rdy())
1777            return false;
1778    }
1779    for (int i=0; i < numLocMemUnits; ++i) {
1780        if (!vrfToLocalMemPipeBus[i].rdy())
1781            return false;
1782    }
1783    if (!aluPipe[simdId].rdy()) {
1784        return false;
1785    }
1786
1787    for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){
1788        if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) {
1789            return false;
1790        }
1791    }
1792
1793    return true;
1794}
1795
1796/**
1797 * send a general request to the LDS
1798 * make sure to look at the return value here as your request might be
1799 * NACK'd and returning false means that you have to have some backup plan
1800 */
1801bool
1802ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst)
1803{
1804    // this is just a request to carry the GPUDynInstPtr
1805    // back and forth
1806    Request *newRequest = new Request();
1807    newRequest->setPaddr(0x0);
1808
1809    // ReadReq is not evaluted by the LDS but the Packet ctor requires this
1810    PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq);
1811
1812    // This is the SenderState needed upon return
1813    newPacket->senderState = new LDSPort::SenderState(gpuDynInst);
1814
1815    return ldsPort->sendTimingReq(newPacket);
1816}
1817
1818/**
1819 * get the result of packets sent to the LDS when they return
1820 */
1821bool
1822ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet)
1823{
1824    const ComputeUnit::LDSPort::SenderState *senderState =
1825        dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState);
1826
1827    fatal_if(!senderState, "did not get the right sort of sender state");
1828
1829    GPUDynInstPtr gpuDynInst = senderState->getMemInst();
1830
1831    delete packet->senderState;
1832    delete packet->req;
1833    delete packet;
1834
1835    computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst);
1836    return true;
1837}
1838
1839/**
1840 * attempt to send this packet, either the port is already stalled, the request
1841 * is nack'd and must stall or the request goes through
1842 * when a request cannot be sent, add it to the retries queue
1843 */
1844bool
1845ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt)
1846{
1847    ComputeUnit::LDSPort::SenderState *sender_state =
1848            dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState);
1849    fatal_if(!sender_state, "packet without a valid sender state");
1850
1851    GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst();
1852
1853    if (isStalled()) {
1854        fatal_if(retries.empty(), "must have retries waiting to be stalled");
1855
1856        retries.push(pkt);
1857
1858        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n",
1859                        computeUnit->cu_id, gpuDynInst->simdId,
1860                        gpuDynInst->wfSlotId);
1861        return false;
1862    } else if (!MasterPort::sendTimingReq(pkt)) {
1863        // need to stall the LDS port until a recvReqRetry() is received
1864        // this indicates that there is more space
1865        stallPort();
1866        retries.push(pkt);
1867
1868        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n",
1869                computeUnit->cu_id, gpuDynInst->simdId,
1870                gpuDynInst->wfSlotId, pkt->req->getPaddr());
1871        return false;
1872    } else {
1873        DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n",
1874                computeUnit->cu_id, gpuDynInst->simdId,
1875                gpuDynInst->wfSlotId, pkt->req->getPaddr());
1876        return true;
1877    }
1878}
1879
1880/**
1881 * the bus is telling the port that there is now space so retrying stalled
1882 * requests should work now
1883 * this allows the port to have a request be nack'd and then have the receiver
1884 * say when there is space, rather than simply retrying the send every cycle
1885 */
1886void
1887ComputeUnit::LDSPort::recvReqRetry()
1888{
1889    auto queueSize = retries.size();
1890
1891    DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n",
1892            computeUnit->cu_id, queueSize);
1893
1894    fatal_if(queueSize < 1,
1895             "why was there a recvReqRetry() with no pending reqs?");
1896    fatal_if(!isStalled(),
1897             "recvReqRetry() happened when the port was not stalled");
1898
1899    unstallPort();
1900
1901    while (!retries.empty()) {
1902        PacketPtr packet = retries.front();
1903
1904        DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id);
1905
1906        if (!MasterPort::sendTimingReq(packet)) {
1907            // Stall port
1908            stallPort();
1909            DPRINTF(GPUPort, ": LDS send failed again\n");
1910            break;
1911        } else {
1912            DPRINTF(GPUTLB, ": LDS send successful\n");
1913            retries.pop();
1914        }
1915    }
1916}
1917