compute_unit.cc revision 12749:223c83ed9979
12381SN/A/* 29542Sandreas.hansson@arm.com * Copyright (c) 2011-2015 Advanced Micro Devices, Inc. 38949Sandreas.hansson@arm.com * All rights reserved. 48949Sandreas.hansson@arm.com * 58949Sandreas.hansson@arm.com * For use for simulation and test purposes only 68949Sandreas.hansson@arm.com * 78949Sandreas.hansson@arm.com * Redistribution and use in source and binary forms, with or without 88949Sandreas.hansson@arm.com * modification, are permitted provided that the following conditions are met: 98949Sandreas.hansson@arm.com * 108949Sandreas.hansson@arm.com * 1. Redistributions of source code must retain the above copyright notice, 118949Sandreas.hansson@arm.com * this list of conditions and the following disclaimer. 128949Sandreas.hansson@arm.com * 138949Sandreas.hansson@arm.com * 2. Redistributions in binary form must reproduce the above copyright notice, 142592SN/A * this list of conditions and the following disclaimer in the documentation 157636Ssteve.reinhardt@amd.com * and/or other materials provided with the distribution. 162381SN/A * 172381SN/A * 3. Neither the name of the copyright holder nor the names of its 182381SN/A * contributors may be used to endorse or promote products derived from this 192381SN/A * software without specific prior written permission. 202381SN/A * 212381SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 222381SN/A * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 232381SN/A * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 242381SN/A * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 252381SN/A * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 262381SN/A * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 272381SN/A * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 282381SN/A * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 292381SN/A * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 302381SN/A * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 312381SN/A * POSSIBILITY OF SUCH DAMAGE. 322381SN/A * 332381SN/A * Authors: John Kalamatianos, 342381SN/A * Anthony Gutierrez 352381SN/A */ 362381SN/A 372381SN/A#include "gpu-compute/compute_unit.hh" 382381SN/A 392381SN/A#include <limits> 402665Ssaidi@eecs.umich.edu 412665Ssaidi@eecs.umich.edu#include "base/output.hh" 422665Ssaidi@eecs.umich.edu#include "debug/GPUDisp.hh" 432665Ssaidi@eecs.umich.edu#include "debug/GPUExec.hh" 449031Sandreas.hansson@arm.com#include "debug/GPUFetch.hh" 452381SN/A#include "debug/GPUMem.hh" 462381SN/A#include "debug/GPUPort.hh" 472381SN/A#include "debug/GPUPrefetch.hh" 482381SN/A#include "debug/GPUSync.hh" 492662Sstever@eecs.umich.edu#include "debug/GPUTLB.hh" 502381SN/A#include "gpu-compute/dispatcher.hh" 512381SN/A#include "gpu-compute/gpu_dyn_inst.hh" 522381SN/A#include "gpu-compute/gpu_static_inst.hh" 532381SN/A#include "gpu-compute/ndrange.hh" 542381SN/A#include "gpu-compute/shader.hh" 558229Snate@binkert.org#include "gpu-compute/simple_pool_manager.hh" 563348Sbinkertn@umich.edu#include "gpu-compute/vector_register_file.hh" 573348Sbinkertn@umich.edu#include "gpu-compute/wavefront.hh" 583348Sbinkertn@umich.edu#include "mem/page_table.hh" 595735Snate@binkert.org#include "sim/process.hh" 604024Sbinkertn@umich.edu 615735Snate@binkert.orgComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p), 623940Ssaidi@eecs.umich.edu scoreboardCheckStage(p), scheduleStage(p), execStage(p), 635314Sstever@gmail.com globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0), 646216Snate@binkert.org cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs), 652392SN/A spBypassPipeLength(p->spbypass_pipe_length), 664167Sbinkertn@umich.edu dpBypassPipeLength(p->dpbypass_pipe_length), 672394SN/A issuePeriod(p->issue_period), 688737Skoansin.tan@gmail.com numGlbMemUnits(p->num_global_mem_pipes), 693349Sbinkertn@umich.edu numLocMemUnits(p->num_shared_mem_pipes), 702394SN/A perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth), 712812Srdreslin@umich.edu prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type), 722812Srdreslin@umich.edu xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault), 734022Sstever@eecs.umich.edu functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier), 744022Sstever@eecs.umich.edu countPages(p->countPages), barrier_id(0), 755735Snate@binkert.org vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width), 765735Snate@binkert.org coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width), 774022Sstever@eecs.umich.edu req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()), 785735Snate@binkert.org resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()), 795735Snate@binkert.org _masterId(p->system->getMasterId(this, "ComputeUnit")), 805735Snate@binkert.org lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()), 814022Sstever@eecs.umich.edu globalSeqNum(0), wavefrontSize(p->wfSize), 824022Sstever@eecs.umich.edu kernelLaunchInst(new KernelLaunchStaticInst()) 834022Sstever@eecs.umich.edu{ 844022Sstever@eecs.umich.edu /** 854473Sstever@eecs.umich.edu * This check is necessary because std::bitset only provides conversion 865319Sstever@gmail.com * to unsigned long or unsigned long long via to_ulong() or to_ullong(). 874022Sstever@eecs.umich.edu * there are * a few places in the code where to_ullong() is used, however 884022Sstever@eecs.umich.edu * if VSZ is larger than a value the host can support then bitset will 894022Sstever@eecs.umich.edu * throw a runtime exception. we should remove all use of to_long() or 904022Sstever@eecs.umich.edu * to_ullong() so we can have VSZ greater than 64b, however until that is 914022Sstever@eecs.umich.edu * done this assert is required. 924022Sstever@eecs.umich.edu */ 934022Sstever@eecs.umich.edu fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits || 949018Sandreas.hansson@arm.com p->wfSize <= 0, 959018Sandreas.hansson@arm.com "WF size is larger than the host can support"); 969018Sandreas.hansson@arm.com fatal_if(!isPowerOf2(wavefrontSize), 979018Sandreas.hansson@arm.com "Wavefront size should be a power of 2"); 989018Sandreas.hansson@arm.com // calculate how many cycles a vector load or store will need to transfer 999018Sandreas.hansson@arm.com // its data over the corresponding buses 1009018Sandreas.hansson@arm.com numCyclesPerStoreTransfer = 1019018Sandreas.hansson@arm.com (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) / 1024022Sstever@eecs.umich.edu (double)vrfToCoalescerBusWidth); 1034022Sstever@eecs.umich.edu 1044022Sstever@eecs.umich.edu numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t)) 1057465Ssteve.reinhardt@amd.com / coalescerToVrfBusWidth; 1064628Sstever@eecs.umich.edu 1077465Ssteve.reinhardt@amd.com lastVaddrWF.resize(numSIMDs); 1087465Ssteve.reinhardt@amd.com wfList.resize(numSIMDs); 1094022Sstever@eecs.umich.edu 1104022Sstever@eecs.umich.edu for (int j = 0; j < numSIMDs; ++j) { 1114626Sstever@eecs.umich.edu lastVaddrWF[j].resize(p->n_wf); 1124626Sstever@eecs.umich.edu 1137669Ssteve.reinhardt@amd.com for (int i = 0; i < p->n_wf; ++i) { 1144626Sstever@eecs.umich.edu lastVaddrWF[j][i].resize(wfSize()); 1154040Ssaidi@eecs.umich.edu 1164040Ssaidi@eecs.umich.edu wfList[j].push_back(p->wavefronts[j * p->n_wf + i]); 1175650Sgblack@eecs.umich.edu wfList[j][i]->setParent(this); 1185650Sgblack@eecs.umich.edu 1194870Sstever@eecs.umich.edu for (int k = 0; k < wfSize(); ++k) { 1204870Sstever@eecs.umich.edu lastVaddrWF[j][i][k] = 0; 1214870Sstever@eecs.umich.edu } 1224870Sstever@eecs.umich.edu } 1234870Sstever@eecs.umich.edu } 1244870Sstever@eecs.umich.edu 1258436SBrad.Beckmann@amd.com lastVaddrSimd.resize(numSIMDs); 1268436SBrad.Beckmann@amd.com 1275314Sstever@gmail.com for (int i = 0; i < numSIMDs; ++i) { 1285314Sstever@gmail.com lastVaddrSimd[i].resize(wfSize(), 0); 1298184Ssomayeh@cs.wisc.edu } 1308716Snilay@cs.wisc.edu 1314022Sstever@eecs.umich.edu lastVaddrCU.resize(wfSize()); 1324022Sstever@eecs.umich.edu 1334022Sstever@eecs.umich.edu lds.setParent(this); 1344022Sstever@eecs.umich.edu 1355735Snate@binkert.org if (p->execPolicy == "OLDEST-FIRST") { 1365735Snate@binkert.org exec_policy = EXEC_POLICY::OLDEST; 1375735Snate@binkert.org } else if (p->execPolicy == "ROUND-ROBIN") { 1384022Sstever@eecs.umich.edu exec_policy = EXEC_POLICY::RR; 1394022Sstever@eecs.umich.edu } else { 1404626Sstever@eecs.umich.edu fatal("Invalid WF execution policy (CU)\n"); 1414626Sstever@eecs.umich.edu } 1427465Ssteve.reinhardt@amd.com 1434022Sstever@eecs.umich.edu memPort.resize(wfSize()); 1444626Sstever@eecs.umich.edu 1454626Sstever@eecs.umich.edu // resize the tlbPort vectorArray 1464626Sstever@eecs.umich.edu int tlbPort_width = perLaneTLB ? wfSize() : 1; 1474626Sstever@eecs.umich.edu tlbPort.resize(tlbPort_width); 1484022Sstever@eecs.umich.edu 1494022Sstever@eecs.umich.edu cuExitCallback = new CUExitCallback(this); 1506076Sgblack@eecs.umich.edu registerExitCallback(cuExitCallback); 1514626Sstever@eecs.umich.edu 1524870Sstever@eecs.umich.edu xactCasLoadMap.clear(); 1535314Sstever@gmail.com lastExecCycle.resize(numSIMDs, 0); 1548184Ssomayeh@cs.wisc.edu 1554022Sstever@eecs.umich.edu for (int i = 0; i < vrf.size(); ++i) { 1564022Sstever@eecs.umich.edu vrf[i]->setParent(this); 1574022Sstever@eecs.umich.edu } 1585735Snate@binkert.org 1595735Snate@binkert.org numVecRegsPerSimd = vrf[0]->numRegs(); 1605735Snate@binkert.org} 1615735Snate@binkert.org 1625735Snate@binkert.orgComputeUnit::~ComputeUnit() 1635735Snate@binkert.org{ 1645735Snate@binkert.org // Delete wavefront slots 1654022Sstever@eecs.umich.edu for (int j = 0; j < numSIMDs; ++j) { 1665735Snate@binkert.org for (int i = 0; i < shader->n_wf; ++i) { 1675735Snate@binkert.org delete wfList[j][i]; 1684022Sstever@eecs.umich.edu } 1695735Snate@binkert.org lastVaddrSimd[j].clear(); 1704022Sstever@eecs.umich.edu } 1714022Sstever@eecs.umich.edu lastVaddrCU.clear(); 1724022Sstever@eecs.umich.edu readyList.clear(); 1735735Snate@binkert.org waveStatusList.clear(); 1744022Sstever@eecs.umich.edu dispatchList.clear(); 1754022Sstever@eecs.umich.edu vectorAluInstAvail.clear(); 1764022Sstever@eecs.umich.edu delete cuExitCallback; 1774022Sstever@eecs.umich.edu delete ldsPort; 1784022Sstever@eecs.umich.edu} 1794022Sstever@eecs.umich.edu 1805735Snate@binkert.orgvoid 1815735Snate@binkert.orgComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr) 1825735Snate@binkert.org{ 1834022Sstever@eecs.umich.edu w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount); 1844022Sstever@eecs.umich.edu 1854022Sstever@eecs.umich.edu w->workGroupSz[0] = ndr->q.wgSize[0]; 1864022Sstever@eecs.umich.edu w->workGroupSz[1] = ndr->q.wgSize[1]; 1874022Sstever@eecs.umich.edu w->workGroupSz[2] = ndr->q.wgSize[2]; 1884022Sstever@eecs.umich.edu w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2]; 1897465Ssteve.reinhardt@amd.com w->gridSz[0] = ndr->q.gdSize[0]; 1907465Ssteve.reinhardt@amd.com w->gridSz[1] = ndr->q.gdSize[1]; 1914022Sstever@eecs.umich.edu w->gridSz[2] = ndr->q.gdSize[2]; 1924022Sstever@eecs.umich.edu w->kernelArgs = ndr->q.args; 1934870Sstever@eecs.umich.edu w->privSizePerItem = ndr->q.privMemPerItem; 1944022Sstever@eecs.umich.edu w->spillSizePerItem = ndr->q.spillMemPerItem; 1954022Sstever@eecs.umich.edu w->roBase = ndr->q.roMemStart; 1964022Sstever@eecs.umich.edu w->roSize = ndr->q.roMemTotal; 1974626Sstever@eecs.umich.edu w->computeActualWgSz(ndr); 1986102Sgblack@eecs.umich.edu} 1994870Sstever@eecs.umich.edu 2005314Sstever@gmail.comvoid 2018184Ssomayeh@cs.wisc.eduComputeUnit::updateEvents() { 2024022Sstever@eecs.umich.edu 2035735Snate@binkert.org if (!timestampVec.empty()) { 2045735Snate@binkert.org uint32_t vecSize = timestampVec.size(); 2055735Snate@binkert.org uint32_t i = 0; 2064022Sstever@eecs.umich.edu while (i < vecSize) { 2074022Sstever@eecs.umich.edu if (timestampVec[i] <= shader->tick_cnt) { 2084022Sstever@eecs.umich.edu std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i]; 2095735Snate@binkert.org vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t), 2105735Snate@binkert.org statusVec[i]); 2114022Sstever@eecs.umich.edu timestampVec.erase(timestampVec.begin() + i); 2124022Sstever@eecs.umich.edu regIdxVec.erase(regIdxVec.begin() + i); 2135735Snate@binkert.org statusVec.erase(statusVec.begin() + i); 2145735Snate@binkert.org --vecSize; 2155735Snate@binkert.org --i; 2164022Sstever@eecs.umich.edu } 2175735Snate@binkert.org ++i; 2185735Snate@binkert.org } 2194022Sstever@eecs.umich.edu } 2204022Sstever@eecs.umich.edu 2212381SN/A for (int i = 0; i< numSIMDs; ++i) { 2222662Sstever@eecs.umich.edu vrf[i]->updateEvents(); 2232662Sstever@eecs.umich.edu } 2242662Sstever@eecs.umich.edu} 2252662Sstever@eecs.umich.edu 2262662Sstever@eecs.umich.edu 2272381SN/Avoid 2289044SAli.Saidi@ARM.comComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk, 2292381SN/A NDRange *ndr) 2302813Srdreslin@umich.edu{ 2315735Snate@binkert.org static int _n_wave = 0; 2325735Snate@binkert.org 2334022Sstever@eecs.umich.edu VectorMask init_mask; 2345735Snate@binkert.org init_mask.reset(); 2355735Snate@binkert.org 2365735Snate@binkert.org for (int k = 0; k < wfSize(); ++k) { 2375735Snate@binkert.org if (k + waveId * wfSize() < w->actualWgSzTotal) 2385735Snate@binkert.org init_mask[k] = 1; 2395735Snate@binkert.org } 2405735Snate@binkert.org 2415735Snate@binkert.org w->kernId = ndr->dispatchId; 2425735Snate@binkert.org w->wfId = waveId; 2435735Snate@binkert.org w->initMask = init_mask.to_ullong(); 2445735Snate@binkert.org 2455735Snate@binkert.org for (int k = 0; k < wfSize(); ++k) { 2465735Snate@binkert.org w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0]; 2475735Snate@binkert.org w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) % 2485735Snate@binkert.org w->actualWgSz[1]; 2495735Snate@binkert.org w->workItemId[2][k] = (k + waveId * wfSize()) / 2505735Snate@binkert.org (w->actualWgSz[0] * w->actualWgSz[1]); 2515735Snate@binkert.org 2525735Snate@binkert.org w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] * 2535735Snate@binkert.org w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] + 2545735Snate@binkert.org w->workItemId[0][k]; 2555735Snate@binkert.org } 2565735Snate@binkert.org 2575735Snate@binkert.org w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize()); 2585735Snate@binkert.org 2595735Snate@binkert.org w->barCnt.resize(wfSize(), 0); 2608436SBrad.Beckmann@amd.com 2618436SBrad.Beckmann@amd.com w->maxBarCnt = 0; 2628436SBrad.Beckmann@amd.com w->oldBarrierCnt = 0; 2635735Snate@binkert.org w->barrierCnt = 0; 2645735Snate@binkert.org 2655735Snate@binkert.org w->privBase = ndr->q.privMemStart; 2665735Snate@binkert.org ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize(); 2674022Sstever@eecs.umich.edu 2684022Sstever@eecs.umich.edu w->spillBase = ndr->q.spillMemStart; 2695735Snate@binkert.org ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize(); 2704870Sstever@eecs.umich.edu 2714870Sstever@eecs.umich.edu w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong()); 2725735Snate@binkert.org 2734870Sstever@eecs.umich.edu // WG state 2744870Sstever@eecs.umich.edu w->wgId = ndr->globalWgId; 2752566SN/A w->dispatchId = ndr->dispatchId; 2765735Snate@binkert.org w->workGroupId[0] = w->wgId % ndr->numWg[0]; 2775735Snate@binkert.org w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1]; 2785735Snate@binkert.org w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]); 2795735Snate@binkert.org 2805735Snate@binkert.org w->barrierId = barrier_id; 2815735Snate@binkert.org w->stalledAtBarrier = false; 2822566SN/A 2832566SN/A // set the wavefront context to have a pointer to this section of the LDS 2842566SN/A w->ldsChunk = ldsChunk; 2855735Snate@binkert.org 2865735Snate@binkert.org int32_t refCount M5_VAR_USED = 2872381SN/A lds.increaseRefCounter(w->dispatchId, w->wgId); 2882381SN/A DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n", 2895735Snate@binkert.org cu_id, w->wgId, refCount); 2906227Snate@binkert.org 2912381SN/A w->instructionBuffer.clear(); 2925735Snate@binkert.org 2939030Sandreas.hansson@arm.com if (w->pendingFetch) 2949030Sandreas.hansson@arm.com w->dropFetch = true; 2959030Sandreas.hansson@arm.com 2969030Sandreas.hansson@arm.com // is this the last wavefront in the workgroup 2979030Sandreas.hansson@arm.com // if set the spillWidth to be the remaining work-items 2989030Sandreas.hansson@arm.com // so that the vector access is correct 2999030Sandreas.hansson@arm.com if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) { 3009030Sandreas.hansson@arm.com w->spillWidth = w->actualWgSzTotal - (waveId * wfSize()); 3019030Sandreas.hansson@arm.com } else { 3029030Sandreas.hansson@arm.com w->spillWidth = wfSize(); 3035735Snate@binkert.org } 3049031Sandreas.hansson@arm.com 3052381SN/A DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: " 3065735Snate@binkert.org "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId); 3079030Sandreas.hansson@arm.com 3089030Sandreas.hansson@arm.com w->start(++_n_wave, ndr->q.code_ptr); 3099030Sandreas.hansson@arm.com} 3109030Sandreas.hansson@arm.com 3119030Sandreas.hansson@arm.comvoid 3129030Sandreas.hansson@arm.comComputeUnit::StartWorkgroup(NDRange *ndr) 3135735Snate@binkert.org{ 3149031Sandreas.hansson@arm.com // reserve the LDS capacity allocated to the work group 3152641Sstever@eecs.umich.edu // disambiguated by the dispatch ID and workgroup ID, which should be 3165735Snate@binkert.org // globally unique 3175735Snate@binkert.org LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId, 3184870Sstever@eecs.umich.edu ndr->q.ldsSize); 3194870Sstever@eecs.umich.edu 3204870Sstever@eecs.umich.edu // Send L1 cache acquire 3214870Sstever@eecs.umich.edu // isKernel + isAcquire = Kernel Begin 3224870Sstever@eecs.umich.edu if (shader->impl_kern_boundary_sync) { 3234870Sstever@eecs.umich.edu GPUDynInstPtr gpuDynInst = 3248668Sgeoffrey.blake@arm.com std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst, 3258668Sgeoffrey.blake@arm.com getAndIncSeqNum()); 3268668Sgeoffrey.blake@arm.com 3278668Sgeoffrey.blake@arm.com gpuDynInst->useContinuation = false; 3288668Sgeoffrey.blake@arm.com injectGlobalMemFence(gpuDynInst, true); 3298668Sgeoffrey.blake@arm.com } 3308668Sgeoffrey.blake@arm.com 3312641Sstever@eecs.umich.edu // calculate the number of 32-bit vector registers required by wavefront 3322811Srdreslin@umich.edu int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 3339547Sandreas.hansson@arm.com int wave_id = 0; 3349547Sandreas.hansson@arm.com 3359547Sandreas.hansson@arm.com // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time 3369547Sandreas.hansson@arm.com for (int m = 0; m < shader->n_wf * numSIMDs; ++m) { 3379547Sandreas.hansson@arm.com Wavefront *w = wfList[m % numSIMDs][m / numSIMDs]; 3389547Sandreas.hansson@arm.com // Check if this wavefront slot is available: 3399547Sandreas.hansson@arm.com // It must be stopped and not waiting 3409547Sandreas.hansson@arm.com // for a release to complete S_RETURNING 3413218Sgblack@eecs.umich.edu if (w->status == Wavefront::S_STOPPED) { 3429547Sandreas.hansson@arm.com fillKernelState(w, ndr); 3439547Sandreas.hansson@arm.com // if we have scheduled all work items then stop 3449547Sandreas.hansson@arm.com // scheduling wavefronts 3459547Sandreas.hansson@arm.com if (wave_id * wfSize() >= w->actualWgSzTotal) 3469547Sandreas.hansson@arm.com break; 3479547Sandreas.hansson@arm.com 3489547Sandreas.hansson@arm.com // reserve vector registers for the scheduled wavefront 3499547Sandreas.hansson@arm.com assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd); 3503218Sgblack@eecs.umich.edu uint32_t normSize = 0; 3515735Snate@binkert.org 3525735Snate@binkert.org w->startVgprIndex = vrf[m % numSIMDs]->manager-> 3539542Sandreas.hansson@arm.com allocateRegion(vregDemand, &normSize); 3549542Sandreas.hansson@arm.com 3559542Sandreas.hansson@arm.com w->reservedVectorRegs = normSize; 3569542Sandreas.hansson@arm.com vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs; 3579542Sandreas.hansson@arm.com 3589542Sandreas.hansson@arm.com startWavefront(w, wave_id, ldsChunk, ndr); 3599542Sandreas.hansson@arm.com ++wave_id; 3609542Sandreas.hansson@arm.com } 3619542Sandreas.hansson@arm.com } 3629542Sandreas.hansson@arm.com ++barrier_id; 3639542Sandreas.hansson@arm.com} 3649542Sandreas.hansson@arm.com 3659542Sandreas.hansson@arm.comint 3669542Sandreas.hansson@arm.comComputeUnit::ReadyWorkgroup(NDRange *ndr) 3675735Snate@binkert.org{ 3685735Snate@binkert.org // Get true size of workgroup (after clamping to grid size) 3695735Snate@binkert.org int trueWgSize[3]; 3709542Sandreas.hansson@arm.com int trueWgSizeTotal = 1; 3719542Sandreas.hansson@arm.com 3722641Sstever@eecs.umich.edu for (int d = 0; d < 3; ++d) { 3732641Sstever@eecs.umich.edu trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] - 3742641Sstever@eecs.umich.edu ndr->wgId[d] * ndr->q.wgSize[d]); 3755315Sstever@gmail.com 3765315Sstever@gmail.com trueWgSizeTotal *= trueWgSize[d]; 3775315Sstever@gmail.com DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]); 3785315Sstever@gmail.com } 3799044SAli.Saidi@ARM.com 3805735Snate@binkert.org DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal); 3815735Snate@binkert.org 3825735Snate@binkert.org // calculate the number of 32-bit vector registers required by each 3835735Snate@binkert.org // work item of the work group 3845735Snate@binkert.org int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 3855735Snate@binkert.org bool vregAvail = true; 3865735Snate@binkert.org int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize(); 3875314Sstever@gmail.com int freeWfSlots = 0; 3885314Sstever@gmail.com // check if the total number of VGPRs required by all WFs of the WG 3895314Sstever@gmail.com // fit in the VRFs of all SIMD units 3905735Snate@binkert.org assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd)); 3915314Sstever@gmail.com int numMappedWfs = 0; 3925314Sstever@gmail.com std::vector<int> numWfsPerSimd; 3935314Sstever@gmail.com numWfsPerSimd.resize(numSIMDs, 0); 3945314Sstever@gmail.com // find how many free WF slots we have across all SIMDs 3955314Sstever@gmail.com for (int j = 0; j < shader->n_wf; ++j) { 3965314Sstever@gmail.com for (int i = 0; i < numSIMDs; ++i) { 3975314Sstever@gmail.com if (wfList[i][j]->status == Wavefront::S_STOPPED) { 3985314Sstever@gmail.com // count the number of free WF slots 3995314Sstever@gmail.com ++freeWfSlots; 4005314Sstever@gmail.com if (numMappedWfs < numWfs) { 4015314Sstever@gmail.com // count the WFs to be assigned per SIMD 4025314Sstever@gmail.com numWfsPerSimd[i]++; 4035314Sstever@gmail.com } 4045314Sstever@gmail.com numMappedWfs++; 4055735Snate@binkert.org } 4065735Snate@binkert.org } 4075735Snate@binkert.org } 4085314Sstever@gmail.com 4095315Sstever@gmail.com // if there are enough free WF slots then find if there are enough 4105735Snate@binkert.org // free VGPRs per SIMD based on the WF->SIMD mapping 4115735Snate@binkert.org if (freeWfSlots >= numWfs) { 4125315Sstever@gmail.com for (int j = 0; j < numSIMDs; ++j) { 4135735Snate@binkert.org // find if there are enough free VGPR regions in the SIMD's VRF 4145735Snate@binkert.org // to accommodate the WFs of the new WG that would be mapped to 4155314Sstever@gmail.com // this SIMD unit 4165314Sstever@gmail.com vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j], 4175735Snate@binkert.org vregDemandPerWI); 4185735Snate@binkert.org 4195735Snate@binkert.org // stop searching if there is at least one SIMD 4205735Snate@binkert.org // whose VRF does not have enough free VGPR pools. 4215314Sstever@gmail.com // This is because a WG is scheduled only if ALL 4225735Snate@binkert.org // of its WFs can be scheduled 4235735Snate@binkert.org if (!vregAvail) 4245735Snate@binkert.org break; 4255315Sstever@gmail.com } 4265735Snate@binkert.org } 4275735Snate@binkert.org 4285314Sstever@gmail.com DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n", 4295735Snate@binkert.org freeWfSlots, vregAvail); 4305735Snate@binkert.org 4315735Snate@binkert.org if (!vregAvail) { 4325735Snate@binkert.org ++numTimesWgBlockedDueVgprAlloc; 4335735Snate@binkert.org } 4345314Sstever@gmail.com 4355314Sstever@gmail.com // Return true if enough WF slots to submit workgroup and if there are 4365314Sstever@gmail.com // enough VGPRs to schedule all WFs to their SIMD units 4375735Snate@binkert.org if (!lds.canReserve(ndr->q.ldsSize)) { 4385735Snate@binkert.org wgBlockedDueLdsAllocation++; 4395735Snate@binkert.org } 4405735Snate@binkert.org 4419542Sandreas.hansson@arm.com // Return true if (a) there are enough free WF slots to submit 4425735Snate@binkert.org // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their 4435735Snate@binkert.org // SIMD units and (c) if there is enough space in LDS 4445735Snate@binkert.org return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize); 4452662Sstever@eecs.umich.edu} 4462641Sstever@eecs.umich.edu 4479542Sandreas.hansson@arm.comint 4489542Sandreas.hansson@arm.comComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots) 4499542Sandreas.hansson@arm.com{ 4509542Sandreas.hansson@arm.com DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id); 4519542Sandreas.hansson@arm.com int ccnt = 0; 4529542Sandreas.hansson@arm.com 4539542Sandreas.hansson@arm.com for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) { 4549542Sandreas.hansson@arm.com for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) { 4559542Sandreas.hansson@arm.com Wavefront *w = wfList[i_simd][i_wf]; 4569542Sandreas.hansson@arm.com 4579542Sandreas.hansson@arm.com if (w->status == Wavefront::S_RUNNING) { 4589542Sandreas.hansson@arm.com DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf); 4599542Sandreas.hansson@arm.com 4609542Sandreas.hansson@arm.com DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n", 4619542Sandreas.hansson@arm.com w->barrierId, _barrier_id); 4629542Sandreas.hansson@arm.com 4639542Sandreas.hansson@arm.com DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n", 4649542Sandreas.hansson@arm.com w->barrierCnt, bcnt); 4659542Sandreas.hansson@arm.com } 4669542Sandreas.hansson@arm.com 4679543Ssascha.bischoff@arm.com if (w->status == Wavefront::S_RUNNING && 4689543Ssascha.bischoff@arm.com w->barrierId == _barrier_id && w->barrierCnt == bcnt && 4699543Ssascha.bischoff@arm.com !w->outstandingReqs) { 4709543Ssascha.bischoff@arm.com ++ccnt; 4719543Ssascha.bischoff@arm.com 4729543Ssascha.bischoff@arm.com DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to " 4739543Ssascha.bischoff@arm.com "%d\n", i_simd, i_wf, ccnt); 4749543Ssascha.bischoff@arm.com } 4759543Ssascha.bischoff@arm.com } 4769543Ssascha.bischoff@arm.com } 4779543Ssascha.bischoff@arm.com 4789543Ssascha.bischoff@arm.com DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n", 4799543Ssascha.bischoff@arm.com cu_id, ccnt, bslots); 4809543Ssascha.bischoff@arm.com 4819543Ssascha.bischoff@arm.com return ccnt == bslots; 4829543Ssascha.bischoff@arm.com} 4839543Ssascha.bischoff@arm.com 4849543Ssascha.bischoff@arm.com// Check if the current wavefront is blocked on additional resources. 4859543Ssascha.bischoff@arm.combool 4865735Snate@binkert.orgComputeUnit::cedeSIMD(int simdId, int wfSlotId) 4875735Snate@binkert.org{ 4884022Sstever@eecs.umich.edu bool cede = false; 4892811Srdreslin@umich.edu 4905735Snate@binkert.org // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld 4914022Sstever@eecs.umich.edu // magic instructions will impact the scheduling of wavefronts 4922811Srdreslin@umich.edu if (xact_cas_mode) { 4934022Sstever@eecs.umich.edu /* 4947465Ssteve.reinhardt@amd.com * When a wavefront calls xact_cas_ld, it adds itself to a per address 4957465Ssteve.reinhardt@amd.com * queue. All per address queues are managed by the xactCasLoadMap. 4964022Sstever@eecs.umich.edu * 4974022Sstever@eecs.umich.edu * A wavefront is not blocked if: it is not in ANY per address queue or 4984870Sstever@eecs.umich.edu * if it is at the head of a per address queue. 4994022Sstever@eecs.umich.edu */ 5004022Sstever@eecs.umich.edu for (auto itMap : xactCasLoadMap) { 5014022Sstever@eecs.umich.edu std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue; 5024040Ssaidi@eecs.umich.edu 5036102Sgblack@eecs.umich.edu if (!curWaveIDQueue.empty()) { 5044870Sstever@eecs.umich.edu for (auto it : curWaveIDQueue) { 5055314Sstever@gmail.com waveIdentifier cur_wave = it; 5068184Ssomayeh@cs.wisc.edu 5072812Srdreslin@umich.edu if (cur_wave.simdId == simdId && 5084870Sstever@eecs.umich.edu cur_wave.wfSlotId == wfSlotId) { 5095735Snate@binkert.org // 2 possibilities 5105764Snate@binkert.org // 1: this WF has a green light 5115735Snate@binkert.org // 2: another WF has a green light 5125764Snate@binkert.org waveIdentifier owner_wave = curWaveIDQueue.front(); 5134870Sstever@eecs.umich.edu 5144895Sstever@eecs.umich.edu if (owner_wave.simdId != cur_wave.simdId || 5155735Snate@binkert.org owner_wave.wfSlotId != cur_wave.wfSlotId) { 5165764Snate@binkert.org // possibility 2 5175735Snate@binkert.org cede = true; 5187687Ssteve.reinhardt@amd.com break; 5195764Snate@binkert.org } else { 5208436SBrad.Beckmann@amd.com // possibility 1 5218436SBrad.Beckmann@amd.com break; 5224895Sstever@eecs.umich.edu } 5234870Sstever@eecs.umich.edu } 5244870Sstever@eecs.umich.edu } 5254870Sstever@eecs.umich.edu } 5265735Snate@binkert.org } 5275735Snate@binkert.org } 5285735Snate@binkert.org 5295735Snate@binkert.org return cede; 5305735Snate@binkert.org} 5315735Snate@binkert.org 5325735Snate@binkert.org// Execute one clock worth of work on the ComputeUnit. 5335735Snate@binkert.orgvoid 5344986Ssaidi@eecs.umich.eduComputeUnit::exec() 5352814Srdreslin@umich.edu{ 5369031Sandreas.hansson@arm.com updateEvents(); 5375735Snate@binkert.org // Execute pipeline stages in reverse order to simulate 5389031Sandreas.hansson@arm.com // the pipeline latency 5395735Snate@binkert.org globalMemoryPipe.exec(); 5409031Sandreas.hansson@arm.com localMemoryPipe.exec(); 5415735Snate@binkert.org execStage.exec(); 5429031Sandreas.hansson@arm.com scheduleStage.exec(); 5432641Sstever@eecs.umich.edu scoreboardCheckStage.exec(); 5449031Sandreas.hansson@arm.com fetchStage.exec(); 5455735Snate@binkert.org 5469031Sandreas.hansson@arm.com totalCycles++; 5475735Snate@binkert.org} 5489031Sandreas.hansson@arm.com 5498949Sandreas.hansson@arm.comvoid 5509031Sandreas.hansson@arm.comComputeUnit::init() 5512381SN/A{ 5525764Snate@binkert.org // Initialize CU Bus models 5539259SAli.Saidi@ARM.com glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 5549259SAli.Saidi@ARM.com locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 5559259SAli.Saidi@ARM.com nextGlbMemBus = 0; 5569259SAli.Saidi@ARM.com nextLocMemBus = 0; 5579259SAli.Saidi@ARM.com fatal_if(numGlbMemUnits > 1, 5589259SAli.Saidi@ARM.com "No support for multiple Global Memory Pipelines exists!!!"); 5599259SAli.Saidi@ARM.com vrfToGlobalMemPipeBus.resize(numGlbMemUnits); 5609259SAli.Saidi@ARM.com for (int j = 0; j < numGlbMemUnits; ++j) { 5619259SAli.Saidi@ARM.com vrfToGlobalMemPipeBus[j] = WaitClass(); 5626227Snate@binkert.org vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 5635735Snate@binkert.org } 5642549SN/A 5655735Snate@binkert.org fatal_if(numLocMemUnits > 1, 5667550SBrad.Beckmann@amd.com "No support for multiple Local Memory Pipelines exists!!!"); 5677550SBrad.Beckmann@amd.com vrfToLocalMemPipeBus.resize(numLocMemUnits); 5687550SBrad.Beckmann@amd.com for (int j = 0; j < numLocMemUnits; ++j) { 5697550SBrad.Beckmann@amd.com vrfToLocalMemPipeBus[j] = WaitClass(); 5707550SBrad.Beckmann@amd.com vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 5717550SBrad.Beckmann@amd.com } 5727550SBrad.Beckmann@amd.com vectorRegsReserved.resize(numSIMDs, 0); 5737550SBrad.Beckmann@amd.com aluPipe.resize(numSIMDs); 5747550SBrad.Beckmann@amd.com wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits); 5757550SBrad.Beckmann@amd.com 5767550SBrad.Beckmann@amd.com for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) { 5777550SBrad.Beckmann@amd.com wfWait[i] = WaitClass(); 5787550SBrad.Beckmann@amd.com wfWait[i].init(&shader->tick_cnt, shader->ticks(1)); 5797550SBrad.Beckmann@amd.com } 5807550SBrad.Beckmann@amd.com 5817550SBrad.Beckmann@amd.com for (int i = 0; i < numSIMDs; ++i) { 5827550SBrad.Beckmann@amd.com aluPipe[i] = WaitClass(); 5837550SBrad.Beckmann@amd.com aluPipe[i].init(&shader->tick_cnt, shader->ticks(1)); 5847550SBrad.Beckmann@amd.com } 5857550SBrad.Beckmann@amd.com 5867550SBrad.Beckmann@amd.com // Setup space for call args 5877550SBrad.Beckmann@amd.com for (int j = 0; j < numSIMDs; ++j) { 5887550SBrad.Beckmann@amd.com for (int i = 0; i < shader->n_wf; ++i) { 5897550SBrad.Beckmann@amd.com wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize); 5905735Snate@binkert.org } 5915735Snate@binkert.org } 5929030Sandreas.hansson@arm.com 5935735Snate@binkert.org // Initializing pipeline resources 5948949Sandreas.hansson@arm.com readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits); 5958949Sandreas.hansson@arm.com waveStatusList.resize(numSIMDs); 5969031Sandreas.hansson@arm.com 5978949Sandreas.hansson@arm.com for (int j = 0; j < numSIMDs; ++j) { 5989547Sandreas.hansson@arm.com for (int i = 0; i < shader->n_wf; ++i) { 5999546Sandreas.hansson@arm.com waveStatusList[j].push_back( 6002641Sstever@eecs.umich.edu std::make_pair(wfList[j][i], BLOCKED)); 6016104Ssteve.reinhardt@amd.com } 6026104Ssteve.reinhardt@amd.com } 6036104Ssteve.reinhardt@amd.com 6046104Ssteve.reinhardt@amd.com for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) { 6056104Ssteve.reinhardt@amd.com dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY)); 6066104Ssteve.reinhardt@amd.com } 6076104Ssteve.reinhardt@amd.com 6086104Ssteve.reinhardt@amd.com fetchStage.init(this); 6092813Srdreslin@umich.edu scoreboardCheckStage.init(this); 6102813Srdreslin@umich.edu scheduleStage.init(this); 6115735Snate@binkert.org execStage.init(this); 6125735Snate@binkert.org globalMemoryPipe.init(this); 6135735Snate@binkert.org localMemoryPipe.init(this); 6145735Snate@binkert.org // initialize state for statistics calculation 6155735Snate@binkert.org vectorAluInstAvail.resize(numSIMDs, false); 6168949Sandreas.hansson@arm.com shrMemInstAvail = 0; 6178949Sandreas.hansson@arm.com glbMemInstAvail = 0; 6189031Sandreas.hansson@arm.com} 6198949Sandreas.hansson@arm.com 6209547Sandreas.hansson@arm.combool 6219546Sandreas.hansson@arm.comComputeUnit::DataPort::recvTimingResp(PacketPtr pkt) 6222813Srdreslin@umich.edu{ 6236104Ssteve.reinhardt@amd.com // Ruby has completed the memory op. Schedule the mem_resp_event at the 6246104Ssteve.reinhardt@amd.com // appropriate cycle to process the timing memory response 6256104Ssteve.reinhardt@amd.com // This delay represents the pipeline delay 6266104Ssteve.reinhardt@amd.com SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 6276104Ssteve.reinhardt@amd.com int index = sender_state->port_index; 6286104Ssteve.reinhardt@amd.com GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 6294626Sstever@eecs.umich.edu 6304626Sstever@eecs.umich.edu // Is the packet returned a Kernel End or Barrier 6315735Snate@binkert.org if (pkt->req->isKernel() && pkt->req->isRelease()) { 6325735Snate@binkert.org Wavefront *w = 6334887Sstever@eecs.umich.edu computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId]; 6344887Sstever@eecs.umich.edu 6354887Sstever@eecs.umich.edu // Check if we are waiting on Kernel End Release 6365735Snate@binkert.org if (w->status == Wavefront::S_RETURNING) { 6375735Snate@binkert.org DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n", 6385735Snate@binkert.org computeUnit->cu_id, w->simdId, w->wfSlotId, 6395735Snate@binkert.org w->wfDynId, w->kernId); 6405764Snate@binkert.org 6415735Snate@binkert.org computeUnit->shader->dispatcher->notifyWgCompl(w); 6429547Sandreas.hansson@arm.com w->status = Wavefront::S_STOPPED; 6439547Sandreas.hansson@arm.com } else { 6449547Sandreas.hansson@arm.com w->outstandingReqs--; 6459547Sandreas.hansson@arm.com } 6469546Sandreas.hansson@arm.com 6474626Sstever@eecs.umich.edu DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n", 6485735Snate@binkert.org computeUnit->cu_id, gpuDynInst->simdId, 6495735Snate@binkert.org gpuDynInst->wfSlotId, w->barrierCnt); 6505735Snate@binkert.org 6519031Sandreas.hansson@arm.com if (gpuDynInst->useContinuation) { 6525735Snate@binkert.org assert(!gpuDynInst->isNoScope()); 6538668Sgeoffrey.blake@arm.com gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 6542641Sstever@eecs.umich.edu gpuDynInst); 6552549SN/A } 6565735Snate@binkert.org 6575735Snate@binkert.org delete pkt->senderState; 6585735Snate@binkert.org delete pkt; 6592566SN/A return true; 6605387Sstever@gmail.com } else if (pkt->req->isKernel() && pkt->req->isAcquire()) { 6615387Sstever@gmail.com if (gpuDynInst->useContinuation) { 6625387Sstever@gmail.com assert(!gpuDynInst->isNoScope()); 6635387Sstever@gmail.com gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 6645387Sstever@gmail.com gpuDynInst); 6655387Sstever@gmail.com } 6665735Snate@binkert.org 6675387Sstever@gmail.com delete pkt->senderState; 6682566SN/A delete pkt; 6695735Snate@binkert.org return true; 6705735Snate@binkert.org } 6715735Snate@binkert.org 6725735Snate@binkert.org EventFunctionWrapper *mem_resp_event = 6735735Snate@binkert.org computeUnit->memPort[index]->createMemRespEvent(pkt); 6745735Snate@binkert.org 6755735Snate@binkert.org DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n", 6765735Snate@binkert.org computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 6775735Snate@binkert.org index, pkt->req->getPaddr()); 6785735Snate@binkert.org 6796104Ssteve.reinhardt@amd.com computeUnit->schedule(mem_resp_event, 6804870Sstever@eecs.umich.edu curTick() + computeUnit->resp_tick_latency); 6816104Ssteve.reinhardt@amd.com return true; 6826104Ssteve.reinhardt@amd.com} 6835735Snate@binkert.org 6849547Sandreas.hansson@arm.comvoid 6859547Sandreas.hansson@arm.comComputeUnit::DataPort::recvReqRetry() 6869547Sandreas.hansson@arm.com{ 6879547Sandreas.hansson@arm.com int len = retries.size(); 6889547Sandreas.hansson@arm.com 6899547Sandreas.hansson@arm.com assert(len > 0); 6909547Sandreas.hansson@arm.com 6915735Snate@binkert.org for (int i = 0; i < len; ++i) { 6925735Snate@binkert.org PacketPtr pkt = retries.front().first; 6932662Sstever@eecs.umich.edu GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second; 6942566SN/A DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n", 6954626Sstever@eecs.umich.edu computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 6964626Sstever@eecs.umich.edu pkt->req->getPaddr()); 6979030Sandreas.hansson@arm.com 6989030Sandreas.hansson@arm.com /** Currently Ruby can return false due to conflicts for the particular 6999030Sandreas.hansson@arm.com * cache block or address. Thus other requests should be allowed to 7009030Sandreas.hansson@arm.com * pass and the data port should expect multiple retries. */ 7014626Sstever@eecs.umich.edu if (!sendTimingReq(pkt)) { 7025735Snate@binkert.org DPRINTF(GPUMem, "failed again!\n"); 7035735Snate@binkert.org break; 7044626Sstever@eecs.umich.edu } else { 7052662Sstever@eecs.umich.edu DPRINTF(GPUMem, "successful!\n"); 7062855Srdreslin@umich.edu retries.pop_front(); 7074986Ssaidi@eecs.umich.edu } 7084022Sstever@eecs.umich.edu } 7095745Snate@binkert.org} 7107464Ssteve.reinhardt@amd.com 7117464Ssteve.reinhardt@amd.combool 7127464Ssteve.reinhardt@amd.comComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt) 7137464Ssteve.reinhardt@amd.com{ 7145745Snate@binkert.org computeUnit->fetchStage.processFetchReturn(pkt); 7159031Sandreas.hansson@arm.com 7162641Sstever@eecs.umich.edu return true; 7172641Sstever@eecs.umich.edu} 7185735Snate@binkert.org 7195735Snate@binkert.orgvoid 7204870Sstever@eecs.umich.eduComputeUnit::SQCPort::recvReqRetry() 7214870Sstever@eecs.umich.edu{ 7224870Sstever@eecs.umich.edu int len = retries.size(); 7234870Sstever@eecs.umich.edu 7245735Snate@binkert.org assert(len > 0); 7255735Snate@binkert.org 7263348Sbinkertn@umich.edu for (int i = 0; i < len; ++i) { 7274870Sstever@eecs.umich.edu PacketPtr pkt = retries.front().first; 7283135Srdreslin@umich.edu Wavefront *wavefront M5_VAR_USED = retries.front().second; 7293135Srdreslin@umich.edu DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n", 7308436SBrad.Beckmann@amd.com computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId, 7318436SBrad.Beckmann@amd.com pkt->req->getPaddr()); 7328436SBrad.Beckmann@amd.com if (!sendTimingReq(pkt)) { 7338436SBrad.Beckmann@amd.com DPRINTF(GPUFetch, "failed again!\n"); 7348436SBrad.Beckmann@amd.com break; 7358436SBrad.Beckmann@amd.com } else { 7368436SBrad.Beckmann@amd.com DPRINTF(GPUFetch, "successful!\n"); 7378436SBrad.Beckmann@amd.com retries.pop_front(); 7388436SBrad.Beckmann@amd.com } 7398436SBrad.Beckmann@amd.com } 7408436SBrad.Beckmann@amd.com} 7418436SBrad.Beckmann@amd.com 7427006Snate@binkert.orgvoid 7437006Snate@binkert.orgComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 7447006Snate@binkert.org{ 7457006Snate@binkert.org // There must be a way around this check to do the globalMemStart... 7467006Snate@binkert.org Addr tmp_vaddr = pkt->req->getVaddr(); 7477006Snate@binkert.org 7487006Snate@binkert.org updatePageDivergenceDist(tmp_vaddr); 7497006Snate@binkert.org 7507006Snate@binkert.org pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(), 7512685Ssaidi@eecs.umich.edu pkt->req->getFlags(), pkt->req->masterId(), 7523348Sbinkertn@umich.edu pkt->req->getPC()); 7533348Sbinkertn@umich.edu 7543348Sbinkertn@umich.edu // figure out the type of the request to set read/write 7552566SN/A BaseTLB::Mode TLB_mode; 7562566SN/A assert(pkt->isRead() || pkt->isWrite()); 7573348Sbinkertn@umich.edu 7583348Sbinkertn@umich.edu // Check write before read for atomic operations 7593348Sbinkertn@umich.edu // since atomic operations should use BaseTLB::Write 7605764Snate@binkert.org if (pkt->isWrite()){ 7613348Sbinkertn@umich.edu TLB_mode = BaseTLB::Write; 7625735Snate@binkert.org } else if (pkt->isRead()) { 7633348Sbinkertn@umich.edu TLB_mode = BaseTLB::Read; 7642566SN/A } else { 7653348Sbinkertn@umich.edu fatal("pkt is not a read nor a write\n"); 7663348Sbinkertn@umich.edu } 7673348Sbinkertn@umich.edu 7683348Sbinkertn@umich.edu tlbCycles -= curTick(); 7692566SN/A ++tlbRequests; 7703348Sbinkertn@umich.edu 7713348Sbinkertn@umich.edu int tlbPort_index = perLaneTLB ? index : 0; 7723348Sbinkertn@umich.edu 7735764Snate@binkert.org if (shader->timingSim) { 7743348Sbinkertn@umich.edu if (debugSegFault) { 7755735Snate@binkert.org Process *p = shader->gpuTc->getProcessPtr(); 7763348Sbinkertn@umich.edu Addr vaddr = pkt->req->getVaddr(); 7773348Sbinkertn@umich.edu unsigned size = pkt->getSize(); 7783348Sbinkertn@umich.edu 7793348Sbinkertn@umich.edu if ((vaddr + size - 1) % 64 < vaddr % 64) { 7803348Sbinkertn@umich.edu panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n", 7813348Sbinkertn@umich.edu cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr); 7823348Sbinkertn@umich.edu } 7833348Sbinkertn@umich.edu 7843348Sbinkertn@umich.edu Addr paddr; 7853348Sbinkertn@umich.edu 7865764Snate@binkert.org if (!p->pTable->translate(vaddr, paddr)) { 7873348Sbinkertn@umich.edu if (!p->fixupStackFault(vaddr)) { 7885735Snate@binkert.org panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n", 7893348Sbinkertn@umich.edu cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 7903348Sbinkertn@umich.edu vaddr); 7915735Snate@binkert.org } 7925735Snate@binkert.org } 7935735Snate@binkert.org } 7943348Sbinkertn@umich.edu 7953348Sbinkertn@umich.edu // This is the SenderState needed upon return 7967915SBrad.Beckmann@amd.com pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index); 7973348Sbinkertn@umich.edu 7987915SBrad.Beckmann@amd.com // This is the senderState needed by the TLB hierarchy to function 7993348Sbinkertn@umich.edu TheISA::GpuTLB::TranslationState *translation_state = 8003348Sbinkertn@umich.edu new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false, 8012566SN/A pkt->senderState); 8025735Snate@binkert.org 8035735Snate@binkert.org pkt->senderState = translation_state; 8045735Snate@binkert.org 8052566SN/A if (functionalTLB) { 8062592SN/A tlbPort[tlbPort_index]->sendFunctional(pkt); 8072566SN/A 8085735Snate@binkert.org // update the hitLevel distribution 8095735Snate@binkert.org int hit_level = translation_state->hitLevel; 8105735Snate@binkert.org assert(hit_level != -1); 8112566SN/A hitsPerTLBLevel[hit_level]++; 8122592SN/A 8132566SN/A // New SenderState for the memory access 8143348Sbinkertn@umich.edu X86ISA::GpuTLB::TranslationState *sender_state = 8154626Sstever@eecs.umich.edu safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState); 8164626Sstever@eecs.umich.edu 8175735Snate@binkert.org delete sender_state->tlbEntry; 8185735Snate@binkert.org delete sender_state->saved; 8194626Sstever@eecs.umich.edu delete sender_state; 8207691SAli.Saidi@ARM.com 8217691SAli.Saidi@ARM.com assert(pkt->req->hasPaddr()); 8224626Sstever@eecs.umich.edu assert(pkt->req->hasSize()); 8234626Sstever@eecs.umich.edu 8244626Sstever@eecs.umich.edu uint8_t *tmpData = pkt->getPtr<uint8_t>(); 8254626Sstever@eecs.umich.edu 8264626Sstever@eecs.umich.edu // this is necessary because the GPU TLB receives packets instead 8274626Sstever@eecs.umich.edu // of requests. when the translation is complete, all relevent 8285735Snate@binkert.org // fields in the request will be populated, but not in the packet. 8295735Snate@binkert.org // here we create the new packet so we can set the size, addr, 8304626Sstever@eecs.umich.edu // and proper flags. 8314626Sstever@eecs.umich.edu PacketPtr oldPkt = pkt; 8324626Sstever@eecs.umich.edu pkt = new Packet(oldPkt->req, oldPkt->cmd); 8334626Sstever@eecs.umich.edu delete oldPkt; 8344626Sstever@eecs.umich.edu pkt->dataStatic(tmpData); 8354626Sstever@eecs.umich.edu 8364626Sstever@eecs.umich.edu 8374626Sstever@eecs.umich.edu // New SenderState for the memory access 8385735Snate@binkert.org pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, 8395735Snate@binkert.org index, nullptr); 8404626Sstever@eecs.umich.edu 8414626Sstever@eecs.umich.edu gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index); 8424626Sstever@eecs.umich.edu gpuDynInst->tlbHitLevel[index] = hit_level; 8434626Sstever@eecs.umich.edu 8444626Sstever@eecs.umich.edu 8454626Sstever@eecs.umich.edu // translation is done. Schedule the mem_req_event at the 8464626Sstever@eecs.umich.edu // appropriate cycle to send the timing memory request to ruby 8475735Snate@binkert.org EventFunctionWrapper *mem_req_event = 8485735Snate@binkert.org memPort[index]->createMemReqEvent(pkt); 8494626Sstever@eecs.umich.edu 8504626Sstever@eecs.umich.edu DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data " 8514626Sstever@eecs.umich.edu "scheduled\n", cu_id, gpuDynInst->simdId, 8524626Sstever@eecs.umich.edu gpuDynInst->wfSlotId, index, pkt->req->getPaddr()); 8534626Sstever@eecs.umich.edu 8543348Sbinkertn@umich.edu schedule(mem_req_event, curTick() + req_tick_latency); 8553348Sbinkertn@umich.edu } else if (tlbPort[tlbPort_index]->isStalled()) { 8563348Sbinkertn@umich.edu assert(tlbPort[tlbPort_index]->retries.size() > 0); 8575735Snate@binkert.org 8585735Snate@binkert.org DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 8595735Snate@binkert.org "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 8605764Snate@binkert.org tmp_vaddr); 8615735Snate@binkert.org 8625764Snate@binkert.org tlbPort[tlbPort_index]->retries.push_back(pkt); 8635735Snate@binkert.org } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) { 8645735Snate@binkert.org // Stall the data port; 8655735Snate@binkert.org // No more packet will be issued till 8665735Snate@binkert.org // ruby indicates resources are freed by 8675735Snate@binkert.org // a recvReqRetry() call back on this port. 8682566SN/A tlbPort[tlbPort_index]->stallPort(); 8692566SN/A 8705735Snate@binkert.org DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 8715735Snate@binkert.org "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 8725735Snate@binkert.org tmp_vaddr); 8735735Snate@binkert.org 8745764Snate@binkert.org tlbPort[tlbPort_index]->retries.push_back(pkt); 8755745Snate@binkert.org } else { 8765735Snate@binkert.org DPRINTF(GPUTLB, 8775745Snate@binkert.org "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n", 8785764Snate@binkert.org cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr); 8795745Snate@binkert.org } 8805745Snate@binkert.org } else { 8815735Snate@binkert.org if (pkt->cmd == MemCmd::MemFenceReq) { 8825735Snate@binkert.org gpuDynInst->statusBitVector = VectorMask(0); 8834626Sstever@eecs.umich.edu } else { 8844626Sstever@eecs.umich.edu gpuDynInst->statusBitVector &= (~(1ll << index)); 8854626Sstever@eecs.umich.edu } 8864626Sstever@eecs.umich.edu 8874626Sstever@eecs.umich.edu // New SenderState for the memory access 8884626Sstever@eecs.umich.edu delete pkt->senderState; 8894626Sstever@eecs.umich.edu 8905314Sstever@gmail.com // Because it's atomic operation, only need TLB translation state 8914626Sstever@eecs.umich.edu pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode, 8924626Sstever@eecs.umich.edu shader->gpuTc); 8934626Sstever@eecs.umich.edu 8945315Sstever@gmail.com tlbPort[tlbPort_index]->sendFunctional(pkt); 8954626Sstever@eecs.umich.edu 8965735Snate@binkert.org // the addr of the packet is not modified, so we need to create a new 8975735Snate@binkert.org // packet, or otherwise the memory access will have the old virtual 8985735Snate@binkert.org // address sent in the translation packet, instead of the physical 8995735Snate@binkert.org // address returned by the translation. 9005735Snate@binkert.org PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd); 9015735Snate@binkert.org new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 9024626Sstever@eecs.umich.edu 9035314Sstever@gmail.com // Translation is done. It is safe to send the packet to memory. 9045315Sstever@gmail.com memPort[0]->sendFunctional(new_pkt); 9055315Sstever@gmail.com 9065315Sstever@gmail.com DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id, 9075735Snate@binkert.org gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 9085735Snate@binkert.org new_pkt->req->getPaddr()); 9095735Snate@binkert.org 9105735Snate@binkert.org // safe_cast the senderState 9115735Snate@binkert.org TheISA::GpuTLB::TranslationState *sender_state = 9125314Sstever@gmail.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 9135314Sstever@gmail.com 9145315Sstever@gmail.com delete sender_state->tlbEntry; 9155315Sstever@gmail.com delete new_pkt; 9165315Sstever@gmail.com delete pkt->senderState; 9175735Snate@binkert.org delete pkt; 9185735Snate@binkert.org } 9195735Snate@binkert.org} 9205735Snate@binkert.org 9215735Snate@binkert.orgvoid 9225314Sstever@gmail.comComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 9235314Sstever@gmail.com{ 9245314Sstever@gmail.com EventFunctionWrapper *mem_req_event = 9255314Sstever@gmail.com memPort[index]->createMemReqEvent(pkt); 9262381SN/A 9272381SN/A 9282381SN/A // New SenderState for the memory access 929 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index, 930 nullptr); 931 932 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n", 933 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 934 pkt->req->getPaddr()); 935 936 schedule(mem_req_event, curTick() + req_tick_latency); 937} 938 939void 940ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch, 941 RequestPtr req) 942{ 943 assert(gpuDynInst->isGlobalSeg()); 944 945 if (!req) { 946 req = std::make_shared<Request>( 947 0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId); 948 } 949 req->setPaddr(0); 950 if (kernelLaunch) { 951 req->setFlags(Request::KERNEL); 952 } 953 954 // for non-kernel MemFence operations, memorder flags are set depending 955 // on which type of request is currently being sent, so this 956 // should be set by the caller (e.g. if an inst has acq-rel 957 // semantics, it will send one acquire req an one release req) 958 gpuDynInst->setRequestFlags(req, kernelLaunch); 959 960 // a mem fence must correspond to an acquire/release request 961 assert(req->isAcquire() || req->isRelease()); 962 963 // create packet 964 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq); 965 966 // set packet's sender state 967 pkt->senderState = 968 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr); 969 970 // send the packet 971 sendSyncRequest(gpuDynInst, 0, pkt); 972} 973 974void 975ComputeUnit::DataPort::processMemRespEvent(PacketPtr pkt) 976{ 977 DataPort::SenderState *sender_state = 978 safe_cast<DataPort::SenderState*>(pkt->senderState); 979 980 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 981 ComputeUnit *compute_unit = computeUnit; 982 983 assert(gpuDynInst); 984 985 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n", 986 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 987 pkt->req->getPaddr(), index); 988 989 Addr paddr = pkt->req->getPaddr(); 990 991 if (pkt->cmd != MemCmd::MemFenceResp) { 992 int index = gpuDynInst->memStatusVector[paddr].back(); 993 994 DPRINTF(GPUMem, "Response for addr %#x, index %d\n", 995 pkt->req->getPaddr(), index); 996 997 gpuDynInst->memStatusVector[paddr].pop_back(); 998 gpuDynInst->pAddr = pkt->req->getPaddr(); 999 1000 if (pkt->isRead() || pkt->isWrite()) { 1001 1002 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) { 1003 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1004 } else { 1005 assert(gpuDynInst->statusVector[index] > 0); 1006 gpuDynInst->statusVector[index]--; 1007 1008 if (!gpuDynInst->statusVector[index]) 1009 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1010 } 1011 1012 DPRINTF(GPUMem, "bitvector is now %#x\n", 1013 gpuDynInst->statusBitVector); 1014 1015 if (gpuDynInst->statusBitVector == VectorMask(0)) { 1016 auto iter = gpuDynInst->memStatusVector.begin(); 1017 auto end = gpuDynInst->memStatusVector.end(); 1018 1019 while (iter != end) { 1020 assert(iter->second.empty()); 1021 ++iter; 1022 } 1023 1024 gpuDynInst->memStatusVector.clear(); 1025 1026 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST) 1027 gpuDynInst->statusVector.clear(); 1028 1029 compute_unit->globalMemoryPipe.handleResponse(gpuDynInst); 1030 1031 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n", 1032 compute_unit->cu_id, gpuDynInst->simdId, 1033 gpuDynInst->wfSlotId); 1034 1035 // after clearing the status vectors, 1036 // see if there is a continuation to perform 1037 // the continuation may generate more work for 1038 // this memory request 1039 if (gpuDynInst->useContinuation) { 1040 assert(!gpuDynInst->isNoScope()); 1041 gpuDynInst->execContinuation( 1042 gpuDynInst->staticInstruction(), 1043 gpuDynInst); 1044 } 1045 } 1046 } 1047 } else { 1048 gpuDynInst->statusBitVector = VectorMask(0); 1049 1050 if (gpuDynInst->useContinuation) { 1051 assert(!gpuDynInst->isNoScope()); 1052 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1053 gpuDynInst); 1054 } 1055 } 1056 1057 delete pkt->senderState; 1058 delete pkt; 1059} 1060 1061ComputeUnit* 1062ComputeUnitParams::create() 1063{ 1064 return new ComputeUnit(this); 1065} 1066 1067bool 1068ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt) 1069{ 1070 Addr line = pkt->req->getPaddr(); 1071 1072 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id, 1073 pkt->req->getVaddr(), line); 1074 1075 assert(pkt->senderState); 1076 computeUnit->tlbCycles += curTick(); 1077 1078 // pop off the TLB translation state 1079 TheISA::GpuTLB::TranslationState *translation_state = 1080 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1081 1082 // no PageFaults are permitted for data accesses 1083 if (!translation_state->tlbEntry) { 1084 DTLBPort::SenderState *sender_state = 1085 safe_cast<DTLBPort::SenderState*>(translation_state->saved); 1086 1087 Wavefront *w M5_VAR_USED = 1088 computeUnit->wfList[sender_state->_gpuDynInst->simdId] 1089 [sender_state->_gpuDynInst->wfSlotId]; 1090 1091 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId, 1092 pkt->req->getVaddr()); 1093 } 1094 1095 // update the hitLevel distribution 1096 int hit_level = translation_state->hitLevel; 1097 computeUnit->hitsPerTLBLevel[hit_level]++; 1098 1099 delete translation_state->tlbEntry; 1100 assert(!translation_state->ports.size()); 1101 pkt->senderState = translation_state->saved; 1102 1103 // for prefetch pkt 1104 BaseTLB::Mode TLB_mode = translation_state->tlbMode; 1105 1106 delete translation_state; 1107 1108 // use the original sender state to know how to close this transaction 1109 DTLBPort::SenderState *sender_state = 1110 safe_cast<DTLBPort::SenderState*>(pkt->senderState); 1111 1112 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1113 int mp_index = sender_state->portIndex; 1114 Addr vaddr = pkt->req->getVaddr(); 1115 gpuDynInst->memStatusVector[line].push_back(mp_index); 1116 gpuDynInst->tlbHitLevel[mp_index] = hit_level; 1117 1118 MemCmd requestCmd; 1119 1120 if (pkt->cmd == MemCmd::ReadResp) { 1121 requestCmd = MemCmd::ReadReq; 1122 } else if (pkt->cmd == MemCmd::WriteResp) { 1123 requestCmd = MemCmd::WriteReq; 1124 } else if (pkt->cmd == MemCmd::SwapResp) { 1125 requestCmd = MemCmd::SwapReq; 1126 } else { 1127 panic("unsupported response to request conversion %s\n", 1128 pkt->cmd.toString()); 1129 } 1130 1131 if (computeUnit->prefetchDepth) { 1132 int simdId = gpuDynInst->simdId; 1133 int wfSlotId = gpuDynInst->wfSlotId; 1134 Addr last = 0; 1135 1136 switch(computeUnit->prefetchType) { 1137 case Enums::PF_CU: 1138 last = computeUnit->lastVaddrCU[mp_index]; 1139 break; 1140 case Enums::PF_PHASE: 1141 last = computeUnit->lastVaddrSimd[simdId][mp_index]; 1142 break; 1143 case Enums::PF_WF: 1144 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index]; 1145 default: 1146 break; 1147 } 1148 1149 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n", 1150 computeUnit->cu_id, simdId, wfSlotId, mp_index, last); 1151 1152 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) - 1153 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift 1154 : 0; 1155 1156 DPRINTF(GPUPrefetch, "Stride is %d\n", stride); 1157 1158 computeUnit->lastVaddrCU[mp_index] = vaddr; 1159 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr; 1160 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr; 1161 1162 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ? 1163 computeUnit->prefetchStride: stride; 1164 1165 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr, 1166 computeUnit->cu_id, simdId, wfSlotId, mp_index); 1167 1168 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr); 1169 1170 // Prefetch Next few pages atomically 1171 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) { 1172 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride, 1173 vaddr+stride*pf*TheISA::PageBytes); 1174 1175 if (!stride) 1176 break; 1177 1178 RequestPtr prefetch_req = std::make_shared<Request>( 1179 0, vaddr + stride * pf * TheISA::PageBytes, 1180 sizeof(uint8_t), 0, 1181 computeUnit->masterId(), 1182 0, 0, nullptr); 1183 1184 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd); 1185 uint8_t foo = 0; 1186 prefetch_pkt->dataStatic(&foo); 1187 1188 // Because it's atomic operation, only need TLB translation state 1189 prefetch_pkt->senderState = 1190 new TheISA::GpuTLB::TranslationState(TLB_mode, 1191 computeUnit->shader->gpuTc, 1192 true); 1193 1194 // Currently prefetches are zero-latency, hence the sendFunctional 1195 sendFunctional(prefetch_pkt); 1196 1197 /* safe_cast the senderState */ 1198 TheISA::GpuTLB::TranslationState *tlb_state = 1199 safe_cast<TheISA::GpuTLB::TranslationState*>( 1200 prefetch_pkt->senderState); 1201 1202 1203 delete tlb_state->tlbEntry; 1204 delete tlb_state; 1205 delete prefetch_pkt; 1206 } 1207 } 1208 1209 // First we must convert the response cmd back to a request cmd so that 1210 // the request can be sent through the cu's master port 1211 PacketPtr new_pkt = new Packet(pkt->req, requestCmd); 1212 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 1213 delete pkt->senderState; 1214 delete pkt; 1215 1216 // New SenderState for the memory access 1217 new_pkt->senderState = 1218 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index, 1219 nullptr); 1220 1221 // translation is done. Schedule the mem_req_event at the appropriate 1222 // cycle to send the timing memory request to ruby 1223 EventFunctionWrapper *mem_req_event = 1224 computeUnit->memPort[mp_index]->createMemReqEvent(new_pkt); 1225 1226 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n", 1227 computeUnit->cu_id, gpuDynInst->simdId, 1228 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr()); 1229 1230 computeUnit->schedule(mem_req_event, curTick() + 1231 computeUnit->req_tick_latency); 1232 1233 return true; 1234} 1235 1236EventFunctionWrapper* 1237ComputeUnit::DataPort::createMemReqEvent(PacketPtr pkt) 1238{ 1239 return new EventFunctionWrapper( 1240 [this, pkt]{ processMemReqEvent(pkt); }, 1241 "ComputeUnit memory request event", true); 1242} 1243 1244EventFunctionWrapper* 1245ComputeUnit::DataPort::createMemRespEvent(PacketPtr pkt) 1246{ 1247 return new EventFunctionWrapper( 1248 [this, pkt]{ processMemRespEvent(pkt); }, 1249 "ComputeUnit memory response event", true); 1250} 1251 1252void 1253ComputeUnit::DataPort::processMemReqEvent(PacketPtr pkt) 1254{ 1255 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 1256 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1257 ComputeUnit *compute_unit M5_VAR_USED = computeUnit; 1258 1259 if (!(sendTimingReq(pkt))) { 1260 retries.push_back(std::make_pair(pkt, gpuDynInst)); 1261 1262 DPRINTF(GPUPort, 1263 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n", 1264 compute_unit->cu_id, gpuDynInst->simdId, 1265 gpuDynInst->wfSlotId, index, 1266 pkt->req->getPaddr()); 1267 } else { 1268 DPRINTF(GPUPort, 1269 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n", 1270 compute_unit->cu_id, gpuDynInst->simdId, 1271 gpuDynInst->wfSlotId, index, 1272 pkt->req->getPaddr()); 1273 } 1274} 1275 1276/* 1277 * The initial translation request could have been rejected, 1278 * if <retries> queue is not Retry sending the translation 1279 * request. sendRetry() is called from the peer port whenever 1280 * a translation completes. 1281 */ 1282void 1283ComputeUnit::DTLBPort::recvReqRetry() 1284{ 1285 int len = retries.size(); 1286 1287 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n", 1288 computeUnit->cu_id, len); 1289 1290 assert(len > 0); 1291 assert(isStalled()); 1292 // recvReqRetry is an indication that the resource on which this 1293 // port was stalling on is freed. So, remove the stall first 1294 unstallPort(); 1295 1296 for (int i = 0; i < len; ++i) { 1297 PacketPtr pkt = retries.front(); 1298 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1299 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr); 1300 1301 if (!sendTimingReq(pkt)) { 1302 // Stall port 1303 stallPort(); 1304 DPRINTF(GPUTLB, ": failed again\n"); 1305 break; 1306 } else { 1307 DPRINTF(GPUTLB, ": successful\n"); 1308 retries.pop_front(); 1309 } 1310 } 1311} 1312 1313bool 1314ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt) 1315{ 1316 Addr line M5_VAR_USED = pkt->req->getPaddr(); 1317 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n", 1318 computeUnit->cu_id, pkt->req->getVaddr(), line); 1319 1320 assert(pkt->senderState); 1321 1322 // pop off the TLB translation state 1323 TheISA::GpuTLB::TranslationState *translation_state = 1324 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1325 1326 bool success = translation_state->tlbEntry != nullptr; 1327 delete translation_state->tlbEntry; 1328 assert(!translation_state->ports.size()); 1329 pkt->senderState = translation_state->saved; 1330 delete translation_state; 1331 1332 // use the original sender state to know how to close this transaction 1333 ITLBPort::SenderState *sender_state = 1334 safe_cast<ITLBPort::SenderState*>(pkt->senderState); 1335 1336 // get the wavefront associated with this translation request 1337 Wavefront *wavefront = sender_state->wavefront; 1338 delete pkt->senderState; 1339 1340 if (success) { 1341 // pkt is reused in fetch(), don't delete it here. However, we must 1342 // reset the command to be a request so that it can be sent through 1343 // the cu's master port 1344 assert(pkt->cmd == MemCmd::ReadResp); 1345 pkt->cmd = MemCmd::ReadReq; 1346 1347 computeUnit->fetchStage.fetch(pkt, wavefront); 1348 } else { 1349 if (wavefront->dropFetch) { 1350 assert(wavefront->instructionBuffer.empty()); 1351 wavefront->dropFetch = false; 1352 } 1353 1354 wavefront->pendingFetch = 0; 1355 } 1356 1357 return true; 1358} 1359 1360/* 1361 * The initial translation request could have been rejected, if 1362 * <retries> queue is not empty. Retry sending the translation 1363 * request. sendRetry() is called from the peer port whenever 1364 * a translation completes. 1365 */ 1366void 1367ComputeUnit::ITLBPort::recvReqRetry() 1368{ 1369 1370 int len = retries.size(); 1371 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len); 1372 1373 assert(len > 0); 1374 assert(isStalled()); 1375 1376 // recvReqRetry is an indication that the resource on which this 1377 // port was stalling on is freed. So, remove the stall first 1378 unstallPort(); 1379 1380 for (int i = 0; i < len; ++i) { 1381 PacketPtr pkt = retries.front(); 1382 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1383 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr); 1384 1385 if (!sendTimingReq(pkt)) { 1386 stallPort(); // Stall port 1387 DPRINTF(GPUTLB, ": failed again\n"); 1388 break; 1389 } else { 1390 DPRINTF(GPUTLB, ": successful\n"); 1391 retries.pop_front(); 1392 } 1393 } 1394} 1395 1396void 1397ComputeUnit::regStats() 1398{ 1399 MemObject::regStats(); 1400 1401 vALUInsts 1402 .name(name() + ".valu_insts") 1403 .desc("Number of vector ALU insts issued.") 1404 ; 1405 vALUInstsPerWF 1406 .name(name() + ".valu_insts_per_wf") 1407 .desc("The avg. number of vector ALU insts issued per-wavefront.") 1408 ; 1409 sALUInsts 1410 .name(name() + ".salu_insts") 1411 .desc("Number of scalar ALU insts issued.") 1412 ; 1413 sALUInstsPerWF 1414 .name(name() + ".salu_insts_per_wf") 1415 .desc("The avg. number of scalar ALU insts issued per-wavefront.") 1416 ; 1417 instCyclesVALU 1418 .name(name() + ".inst_cycles_valu") 1419 .desc("Number of cycles needed to execute VALU insts.") 1420 ; 1421 instCyclesSALU 1422 .name(name() + ".inst_cycles_salu") 1423 .desc("Number of cycles needed to execute SALU insts.") 1424 ; 1425 threadCyclesVALU 1426 .name(name() + ".thread_cycles_valu") 1427 .desc("Number of thread cycles used to execute vector ALU ops. " 1428 "Similar to instCyclesVALU but multiplied by the number of " 1429 "active threads.") 1430 ; 1431 vALUUtilization 1432 .name(name() + ".valu_utilization") 1433 .desc("Percentage of active vector ALU threads in a wave.") 1434 ; 1435 ldsNoFlatInsts 1436 .name(name() + ".lds_no_flat_insts") 1437 .desc("Number of LDS insts issued, not including FLAT " 1438 "accesses that resolve to LDS.") 1439 ; 1440 ldsNoFlatInstsPerWF 1441 .name(name() + ".lds_no_flat_insts_per_wf") 1442 .desc("The avg. number of LDS insts (not including FLAT " 1443 "accesses that resolve to LDS) per-wavefront.") 1444 ; 1445 flatVMemInsts 1446 .name(name() + ".flat_vmem_insts") 1447 .desc("The number of FLAT insts that resolve to vmem issued.") 1448 ; 1449 flatVMemInstsPerWF 1450 .name(name() + ".flat_vmem_insts_per_wf") 1451 .desc("The average number of FLAT insts that resolve to vmem " 1452 "issued per-wavefront.") 1453 ; 1454 flatLDSInsts 1455 .name(name() + ".flat_lds_insts") 1456 .desc("The number of FLAT insts that resolve to LDS issued.") 1457 ; 1458 flatLDSInstsPerWF 1459 .name(name() + ".flat_lds_insts_per_wf") 1460 .desc("The average number of FLAT insts that resolve to LDS " 1461 "issued per-wavefront.") 1462 ; 1463 vectorMemWrites 1464 .name(name() + ".vector_mem_writes") 1465 .desc("Number of vector mem write insts (excluding FLAT insts).") 1466 ; 1467 vectorMemWritesPerWF 1468 .name(name() + ".vector_mem_writes_per_wf") 1469 .desc("The average number of vector mem write insts " 1470 "(excluding FLAT insts) per-wavefront.") 1471 ; 1472 vectorMemReads 1473 .name(name() + ".vector_mem_reads") 1474 .desc("Number of vector mem read insts (excluding FLAT insts).") 1475 ; 1476 vectorMemReadsPerWF 1477 .name(name() + ".vector_mem_reads_per_wf") 1478 .desc("The avg. number of vector mem read insts (excluding " 1479 "FLAT insts) per-wavefront.") 1480 ; 1481 scalarMemWrites 1482 .name(name() + ".scalar_mem_writes") 1483 .desc("Number of scalar mem write insts.") 1484 ; 1485 scalarMemWritesPerWF 1486 .name(name() + ".scalar_mem_writes_per_wf") 1487 .desc("The average number of scalar mem write insts per-wavefront.") 1488 ; 1489 scalarMemReads 1490 .name(name() + ".scalar_mem_reads") 1491 .desc("Number of scalar mem read insts.") 1492 ; 1493 scalarMemReadsPerWF 1494 .name(name() + ".scalar_mem_reads_per_wf") 1495 .desc("The average number of scalar mem read insts per-wavefront.") 1496 ; 1497 1498 vALUInstsPerWF = vALUInsts / completedWfs; 1499 sALUInstsPerWF = sALUInsts / completedWfs; 1500 vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100; 1501 ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs; 1502 flatVMemInstsPerWF = flatVMemInsts / completedWfs; 1503 flatLDSInstsPerWF = flatLDSInsts / completedWfs; 1504 vectorMemWritesPerWF = vectorMemWrites / completedWfs; 1505 vectorMemReadsPerWF = vectorMemReads / completedWfs; 1506 scalarMemWritesPerWF = scalarMemWrites / completedWfs; 1507 scalarMemReadsPerWF = scalarMemReads / completedWfs; 1508 1509 tlbCycles 1510 .name(name() + ".tlb_cycles") 1511 .desc("total number of cycles for all uncoalesced requests") 1512 ; 1513 1514 tlbRequests 1515 .name(name() + ".tlb_requests") 1516 .desc("number of uncoalesced requests") 1517 ; 1518 1519 tlbLatency 1520 .name(name() + ".avg_translation_latency") 1521 .desc("Avg. translation latency for data translations") 1522 ; 1523 1524 tlbLatency = tlbCycles / tlbRequests; 1525 1526 hitsPerTLBLevel 1527 .init(4) 1528 .name(name() + ".TLB_hits_distribution") 1529 .desc("TLB hits distribution (0 for page table, x for Lx-TLB") 1530 ; 1531 1532 // fixed number of TLB levels 1533 for (int i = 0; i < 4; ++i) { 1534 if (!i) 1535 hitsPerTLBLevel.subname(i,"page_table"); 1536 else 1537 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i)); 1538 } 1539 1540 execRateDist 1541 .init(0, 10, 2) 1542 .name(name() + ".inst_exec_rate") 1543 .desc("Instruction Execution Rate: Number of executed vector " 1544 "instructions per cycle") 1545 ; 1546 1547 ldsBankConflictDist 1548 .init(0, wfSize(), 2) 1549 .name(name() + ".lds_bank_conflicts") 1550 .desc("Number of bank conflicts per LDS memory packet") 1551 ; 1552 1553 ldsBankAccesses 1554 .name(name() + ".lds_bank_access_cnt") 1555 .desc("Total number of LDS bank accesses") 1556 ; 1557 1558 pageDivergenceDist 1559 // A wavefront can touch up to N pages per memory instruction where 1560 // N is equal to the wavefront size 1561 // The number of pages per bin can be configured (here it's 4). 1562 .init(1, wfSize(), 4) 1563 .name(name() + ".page_divergence_dist") 1564 .desc("pages touched per wf (over all mem. instr.)") 1565 ; 1566 1567 controlFlowDivergenceDist 1568 .init(1, wfSize(), 4) 1569 .name(name() + ".warp_execution_dist") 1570 .desc("number of lanes active per instruction (oval all instructions)") 1571 ; 1572 1573 activeLanesPerGMemInstrDist 1574 .init(1, wfSize(), 4) 1575 .name(name() + ".gmem_lanes_execution_dist") 1576 .desc("number of active lanes per global memory instruction") 1577 ; 1578 1579 activeLanesPerLMemInstrDist 1580 .init(1, wfSize(), 4) 1581 .name(name() + ".lmem_lanes_execution_dist") 1582 .desc("number of active lanes per local memory instruction") 1583 ; 1584 1585 numInstrExecuted 1586 .name(name() + ".num_instr_executed") 1587 .desc("number of instructions executed") 1588 ; 1589 1590 numVecOpsExecuted 1591 .name(name() + ".num_vec_ops_executed") 1592 .desc("number of vec ops executed (e.g. WF size/inst)") 1593 ; 1594 1595 totalCycles 1596 .name(name() + ".num_total_cycles") 1597 .desc("number of cycles the CU ran for") 1598 ; 1599 1600 ipc 1601 .name(name() + ".ipc") 1602 .desc("Instructions per cycle (this CU only)") 1603 ; 1604 1605 vpc 1606 .name(name() + ".vpc") 1607 .desc("Vector Operations per cycle (this CU only)") 1608 ; 1609 1610 numALUInstsExecuted 1611 .name(name() + ".num_alu_insts_executed") 1612 .desc("Number of dynamic non-GM memory insts executed") 1613 ; 1614 1615 wgBlockedDueLdsAllocation 1616 .name(name() + ".wg_blocked_due_lds_alloc") 1617 .desc("Workgroup blocked due to LDS capacity") 1618 ; 1619 1620 ipc = numInstrExecuted / totalCycles; 1621 vpc = numVecOpsExecuted / totalCycles; 1622 1623 numTimesWgBlockedDueVgprAlloc 1624 .name(name() + ".times_wg_blocked_due_vgpr_alloc") 1625 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD") 1626 ; 1627 1628 dynamicGMemInstrCnt 1629 .name(name() + ".global_mem_instr_cnt") 1630 .desc("dynamic global memory instructions count") 1631 ; 1632 1633 dynamicLMemInstrCnt 1634 .name(name() + ".local_mem_instr_cnt") 1635 .desc("dynamic local memory intruction count") 1636 ; 1637 1638 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt - 1639 dynamicLMemInstrCnt; 1640 1641 completedWfs 1642 .name(name() + ".num_completed_wfs") 1643 .desc("number of completed wavefronts") 1644 ; 1645 1646 numCASOps 1647 .name(name() + ".num_CAS_ops") 1648 .desc("number of compare and swap operations") 1649 ; 1650 1651 numFailedCASOps 1652 .name(name() + ".num_failed_CAS_ops") 1653 .desc("number of compare and swap operations that failed") 1654 ; 1655 1656 // register stats of pipeline stages 1657 fetchStage.regStats(); 1658 scoreboardCheckStage.regStats(); 1659 scheduleStage.regStats(); 1660 execStage.regStats(); 1661 1662 // register stats of memory pipeline 1663 globalMemoryPipe.regStats(); 1664 localMemoryPipe.regStats(); 1665} 1666 1667void 1668ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst) 1669{ 1670 if (gpuDynInst->isScalar()) { 1671 if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) { 1672 sALUInsts++; 1673 instCyclesSALU++; 1674 } else if (gpuDynInst->isLoad()) { 1675 scalarMemReads++; 1676 } else if (gpuDynInst->isStore()) { 1677 scalarMemWrites++; 1678 } 1679 } else { 1680 if (gpuDynInst->isALU()) { 1681 vALUInsts++; 1682 instCyclesVALU++; 1683 threadCyclesVALU += gpuDynInst->wavefront()->execMask().count(); 1684 } else if (gpuDynInst->isFlat()) { 1685 if (gpuDynInst->isLocalMem()) { 1686 flatLDSInsts++; 1687 } else { 1688 flatVMemInsts++; 1689 } 1690 } else if (gpuDynInst->isLocalMem()) { 1691 ldsNoFlatInsts++; 1692 } else if (gpuDynInst->isLoad()) { 1693 vectorMemReads++; 1694 } else if (gpuDynInst->isStore()) { 1695 vectorMemWrites++; 1696 } 1697 } 1698} 1699 1700void 1701ComputeUnit::updatePageDivergenceDist(Addr addr) 1702{ 1703 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes); 1704 1705 if (!pagesTouched.count(virt_page_addr)) 1706 pagesTouched[virt_page_addr] = 1; 1707 else 1708 pagesTouched[virt_page_addr]++; 1709} 1710 1711void 1712ComputeUnit::CUExitCallback::process() 1713{ 1714 if (computeUnit->countPages) { 1715 std::ostream *page_stat_file = 1716 simout.create(computeUnit->name().c_str())->stream(); 1717 1718 *page_stat_file << "page, wavefront accesses, workitem accesses" << 1719 std::endl; 1720 1721 for (auto iter : computeUnit->pageAccesses) { 1722 *page_stat_file << std::hex << iter.first << ","; 1723 *page_stat_file << std::dec << iter.second.first << ","; 1724 *page_stat_file << std::dec << iter.second.second << std::endl; 1725 } 1726 } 1727 } 1728 1729bool 1730ComputeUnit::isDone() const 1731{ 1732 for (int i = 0; i < numSIMDs; ++i) { 1733 if (!isSimdDone(i)) { 1734 return false; 1735 } 1736 } 1737 1738 bool glbMemBusRdy = true; 1739 for (int j = 0; j < numGlbMemUnits; ++j) { 1740 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy(); 1741 } 1742 bool locMemBusRdy = true; 1743 for (int j = 0; j < numLocMemUnits; ++j) { 1744 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy(); 1745 } 1746 1747 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() || 1748 !globalMemoryPipe.isGMStRespFIFOWrRdy() || 1749 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy() 1750 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() || 1751 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) { 1752 return false; 1753 } 1754 1755 return true; 1756} 1757 1758int32_t 1759ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const 1760{ 1761 return lds.getRefCounter(dispatchId, wgId); 1762} 1763 1764bool 1765ComputeUnit::isSimdDone(uint32_t simdId) const 1766{ 1767 assert(simdId < numSIMDs); 1768 1769 for (int i=0; i < numGlbMemUnits; ++i) { 1770 if (!vrfToGlobalMemPipeBus[i].rdy()) 1771 return false; 1772 } 1773 for (int i=0; i < numLocMemUnits; ++i) { 1774 if (!vrfToLocalMemPipeBus[i].rdy()) 1775 return false; 1776 } 1777 if (!aluPipe[simdId].rdy()) { 1778 return false; 1779 } 1780 1781 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){ 1782 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) { 1783 return false; 1784 } 1785 } 1786 1787 return true; 1788} 1789 1790/** 1791 * send a general request to the LDS 1792 * make sure to look at the return value here as your request might be 1793 * NACK'd and returning false means that you have to have some backup plan 1794 */ 1795bool 1796ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst) 1797{ 1798 // this is just a request to carry the GPUDynInstPtr 1799 // back and forth 1800 RequestPtr newRequest = std::make_shared<Request>(); 1801 newRequest->setPaddr(0x0); 1802 1803 // ReadReq is not evaluted by the LDS but the Packet ctor requires this 1804 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq); 1805 1806 // This is the SenderState needed upon return 1807 newPacket->senderState = new LDSPort::SenderState(gpuDynInst); 1808 1809 return ldsPort->sendTimingReq(newPacket); 1810} 1811 1812/** 1813 * get the result of packets sent to the LDS when they return 1814 */ 1815bool 1816ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet) 1817{ 1818 const ComputeUnit::LDSPort::SenderState *senderState = 1819 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState); 1820 1821 fatal_if(!senderState, "did not get the right sort of sender state"); 1822 1823 GPUDynInstPtr gpuDynInst = senderState->getMemInst(); 1824 1825 delete packet->senderState; 1826 delete packet; 1827 1828 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst); 1829 return true; 1830} 1831 1832/** 1833 * attempt to send this packet, either the port is already stalled, the request 1834 * is nack'd and must stall or the request goes through 1835 * when a request cannot be sent, add it to the retries queue 1836 */ 1837bool 1838ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt) 1839{ 1840 ComputeUnit::LDSPort::SenderState *sender_state = 1841 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState); 1842 fatal_if(!sender_state, "packet without a valid sender state"); 1843 1844 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst(); 1845 1846 if (isStalled()) { 1847 fatal_if(retries.empty(), "must have retries waiting to be stalled"); 1848 1849 retries.push(pkt); 1850 1851 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n", 1852 computeUnit->cu_id, gpuDynInst->simdId, 1853 gpuDynInst->wfSlotId); 1854 return false; 1855 } else if (!MasterPort::sendTimingReq(pkt)) { 1856 // need to stall the LDS port until a recvReqRetry() is received 1857 // this indicates that there is more space 1858 stallPort(); 1859 retries.push(pkt); 1860 1861 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n", 1862 computeUnit->cu_id, gpuDynInst->simdId, 1863 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1864 return false; 1865 } else { 1866 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n", 1867 computeUnit->cu_id, gpuDynInst->simdId, 1868 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1869 return true; 1870 } 1871} 1872 1873/** 1874 * the bus is telling the port that there is now space so retrying stalled 1875 * requests should work now 1876 * this allows the port to have a request be nack'd and then have the receiver 1877 * say when there is space, rather than simply retrying the send every cycle 1878 */ 1879void 1880ComputeUnit::LDSPort::recvReqRetry() 1881{ 1882 auto queueSize = retries.size(); 1883 1884 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n", 1885 computeUnit->cu_id, queueSize); 1886 1887 fatal_if(queueSize < 1, 1888 "why was there a recvReqRetry() with no pending reqs?"); 1889 fatal_if(!isStalled(), 1890 "recvReqRetry() happened when the port was not stalled"); 1891 1892 unstallPort(); 1893 1894 while (!retries.empty()) { 1895 PacketPtr packet = retries.front(); 1896 1897 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id); 1898 1899 if (!MasterPort::sendTimingReq(packet)) { 1900 // Stall port 1901 stallPort(); 1902 DPRINTF(GPUPort, ": LDS send failed again\n"); 1903 break; 1904 } else { 1905 DPRINTF(GPUTLB, ": LDS send successful\n"); 1906 retries.pop(); 1907 } 1908 } 1909} 1910