compute_unit.cc revision 11698
1/* 2 * Copyright (c) 2011-2015 Advanced Micro Devices, Inc. 3 * All rights reserved. 4 * 5 * For use for simulation and test purposes only 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the copyright holder nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Author: John Kalamatianos, Anthony Gutierrez 34 */ 35#include "gpu-compute/compute_unit.hh" 36 37#include <limits> 38 39#include "base/output.hh" 40#include "debug/GPUDisp.hh" 41#include "debug/GPUExec.hh" 42#include "debug/GPUFetch.hh" 43#include "debug/GPUMem.hh" 44#include "debug/GPUPort.hh" 45#include "debug/GPUPrefetch.hh" 46#include "debug/GPUSync.hh" 47#include "debug/GPUTLB.hh" 48#include "gpu-compute/dispatcher.hh" 49#include "gpu-compute/gpu_dyn_inst.hh" 50#include "gpu-compute/gpu_static_inst.hh" 51#include "gpu-compute/ndrange.hh" 52#include "gpu-compute/shader.hh" 53#include "gpu-compute/simple_pool_manager.hh" 54#include "gpu-compute/vector_register_file.hh" 55#include "gpu-compute/wavefront.hh" 56#include "mem/page_table.hh" 57#include "sim/process.hh" 58 59ComputeUnit::ComputeUnit(const Params *p) : MemObject(p), fetchStage(p), 60 scoreboardCheckStage(p), scheduleStage(p), execStage(p), 61 globalMemoryPipe(p), localMemoryPipe(p), rrNextMemID(0), rrNextALUWp(0), 62 cu_id(p->cu_id), vrf(p->vector_register_file), numSIMDs(p->num_SIMDs), 63 spBypassPipeLength(p->spbypass_pipe_length), 64 dpBypassPipeLength(p->dpbypass_pipe_length), 65 issuePeriod(p->issue_period), 66 numGlbMemUnits(p->num_global_mem_pipes), 67 numLocMemUnits(p->num_shared_mem_pipes), 68 perLaneTLB(p->perLaneTLB), prefetchDepth(p->prefetch_depth), 69 prefetchStride(p->prefetch_stride), prefetchType(p->prefetch_prev_type), 70 xact_cas_mode(p->xactCasMode), debugSegFault(p->debugSegFault), 71 functionalTLB(p->functionalTLB), localMemBarrier(p->localMemBarrier), 72 countPages(p->countPages), barrier_id(0), 73 vrfToCoalescerBusWidth(p->vrf_to_coalescer_bus_width), 74 coalescerToVrfBusWidth(p->coalescer_to_vrf_bus_width), 75 req_tick_latency(p->mem_req_latency * p->clk_domain->clockPeriod()), 76 resp_tick_latency(p->mem_resp_latency * p->clk_domain->clockPeriod()), 77 _masterId(p->system->getMasterId(name() + ".ComputeUnit")), 78 lds(*p->localDataStore), _cacheLineSize(p->system->cacheLineSize()), 79 globalSeqNum(0), wavefrontSize(p->wfSize), 80 kernelLaunchInst(new KernelLaunchStaticInst()) 81{ 82 /** 83 * This check is necessary because std::bitset only provides conversion 84 * to unsigned long or unsigned long long via to_ulong() or to_ullong(). 85 * there are * a few places in the code where to_ullong() is used, however 86 * if VSZ is larger than a value the host can support then bitset will 87 * throw a runtime exception. we should remove all use of to_long() or 88 * to_ullong() so we can have VSZ greater than 64b, however until that is 89 * done this assert is required. 90 */ 91 fatal_if(p->wfSize > std::numeric_limits<unsigned long long>::digits || 92 p->wfSize <= 0, 93 "WF size is larger than the host can support"); 94 fatal_if(!isPowerOf2(wavefrontSize), 95 "Wavefront size should be a power of 2"); 96 // calculate how many cycles a vector load or store will need to transfer 97 // its data over the corresponding buses 98 numCyclesPerStoreTransfer = 99 (uint32_t)ceil((double)(wfSize() * sizeof(uint32_t)) / 100 (double)vrfToCoalescerBusWidth); 101 102 numCyclesPerLoadTransfer = (wfSize() * sizeof(uint32_t)) 103 / coalescerToVrfBusWidth; 104 105 lastVaddrWF.resize(numSIMDs); 106 wfList.resize(numSIMDs); 107 108 for (int j = 0; j < numSIMDs; ++j) { 109 lastVaddrWF[j].resize(p->n_wf); 110 111 for (int i = 0; i < p->n_wf; ++i) { 112 lastVaddrWF[j][i].resize(wfSize()); 113 114 wfList[j].push_back(p->wavefronts[j * p->n_wf + i]); 115 wfList[j][i]->setParent(this); 116 117 for (int k = 0; k < wfSize(); ++k) { 118 lastVaddrWF[j][i][k] = 0; 119 } 120 } 121 } 122 123 lastVaddrSimd.resize(numSIMDs); 124 125 for (int i = 0; i < numSIMDs; ++i) { 126 lastVaddrSimd[i].resize(wfSize(), 0); 127 } 128 129 lastVaddrCU.resize(wfSize()); 130 131 lds.setParent(this); 132 133 if (p->execPolicy == "OLDEST-FIRST") { 134 exec_policy = EXEC_POLICY::OLDEST; 135 } else if (p->execPolicy == "ROUND-ROBIN") { 136 exec_policy = EXEC_POLICY::RR; 137 } else { 138 fatal("Invalid WF execution policy (CU)\n"); 139 } 140 141 memPort.resize(wfSize()); 142 143 // resize the tlbPort vectorArray 144 int tlbPort_width = perLaneTLB ? wfSize() : 1; 145 tlbPort.resize(tlbPort_width); 146 147 cuExitCallback = new CUExitCallback(this); 148 registerExitCallback(cuExitCallback); 149 150 xactCasLoadMap.clear(); 151 lastExecCycle.resize(numSIMDs, 0); 152 153 for (int i = 0; i < vrf.size(); ++i) { 154 vrf[i]->setParent(this); 155 } 156 157 numVecRegsPerSimd = vrf[0]->numRegs(); 158} 159 160ComputeUnit::~ComputeUnit() 161{ 162 // Delete wavefront slots 163 for (int j = 0; j < numSIMDs; ++j) { 164 for (int i = 0; i < shader->n_wf; ++i) { 165 delete wfList[j][i]; 166 } 167 lastVaddrSimd[j].clear(); 168 } 169 lastVaddrCU.clear(); 170 readyList.clear(); 171 waveStatusList.clear(); 172 dispatchList.clear(); 173 vectorAluInstAvail.clear(); 174 delete cuExitCallback; 175 delete ldsPort; 176} 177 178void 179ComputeUnit::fillKernelState(Wavefront *w, NDRange *ndr) 180{ 181 w->resizeRegFiles(ndr->q.cRegCount, ndr->q.sRegCount, ndr->q.dRegCount); 182 183 w->workGroupSz[0] = ndr->q.wgSize[0]; 184 w->workGroupSz[1] = ndr->q.wgSize[1]; 185 w->workGroupSz[2] = ndr->q.wgSize[2]; 186 w->wgSz = w->workGroupSz[0] * w->workGroupSz[1] * w->workGroupSz[2]; 187 w->gridSz[0] = ndr->q.gdSize[0]; 188 w->gridSz[1] = ndr->q.gdSize[1]; 189 w->gridSz[2] = ndr->q.gdSize[2]; 190 w->kernelArgs = ndr->q.args; 191 w->privSizePerItem = ndr->q.privMemPerItem; 192 w->spillSizePerItem = ndr->q.spillMemPerItem; 193 w->roBase = ndr->q.roMemStart; 194 w->roSize = ndr->q.roMemTotal; 195 w->computeActualWgSz(ndr); 196} 197 198void 199ComputeUnit::updateEvents() { 200 201 if (!timestampVec.empty()) { 202 uint32_t vecSize = timestampVec.size(); 203 uint32_t i = 0; 204 while (i < vecSize) { 205 if (timestampVec[i] <= shader->tick_cnt) { 206 std::pair<uint32_t, uint32_t> regInfo = regIdxVec[i]; 207 vrf[regInfo.first]->markReg(regInfo.second, sizeof(uint32_t), 208 statusVec[i]); 209 timestampVec.erase(timestampVec.begin() + i); 210 regIdxVec.erase(regIdxVec.begin() + i); 211 statusVec.erase(statusVec.begin() + i); 212 --vecSize; 213 --i; 214 } 215 ++i; 216 } 217 } 218 219 for (int i = 0; i< numSIMDs; ++i) { 220 vrf[i]->updateEvents(); 221 } 222} 223 224 225void 226ComputeUnit::startWavefront(Wavefront *w, int waveId, LdsChunk *ldsChunk, 227 NDRange *ndr) 228{ 229 static int _n_wave = 0; 230 231 VectorMask init_mask; 232 init_mask.reset(); 233 234 for (int k = 0; k < wfSize(); ++k) { 235 if (k + waveId * wfSize() < w->actualWgSzTotal) 236 init_mask[k] = 1; 237 } 238 239 w->kernId = ndr->dispatchId; 240 w->wfId = waveId; 241 w->initMask = init_mask.to_ullong(); 242 243 for (int k = 0; k < wfSize(); ++k) { 244 w->workItemId[0][k] = (k + waveId * wfSize()) % w->actualWgSz[0]; 245 w->workItemId[1][k] = ((k + waveId * wfSize()) / w->actualWgSz[0]) % 246 w->actualWgSz[1]; 247 w->workItemId[2][k] = (k + waveId * wfSize()) / 248 (w->actualWgSz[0] * w->actualWgSz[1]); 249 250 w->workItemFlatId[k] = w->workItemId[2][k] * w->actualWgSz[0] * 251 w->actualWgSz[1] + w->workItemId[1][k] * w->actualWgSz[0] + 252 w->workItemId[0][k]; 253 } 254 255 w->barrierSlots = divCeil(w->actualWgSzTotal, wfSize()); 256 257 w->barCnt.resize(wfSize(), 0); 258 259 w->maxBarCnt = 0; 260 w->oldBarrierCnt = 0; 261 w->barrierCnt = 0; 262 263 w->privBase = ndr->q.privMemStart; 264 ndr->q.privMemStart += ndr->q.privMemPerItem * wfSize(); 265 266 w->spillBase = ndr->q.spillMemStart; 267 ndr->q.spillMemStart += ndr->q.spillMemPerItem * wfSize(); 268 269 w->pushToReconvergenceStack(0, UINT32_MAX, init_mask.to_ulong()); 270 271 // WG state 272 w->wgId = ndr->globalWgId; 273 w->dispatchId = ndr->dispatchId; 274 w->workGroupId[0] = w->wgId % ndr->numWg[0]; 275 w->workGroupId[1] = (w->wgId / ndr->numWg[0]) % ndr->numWg[1]; 276 w->workGroupId[2] = w->wgId / (ndr->numWg[0] * ndr->numWg[1]); 277 278 w->barrierId = barrier_id; 279 w->stalledAtBarrier = false; 280 281 // set the wavefront context to have a pointer to this section of the LDS 282 w->ldsChunk = ldsChunk; 283 284 int32_t refCount M5_VAR_USED = 285 lds.increaseRefCounter(w->dispatchId, w->wgId); 286 DPRINTF(GPUDisp, "CU%d: increase ref ctr wg[%d] to [%d]\n", 287 cu_id, w->wgId, refCount); 288 289 w->instructionBuffer.clear(); 290 291 if (w->pendingFetch) 292 w->dropFetch = true; 293 294 // is this the last wavefront in the workgroup 295 // if set the spillWidth to be the remaining work-items 296 // so that the vector access is correct 297 if ((waveId + 1) * wfSize() >= w->actualWgSzTotal) { 298 w->spillWidth = w->actualWgSzTotal - (waveId * wfSize()); 299 } else { 300 w->spillWidth = wfSize(); 301 } 302 303 DPRINTF(GPUDisp, "Scheduling wfDynId/barrier_id %d/%d on CU%d: " 304 "WF[%d][%d]\n", _n_wave, barrier_id, cu_id, w->simdId, w->wfSlotId); 305 306 w->start(++_n_wave, ndr->q.code_ptr); 307} 308 309void 310ComputeUnit::StartWorkgroup(NDRange *ndr) 311{ 312 // reserve the LDS capacity allocated to the work group 313 // disambiguated by the dispatch ID and workgroup ID, which should be 314 // globally unique 315 LdsChunk *ldsChunk = lds.reserveSpace(ndr->dispatchId, ndr->globalWgId, 316 ndr->q.ldsSize); 317 318 // Send L1 cache acquire 319 // isKernel + isAcquire = Kernel Begin 320 if (shader->impl_kern_boundary_sync) { 321 GPUDynInstPtr gpuDynInst = 322 std::make_shared<GPUDynInst>(this, nullptr, kernelLaunchInst, 323 getAndIncSeqNum()); 324 325 gpuDynInst->useContinuation = false; 326 injectGlobalMemFence(gpuDynInst, true); 327 } 328 329 // calculate the number of 32-bit vector registers required by wavefront 330 int vregDemand = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 331 int wave_id = 0; 332 333 // Assign WFs by spreading them across SIMDs, 1 WF per SIMD at a time 334 for (int m = 0; m < shader->n_wf * numSIMDs; ++m) { 335 Wavefront *w = wfList[m % numSIMDs][m / numSIMDs]; 336 // Check if this wavefront slot is available: 337 // It must be stopped and not waiting 338 // for a release to complete S_RETURNING 339 if (w->status == Wavefront::S_STOPPED) { 340 fillKernelState(w, ndr); 341 // if we have scheduled all work items then stop 342 // scheduling wavefronts 343 if (wave_id * wfSize() >= w->actualWgSzTotal) 344 break; 345 346 // reserve vector registers for the scheduled wavefront 347 assert(vectorRegsReserved[m % numSIMDs] <= numVecRegsPerSimd); 348 uint32_t normSize = 0; 349 350 w->startVgprIndex = vrf[m % numSIMDs]->manager-> 351 allocateRegion(vregDemand, &normSize); 352 353 w->reservedVectorRegs = normSize; 354 vectorRegsReserved[m % numSIMDs] += w->reservedVectorRegs; 355 356 startWavefront(w, wave_id, ldsChunk, ndr); 357 ++wave_id; 358 } 359 } 360 ++barrier_id; 361} 362 363int 364ComputeUnit::ReadyWorkgroup(NDRange *ndr) 365{ 366 // Get true size of workgroup (after clamping to grid size) 367 int trueWgSize[3]; 368 int trueWgSizeTotal = 1; 369 370 for (int d = 0; d < 3; ++d) { 371 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] - 372 ndr->wgId[d] * ndr->q.wgSize[d]); 373 374 trueWgSizeTotal *= trueWgSize[d]; 375 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]); 376 } 377 378 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal); 379 380 // calculate the number of 32-bit vector registers required by each 381 // work item of the work group 382 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 383 bool vregAvail = true; 384 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize(); 385 int freeWfSlots = 0; 386 // check if the total number of VGPRs required by all WFs of the WG 387 // fit in the VRFs of all SIMD units 388 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd)); 389 int numMappedWfs = 0; 390 std::vector<int> numWfsPerSimd; 391 numWfsPerSimd.resize(numSIMDs, 0); 392 // find how many free WF slots we have across all SIMDs 393 for (int j = 0; j < shader->n_wf; ++j) { 394 for (int i = 0; i < numSIMDs; ++i) { 395 if (wfList[i][j]->status == Wavefront::S_STOPPED) { 396 // count the number of free WF slots 397 ++freeWfSlots; 398 if (numMappedWfs < numWfs) { 399 // count the WFs to be assigned per SIMD 400 numWfsPerSimd[i]++; 401 } 402 numMappedWfs++; 403 } 404 } 405 } 406 407 // if there are enough free WF slots then find if there are enough 408 // free VGPRs per SIMD based on the WF->SIMD mapping 409 if (freeWfSlots >= numWfs) { 410 for (int j = 0; j < numSIMDs; ++j) { 411 // find if there are enough free VGPR regions in the SIMD's VRF 412 // to accommodate the WFs of the new WG that would be mapped to 413 // this SIMD unit 414 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j], 415 vregDemandPerWI); 416 417 // stop searching if there is at least one SIMD 418 // whose VRF does not have enough free VGPR pools. 419 // This is because a WG is scheduled only if ALL 420 // of its WFs can be scheduled 421 if (!vregAvail) 422 break; 423 } 424 } 425 426 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n", 427 freeWfSlots, vregAvail); 428 429 if (!vregAvail) { 430 ++numTimesWgBlockedDueVgprAlloc; 431 } 432 433 // Return true if enough WF slots to submit workgroup and if there are 434 // enough VGPRs to schedule all WFs to their SIMD units 435 if (!lds.canReserve(ndr->q.ldsSize)) { 436 wgBlockedDueLdsAllocation++; 437 } 438 439 // Return true if (a) there are enough free WF slots to submit 440 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their 441 // SIMD units and (c) if there is enough space in LDS 442 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize); 443} 444 445int 446ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots) 447{ 448 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id); 449 int ccnt = 0; 450 451 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) { 452 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) { 453 Wavefront *w = wfList[i_simd][i_wf]; 454 455 if (w->status == Wavefront::S_RUNNING) { 456 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf); 457 458 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n", 459 w->barrierId, _barrier_id); 460 461 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n", 462 w->barrierCnt, bcnt); 463 } 464 465 if (w->status == Wavefront::S_RUNNING && 466 w->barrierId == _barrier_id && w->barrierCnt == bcnt && 467 !w->outstandingReqs) { 468 ++ccnt; 469 470 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to " 471 "%d\n", i_simd, i_wf, ccnt); 472 } 473 } 474 } 475 476 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n", 477 cu_id, ccnt, bslots); 478 479 return ccnt == bslots; 480} 481 482// Check if the current wavefront is blocked on additional resources. 483bool 484ComputeUnit::cedeSIMD(int simdId, int wfSlotId) 485{ 486 bool cede = false; 487 488 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld 489 // magic instructions will impact the scheduling of wavefronts 490 if (xact_cas_mode) { 491 /* 492 * When a wavefront calls xact_cas_ld, it adds itself to a per address 493 * queue. All per address queues are managed by the xactCasLoadMap. 494 * 495 * A wavefront is not blocked if: it is not in ANY per address queue or 496 * if it is at the head of a per address queue. 497 */ 498 for (auto itMap : xactCasLoadMap) { 499 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue; 500 501 if (!curWaveIDQueue.empty()) { 502 for (auto it : curWaveIDQueue) { 503 waveIdentifier cur_wave = it; 504 505 if (cur_wave.simdId == simdId && 506 cur_wave.wfSlotId == wfSlotId) { 507 // 2 possibilities 508 // 1: this WF has a green light 509 // 2: another WF has a green light 510 waveIdentifier owner_wave = curWaveIDQueue.front(); 511 512 if (owner_wave.simdId != cur_wave.simdId || 513 owner_wave.wfSlotId != cur_wave.wfSlotId) { 514 // possibility 2 515 cede = true; 516 break; 517 } else { 518 // possibility 1 519 break; 520 } 521 } 522 } 523 } 524 } 525 } 526 527 return cede; 528} 529 530// Execute one clock worth of work on the ComputeUnit. 531void 532ComputeUnit::exec() 533{ 534 updateEvents(); 535 // Execute pipeline stages in reverse order to simulate 536 // the pipeline latency 537 globalMemoryPipe.exec(); 538 localMemoryPipe.exec(); 539 execStage.exec(); 540 scheduleStage.exec(); 541 scoreboardCheckStage.exec(); 542 fetchStage.exec(); 543 544 totalCycles++; 545} 546 547void 548ComputeUnit::init() 549{ 550 // Initialize CU Bus models 551 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 552 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 553 nextGlbMemBus = 0; 554 nextLocMemBus = 0; 555 fatal_if(numGlbMemUnits > 1, 556 "No support for multiple Global Memory Pipelines exists!!!"); 557 vrfToGlobalMemPipeBus.resize(numGlbMemUnits); 558 for (int j = 0; j < numGlbMemUnits; ++j) { 559 vrfToGlobalMemPipeBus[j] = WaitClass(); 560 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 561 } 562 563 fatal_if(numLocMemUnits > 1, 564 "No support for multiple Local Memory Pipelines exists!!!"); 565 vrfToLocalMemPipeBus.resize(numLocMemUnits); 566 for (int j = 0; j < numLocMemUnits; ++j) { 567 vrfToLocalMemPipeBus[j] = WaitClass(); 568 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 569 } 570 vectorRegsReserved.resize(numSIMDs, 0); 571 aluPipe.resize(numSIMDs); 572 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits); 573 574 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) { 575 wfWait[i] = WaitClass(); 576 wfWait[i].init(&shader->tick_cnt, shader->ticks(1)); 577 } 578 579 for (int i = 0; i < numSIMDs; ++i) { 580 aluPipe[i] = WaitClass(); 581 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1)); 582 } 583 584 // Setup space for call args 585 for (int j = 0; j < numSIMDs; ++j) { 586 for (int i = 0; i < shader->n_wf; ++i) { 587 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize); 588 } 589 } 590 591 // Initializing pipeline resources 592 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits); 593 waveStatusList.resize(numSIMDs); 594 595 for (int j = 0; j < numSIMDs; ++j) { 596 for (int i = 0; i < shader->n_wf; ++i) { 597 waveStatusList[j].push_back( 598 std::make_pair(wfList[j][i], BLOCKED)); 599 } 600 } 601 602 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) { 603 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY)); 604 } 605 606 fetchStage.init(this); 607 scoreboardCheckStage.init(this); 608 scheduleStage.init(this); 609 execStage.init(this); 610 globalMemoryPipe.init(this); 611 localMemoryPipe.init(this); 612 // initialize state for statistics calculation 613 vectorAluInstAvail.resize(numSIMDs, false); 614 shrMemInstAvail = 0; 615 glbMemInstAvail = 0; 616} 617 618bool 619ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt) 620{ 621 // Ruby has completed the memory op. Schedule the mem_resp_event at the 622 // appropriate cycle to process the timing memory response 623 // This delay represents the pipeline delay 624 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 625 int index = sender_state->port_index; 626 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 627 628 // Is the packet returned a Kernel End or Barrier 629 if (pkt->req->isKernel() && pkt->req->isRelease()) { 630 Wavefront *w = 631 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId]; 632 633 // Check if we are waiting on Kernel End Release 634 if (w->status == Wavefront::S_RETURNING) { 635 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n", 636 computeUnit->cu_id, w->simdId, w->wfSlotId, 637 w->wfDynId, w->kernId); 638 639 computeUnit->shader->dispatcher->notifyWgCompl(w); 640 w->status = Wavefront::S_STOPPED; 641 } else { 642 w->outstandingReqs--; 643 } 644 645 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n", 646 computeUnit->cu_id, gpuDynInst->simdId, 647 gpuDynInst->wfSlotId, w->barrierCnt); 648 649 if (gpuDynInst->useContinuation) { 650 assert(!gpuDynInst->isNoScope()); 651 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 652 gpuDynInst); 653 } 654 655 delete pkt->senderState; 656 delete pkt->req; 657 delete pkt; 658 return true; 659 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) { 660 if (gpuDynInst->useContinuation) { 661 assert(!gpuDynInst->isNoScope()); 662 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 663 gpuDynInst); 664 } 665 666 delete pkt->senderState; 667 delete pkt->req; 668 delete pkt; 669 return true; 670 } 671 672 ComputeUnit::DataPort::MemRespEvent *mem_resp_event = 673 new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index], 674 pkt); 675 676 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n", 677 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 678 index, pkt->req->getPaddr()); 679 680 computeUnit->schedule(mem_resp_event, 681 curTick() + computeUnit->resp_tick_latency); 682 return true; 683} 684 685void 686ComputeUnit::DataPort::recvReqRetry() 687{ 688 int len = retries.size(); 689 690 assert(len > 0); 691 692 for (int i = 0; i < len; ++i) { 693 PacketPtr pkt = retries.front().first; 694 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second; 695 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n", 696 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 697 pkt->req->getPaddr()); 698 699 /** Currently Ruby can return false due to conflicts for the particular 700 * cache block or address. Thus other requests should be allowed to 701 * pass and the data port should expect multiple retries. */ 702 if (!sendTimingReq(pkt)) { 703 DPRINTF(GPUMem, "failed again!\n"); 704 break; 705 } else { 706 DPRINTF(GPUMem, "successful!\n"); 707 retries.pop_front(); 708 } 709 } 710} 711 712bool 713ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt) 714{ 715 computeUnit->fetchStage.processFetchReturn(pkt); 716 717 return true; 718} 719 720void 721ComputeUnit::SQCPort::recvReqRetry() 722{ 723 int len = retries.size(); 724 725 assert(len > 0); 726 727 for (int i = 0; i < len; ++i) { 728 PacketPtr pkt = retries.front().first; 729 Wavefront *wavefront M5_VAR_USED = retries.front().second; 730 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n", 731 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId, 732 pkt->req->getPaddr()); 733 if (!sendTimingReq(pkt)) { 734 DPRINTF(GPUFetch, "failed again!\n"); 735 break; 736 } else { 737 DPRINTF(GPUFetch, "successful!\n"); 738 retries.pop_front(); 739 } 740 } 741} 742 743void 744ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 745{ 746 // There must be a way around this check to do the globalMemStart... 747 Addr tmp_vaddr = pkt->req->getVaddr(); 748 749 updatePageDivergenceDist(tmp_vaddr); 750 751 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(), 752 pkt->req->getFlags(), pkt->req->masterId(), 753 pkt->req->getPC()); 754 755 // figure out the type of the request to set read/write 756 BaseTLB::Mode TLB_mode; 757 assert(pkt->isRead() || pkt->isWrite()); 758 759 // Check write before read for atomic operations 760 // since atomic operations should use BaseTLB::Write 761 if (pkt->isWrite()){ 762 TLB_mode = BaseTLB::Write; 763 } else if (pkt->isRead()) { 764 TLB_mode = BaseTLB::Read; 765 } else { 766 fatal("pkt is not a read nor a write\n"); 767 } 768 769 tlbCycles -= curTick(); 770 ++tlbRequests; 771 772 int tlbPort_index = perLaneTLB ? index : 0; 773 774 if (shader->timingSim) { 775 if (debugSegFault) { 776 Process *p = shader->gpuTc->getProcessPtr(); 777 Addr vaddr = pkt->req->getVaddr(); 778 unsigned size = pkt->getSize(); 779 780 if ((vaddr + size - 1) % 64 < vaddr % 64) { 781 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n", 782 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr); 783 } 784 785 Addr paddr; 786 787 if (!p->pTable->translate(vaddr, paddr)) { 788 if (!p->fixupStackFault(vaddr)) { 789 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n", 790 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 791 vaddr); 792 } 793 } 794 } 795 796 // This is the SenderState needed upon return 797 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index); 798 799 // This is the senderState needed by the TLB hierarchy to function 800 TheISA::GpuTLB::TranslationState *translation_state = 801 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false, 802 pkt->senderState); 803 804 pkt->senderState = translation_state; 805 806 if (functionalTLB) { 807 tlbPort[tlbPort_index]->sendFunctional(pkt); 808 809 // update the hitLevel distribution 810 int hit_level = translation_state->hitLevel; 811 assert(hit_level != -1); 812 hitsPerTLBLevel[hit_level]++; 813 814 // New SenderState for the memory access 815 X86ISA::GpuTLB::TranslationState *sender_state = 816 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState); 817 818 delete sender_state->tlbEntry; 819 delete sender_state->saved; 820 delete sender_state; 821 822 assert(pkt->req->hasPaddr()); 823 assert(pkt->req->hasSize()); 824 825 uint8_t *tmpData = pkt->getPtr<uint8_t>(); 826 827 // this is necessary because the GPU TLB receives packets instead 828 // of requests. when the translation is complete, all relevent 829 // fields in the request will be populated, but not in the packet. 830 // here we create the new packet so we can set the size, addr, 831 // and proper flags. 832 PacketPtr oldPkt = pkt; 833 pkt = new Packet(oldPkt->req, oldPkt->cmd); 834 delete oldPkt; 835 pkt->dataStatic(tmpData); 836 837 838 // New SenderState for the memory access 839 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, 840 index, nullptr); 841 842 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index); 843 gpuDynInst->tlbHitLevel[index] = hit_level; 844 845 846 // translation is done. Schedule the mem_req_event at the 847 // appropriate cycle to send the timing memory request to ruby 848 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 849 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 850 851 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data " 852 "scheduled\n", cu_id, gpuDynInst->simdId, 853 gpuDynInst->wfSlotId, index, pkt->req->getPaddr()); 854 855 schedule(mem_req_event, curTick() + req_tick_latency); 856 } else if (tlbPort[tlbPort_index]->isStalled()) { 857 assert(tlbPort[tlbPort_index]->retries.size() > 0); 858 859 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 860 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 861 tmp_vaddr); 862 863 tlbPort[tlbPort_index]->retries.push_back(pkt); 864 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) { 865 // Stall the data port; 866 // No more packet will be issued till 867 // ruby indicates resources are freed by 868 // a recvReqRetry() call back on this port. 869 tlbPort[tlbPort_index]->stallPort(); 870 871 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 872 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 873 tmp_vaddr); 874 875 tlbPort[tlbPort_index]->retries.push_back(pkt); 876 } else { 877 DPRINTF(GPUTLB, 878 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n", 879 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr); 880 } 881 } else { 882 if (pkt->cmd == MemCmd::MemFenceReq) { 883 gpuDynInst->statusBitVector = VectorMask(0); 884 } else { 885 gpuDynInst->statusBitVector &= (~(1ll << index)); 886 } 887 888 // New SenderState for the memory access 889 delete pkt->senderState; 890 891 // Because it's atomic operation, only need TLB translation state 892 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode, 893 shader->gpuTc); 894 895 tlbPort[tlbPort_index]->sendFunctional(pkt); 896 897 // the addr of the packet is not modified, so we need to create a new 898 // packet, or otherwise the memory access will have the old virtual 899 // address sent in the translation packet, instead of the physical 900 // address returned by the translation. 901 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd); 902 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 903 904 // Translation is done. It is safe to send the packet to memory. 905 memPort[0]->sendFunctional(new_pkt); 906 907 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id, 908 gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 909 new_pkt->req->getPaddr()); 910 911 // safe_cast the senderState 912 TheISA::GpuTLB::TranslationState *sender_state = 913 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 914 915 delete sender_state->tlbEntry; 916 delete new_pkt; 917 delete pkt->senderState; 918 delete pkt->req; 919 delete pkt; 920 } 921} 922 923void 924ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 925{ 926 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 927 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 928 929 930 // New SenderState for the memory access 931 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index, 932 nullptr); 933 934 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n", 935 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 936 pkt->req->getPaddr()); 937 938 schedule(mem_req_event, curTick() + req_tick_latency); 939} 940 941void 942ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch, 943 Request* req) 944{ 945 assert(gpuDynInst->isGlobalSeg()); 946 947 if (!req) { 948 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId); 949 } 950 req->setPaddr(0); 951 if (kernelLaunch) { 952 req->setFlags(Request::KERNEL); 953 } 954 955 // for non-kernel MemFence operations, memorder flags are set depending 956 // on which type of request is currently being sent, so this 957 // should be set by the caller (e.g. if an inst has acq-rel 958 // semantics, it will send one acquire req an one release req) 959 gpuDynInst->setRequestFlags(req, kernelLaunch); 960 961 // a mem fence must correspond to an acquire/release request 962 assert(req->isAcquire() || req->isRelease()); 963 964 // create packet 965 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq); 966 967 // set packet's sender state 968 pkt->senderState = 969 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr); 970 971 // send the packet 972 sendSyncRequest(gpuDynInst, 0, pkt); 973} 974 975const char* 976ComputeUnit::DataPort::MemRespEvent::description() const 977{ 978 return "ComputeUnit memory response event"; 979} 980 981void 982ComputeUnit::DataPort::MemRespEvent::process() 983{ 984 DataPort::SenderState *sender_state = 985 safe_cast<DataPort::SenderState*>(pkt->senderState); 986 987 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 988 ComputeUnit *compute_unit = dataPort->computeUnit; 989 990 assert(gpuDynInst); 991 992 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n", 993 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 994 pkt->req->getPaddr(), dataPort->index); 995 996 Addr paddr = pkt->req->getPaddr(); 997 998 if (pkt->cmd != MemCmd::MemFenceResp) { 999 int index = gpuDynInst->memStatusVector[paddr].back(); 1000 1001 DPRINTF(GPUMem, "Response for addr %#x, index %d\n", 1002 pkt->req->getPaddr(), index); 1003 1004 gpuDynInst->memStatusVector[paddr].pop_back(); 1005 gpuDynInst->pAddr = pkt->req->getPaddr(); 1006 1007 if (pkt->isRead() || pkt->isWrite()) { 1008 1009 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) { 1010 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1011 } else { 1012 assert(gpuDynInst->statusVector[index] > 0); 1013 gpuDynInst->statusVector[index]--; 1014 1015 if (!gpuDynInst->statusVector[index]) 1016 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1017 } 1018 1019 DPRINTF(GPUMem, "bitvector is now %#x\n", 1020 gpuDynInst->statusBitVector); 1021 1022 if (gpuDynInst->statusBitVector == VectorMask(0)) { 1023 auto iter = gpuDynInst->memStatusVector.begin(); 1024 auto end = gpuDynInst->memStatusVector.end(); 1025 1026 while (iter != end) { 1027 assert(iter->second.empty()); 1028 ++iter; 1029 } 1030 1031 gpuDynInst->memStatusVector.clear(); 1032 1033 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST) 1034 gpuDynInst->statusVector.clear(); 1035 1036 if (gpuDynInst->isLoad() || gpuDynInst->isAtomic()) { 1037 assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy()); 1038 1039 compute_unit->globalMemoryPipe.getGMLdRespFIFO() 1040 .push(gpuDynInst); 1041 } else { 1042 assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy()); 1043 1044 compute_unit->globalMemoryPipe.getGMStRespFIFO() 1045 .push(gpuDynInst); 1046 } 1047 1048 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n", 1049 compute_unit->cu_id, gpuDynInst->simdId, 1050 gpuDynInst->wfSlotId); 1051 1052 // after clearing the status vectors, 1053 // see if there is a continuation to perform 1054 // the continuation may generate more work for 1055 // this memory request 1056 if (gpuDynInst->useContinuation) { 1057 assert(!gpuDynInst->isNoScope()); 1058 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1059 gpuDynInst); 1060 } 1061 } 1062 } 1063 } else { 1064 gpuDynInst->statusBitVector = VectorMask(0); 1065 1066 if (gpuDynInst->useContinuation) { 1067 assert(!gpuDynInst->isNoScope()); 1068 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1069 gpuDynInst); 1070 } 1071 } 1072 1073 delete pkt->senderState; 1074 delete pkt->req; 1075 delete pkt; 1076} 1077 1078ComputeUnit* 1079ComputeUnitParams::create() 1080{ 1081 return new ComputeUnit(this); 1082} 1083 1084bool 1085ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt) 1086{ 1087 Addr line = pkt->req->getPaddr(); 1088 1089 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id, 1090 pkt->req->getVaddr(), line); 1091 1092 assert(pkt->senderState); 1093 computeUnit->tlbCycles += curTick(); 1094 1095 // pop off the TLB translation state 1096 TheISA::GpuTLB::TranslationState *translation_state = 1097 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1098 1099 // no PageFaults are permitted for data accesses 1100 if (!translation_state->tlbEntry->valid) { 1101 DTLBPort::SenderState *sender_state = 1102 safe_cast<DTLBPort::SenderState*>(translation_state->saved); 1103 1104 Wavefront *w M5_VAR_USED = 1105 computeUnit->wfList[sender_state->_gpuDynInst->simdId] 1106 [sender_state->_gpuDynInst->wfSlotId]; 1107 1108 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId, 1109 pkt->req->getVaddr()); 1110 } 1111 1112 assert(translation_state->tlbEntry->valid); 1113 1114 // update the hitLevel distribution 1115 int hit_level = translation_state->hitLevel; 1116 computeUnit->hitsPerTLBLevel[hit_level]++; 1117 1118 delete translation_state->tlbEntry; 1119 assert(!translation_state->ports.size()); 1120 pkt->senderState = translation_state->saved; 1121 1122 // for prefetch pkt 1123 BaseTLB::Mode TLB_mode = translation_state->tlbMode; 1124 1125 delete translation_state; 1126 1127 // use the original sender state to know how to close this transaction 1128 DTLBPort::SenderState *sender_state = 1129 safe_cast<DTLBPort::SenderState*>(pkt->senderState); 1130 1131 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1132 int mp_index = sender_state->portIndex; 1133 Addr vaddr = pkt->req->getVaddr(); 1134 gpuDynInst->memStatusVector[line].push_back(mp_index); 1135 gpuDynInst->tlbHitLevel[mp_index] = hit_level; 1136 1137 MemCmd requestCmd; 1138 1139 if (pkt->cmd == MemCmd::ReadResp) { 1140 requestCmd = MemCmd::ReadReq; 1141 } else if (pkt->cmd == MemCmd::WriteResp) { 1142 requestCmd = MemCmd::WriteReq; 1143 } else if (pkt->cmd == MemCmd::SwapResp) { 1144 requestCmd = MemCmd::SwapReq; 1145 } else { 1146 panic("unsupported response to request conversion %s\n", 1147 pkt->cmd.toString()); 1148 } 1149 1150 if (computeUnit->prefetchDepth) { 1151 int simdId = gpuDynInst->simdId; 1152 int wfSlotId = gpuDynInst->wfSlotId; 1153 Addr last = 0; 1154 1155 switch(computeUnit->prefetchType) { 1156 case Enums::PF_CU: 1157 last = computeUnit->lastVaddrCU[mp_index]; 1158 break; 1159 case Enums::PF_PHASE: 1160 last = computeUnit->lastVaddrSimd[simdId][mp_index]; 1161 break; 1162 case Enums::PF_WF: 1163 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index]; 1164 default: 1165 break; 1166 } 1167 1168 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n", 1169 computeUnit->cu_id, simdId, wfSlotId, mp_index, last); 1170 1171 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) - 1172 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift 1173 : 0; 1174 1175 DPRINTF(GPUPrefetch, "Stride is %d\n", stride); 1176 1177 computeUnit->lastVaddrCU[mp_index] = vaddr; 1178 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr; 1179 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr; 1180 1181 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ? 1182 computeUnit->prefetchStride: stride; 1183 1184 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr, 1185 computeUnit->cu_id, simdId, wfSlotId, mp_index); 1186 1187 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr); 1188 1189 // Prefetch Next few pages atomically 1190 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) { 1191 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride, 1192 vaddr+stride*pf*TheISA::PageBytes); 1193 1194 if (!stride) 1195 break; 1196 1197 Request *prefetch_req = new Request(0, vaddr + stride * pf * 1198 TheISA::PageBytes, 1199 sizeof(uint8_t), 0, 1200 computeUnit->masterId(), 1201 0, 0, 0); 1202 1203 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd); 1204 uint8_t foo = 0; 1205 prefetch_pkt->dataStatic(&foo); 1206 1207 // Because it's atomic operation, only need TLB translation state 1208 prefetch_pkt->senderState = 1209 new TheISA::GpuTLB::TranslationState(TLB_mode, 1210 computeUnit->shader->gpuTc, 1211 true); 1212 1213 // Currently prefetches are zero-latency, hence the sendFunctional 1214 sendFunctional(prefetch_pkt); 1215 1216 /* safe_cast the senderState */ 1217 TheISA::GpuTLB::TranslationState *tlb_state = 1218 safe_cast<TheISA::GpuTLB::TranslationState*>( 1219 prefetch_pkt->senderState); 1220 1221 1222 delete tlb_state->tlbEntry; 1223 delete tlb_state; 1224 delete prefetch_pkt->req; 1225 delete prefetch_pkt; 1226 } 1227 } 1228 1229 // First we must convert the response cmd back to a request cmd so that 1230 // the request can be sent through the cu's master port 1231 PacketPtr new_pkt = new Packet(pkt->req, requestCmd); 1232 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 1233 delete pkt->senderState; 1234 delete pkt; 1235 1236 // New SenderState for the memory access 1237 new_pkt->senderState = 1238 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index, 1239 nullptr); 1240 1241 // translation is done. Schedule the mem_req_event at the appropriate 1242 // cycle to send the timing memory request to ruby 1243 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 1244 new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index], 1245 new_pkt); 1246 1247 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n", 1248 computeUnit->cu_id, gpuDynInst->simdId, 1249 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr()); 1250 1251 computeUnit->schedule(mem_req_event, curTick() + 1252 computeUnit->req_tick_latency); 1253 1254 return true; 1255} 1256 1257const char* 1258ComputeUnit::DataPort::MemReqEvent::description() const 1259{ 1260 return "ComputeUnit memory request event"; 1261} 1262 1263void 1264ComputeUnit::DataPort::MemReqEvent::process() 1265{ 1266 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 1267 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1268 ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit; 1269 1270 if (!(dataPort->sendTimingReq(pkt))) { 1271 dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst)); 1272 1273 DPRINTF(GPUPort, 1274 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n", 1275 compute_unit->cu_id, gpuDynInst->simdId, 1276 gpuDynInst->wfSlotId, dataPort->index, 1277 pkt->req->getPaddr()); 1278 } else { 1279 DPRINTF(GPUPort, 1280 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n", 1281 compute_unit->cu_id, gpuDynInst->simdId, 1282 gpuDynInst->wfSlotId, dataPort->index, 1283 pkt->req->getPaddr()); 1284 } 1285} 1286 1287/* 1288 * The initial translation request could have been rejected, 1289 * if <retries> queue is not Retry sending the translation 1290 * request. sendRetry() is called from the peer port whenever 1291 * a translation completes. 1292 */ 1293void 1294ComputeUnit::DTLBPort::recvReqRetry() 1295{ 1296 int len = retries.size(); 1297 1298 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n", 1299 computeUnit->cu_id, len); 1300 1301 assert(len > 0); 1302 assert(isStalled()); 1303 // recvReqRetry is an indication that the resource on which this 1304 // port was stalling on is freed. So, remove the stall first 1305 unstallPort(); 1306 1307 for (int i = 0; i < len; ++i) { 1308 PacketPtr pkt = retries.front(); 1309 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1310 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr); 1311 1312 if (!sendTimingReq(pkt)) { 1313 // Stall port 1314 stallPort(); 1315 DPRINTF(GPUTLB, ": failed again\n"); 1316 break; 1317 } else { 1318 DPRINTF(GPUTLB, ": successful\n"); 1319 retries.pop_front(); 1320 } 1321 } 1322} 1323 1324bool 1325ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt) 1326{ 1327 Addr line M5_VAR_USED = pkt->req->getPaddr(); 1328 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n", 1329 computeUnit->cu_id, pkt->req->getVaddr(), line); 1330 1331 assert(pkt->senderState); 1332 1333 // pop off the TLB translation state 1334 TheISA::GpuTLB::TranslationState *translation_state = 1335 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1336 1337 bool success = translation_state->tlbEntry->valid; 1338 delete translation_state->tlbEntry; 1339 assert(!translation_state->ports.size()); 1340 pkt->senderState = translation_state->saved; 1341 delete translation_state; 1342 1343 // use the original sender state to know how to close this transaction 1344 ITLBPort::SenderState *sender_state = 1345 safe_cast<ITLBPort::SenderState*>(pkt->senderState); 1346 1347 // get the wavefront associated with this translation request 1348 Wavefront *wavefront = sender_state->wavefront; 1349 delete pkt->senderState; 1350 1351 if (success) { 1352 // pkt is reused in fetch(), don't delete it here. However, we must 1353 // reset the command to be a request so that it can be sent through 1354 // the cu's master port 1355 assert(pkt->cmd == MemCmd::ReadResp); 1356 pkt->cmd = MemCmd::ReadReq; 1357 1358 computeUnit->fetchStage.fetch(pkt, wavefront); 1359 } else { 1360 if (wavefront->dropFetch) { 1361 assert(wavefront->instructionBuffer.empty()); 1362 wavefront->dropFetch = false; 1363 } 1364 1365 wavefront->pendingFetch = 0; 1366 } 1367 1368 return true; 1369} 1370 1371/* 1372 * The initial translation request could have been rejected, if 1373 * <retries> queue is not empty. Retry sending the translation 1374 * request. sendRetry() is called from the peer port whenever 1375 * a translation completes. 1376 */ 1377void 1378ComputeUnit::ITLBPort::recvReqRetry() 1379{ 1380 1381 int len = retries.size(); 1382 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len); 1383 1384 assert(len > 0); 1385 assert(isStalled()); 1386 1387 // recvReqRetry is an indication that the resource on which this 1388 // port was stalling on is freed. So, remove the stall first 1389 unstallPort(); 1390 1391 for (int i = 0; i < len; ++i) { 1392 PacketPtr pkt = retries.front(); 1393 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1394 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr); 1395 1396 if (!sendTimingReq(pkt)) { 1397 stallPort(); // Stall port 1398 DPRINTF(GPUTLB, ": failed again\n"); 1399 break; 1400 } else { 1401 DPRINTF(GPUTLB, ": successful\n"); 1402 retries.pop_front(); 1403 } 1404 } 1405} 1406 1407void 1408ComputeUnit::regStats() 1409{ 1410 MemObject::regStats(); 1411 1412 vALUInsts 1413 .name(name() + ".valu_insts") 1414 .desc("Number of vector ALU insts issued.") 1415 ; 1416 vALUInstsPerWF 1417 .name(name() + ".valu_insts_per_wf") 1418 .desc("The avg. number of vector ALU insts issued per-wavefront.") 1419 ; 1420 sALUInsts 1421 .name(name() + ".salu_insts") 1422 .desc("Number of scalar ALU insts issued.") 1423 ; 1424 sALUInstsPerWF 1425 .name(name() + ".salu_insts_per_wf") 1426 .desc("The avg. number of scalar ALU insts issued per-wavefront.") 1427 ; 1428 instCyclesVALU 1429 .name(name() + ".inst_cycles_valu") 1430 .desc("Number of cycles needed to execute VALU insts.") 1431 ; 1432 instCyclesSALU 1433 .name(name() + ".inst_cycles_salu") 1434 .desc("Number of cycles needed to execute SALU insts.") 1435 ; 1436 threadCyclesVALU 1437 .name(name() + ".thread_cycles_valu") 1438 .desc("Number of thread cycles used to execute vector ALU ops. " 1439 "Similar to instCyclesVALU but multiplied by the number of " 1440 "active threads.") 1441 ; 1442 vALUUtilization 1443 .name(name() + ".valu_utilization") 1444 .desc("Percentage of active vector ALU threads in a wave.") 1445 ; 1446 ldsNoFlatInsts 1447 .name(name() + ".lds_no_flat_insts") 1448 .desc("Number of LDS insts issued, not including FLAT " 1449 "accesses that resolve to LDS.") 1450 ; 1451 ldsNoFlatInstsPerWF 1452 .name(name() + ".lds_no_flat_insts_per_wf") 1453 .desc("The avg. number of LDS insts (not including FLAT " 1454 "accesses that resolve to LDS) per-wavefront.") 1455 ; 1456 flatVMemInsts 1457 .name(name() + ".flat_vmem_insts") 1458 .desc("The number of FLAT insts that resolve to vmem issued.") 1459 ; 1460 flatVMemInstsPerWF 1461 .name(name() + ".flat_vmem_insts_per_wf") 1462 .desc("The average number of FLAT insts that resolve to vmem " 1463 "issued per-wavefront.") 1464 ; 1465 flatLDSInsts 1466 .name(name() + ".flat_lds_insts") 1467 .desc("The number of FLAT insts that resolve to LDS issued.") 1468 ; 1469 flatLDSInstsPerWF 1470 .name(name() + ".flat_lds_insts_per_wf") 1471 .desc("The average number of FLAT insts that resolve to LDS " 1472 "issued per-wavefront.") 1473 ; 1474 vectorMemWrites 1475 .name(name() + ".vector_mem_writes") 1476 .desc("Number of vector mem write insts (excluding FLAT insts).") 1477 ; 1478 vectorMemWritesPerWF 1479 .name(name() + ".vector_mem_writes_per_wf") 1480 .desc("The average number of vector mem write insts " 1481 "(excluding FLAT insts) per-wavefront.") 1482 ; 1483 vectorMemReads 1484 .name(name() + ".vector_mem_reads") 1485 .desc("Number of vector mem read insts (excluding FLAT insts).") 1486 ; 1487 vectorMemReadsPerWF 1488 .name(name() + ".vector_mem_reads_per_wf") 1489 .desc("The avg. number of vector mem read insts (excluding " 1490 "FLAT insts) per-wavefront.") 1491 ; 1492 scalarMemWrites 1493 .name(name() + ".scalar_mem_writes") 1494 .desc("Number of scalar mem write insts.") 1495 ; 1496 scalarMemWritesPerWF 1497 .name(name() + ".scalar_mem_writes_per_wf") 1498 .desc("The average number of scalar mem write insts per-wavefront.") 1499 ; 1500 scalarMemReads 1501 .name(name() + ".scalar_mem_reads") 1502 .desc("Number of scalar mem read insts.") 1503 ; 1504 scalarMemReadsPerWF 1505 .name(name() + ".scalar_mem_reads_per_wf") 1506 .desc("The average number of scalar mem read insts per-wavefront.") 1507 ; 1508 1509 vALUInstsPerWF = vALUInsts / completedWfs; 1510 sALUInstsPerWF = sALUInsts / completedWfs; 1511 vALUUtilization = (threadCyclesVALU / (64 * instCyclesVALU)) * 100; 1512 ldsNoFlatInstsPerWF = ldsNoFlatInsts / completedWfs; 1513 flatVMemInstsPerWF = flatVMemInsts / completedWfs; 1514 flatLDSInstsPerWF = flatLDSInsts / completedWfs; 1515 vectorMemWritesPerWF = vectorMemWrites / completedWfs; 1516 vectorMemReadsPerWF = vectorMemReads / completedWfs; 1517 scalarMemWritesPerWF = scalarMemWrites / completedWfs; 1518 scalarMemReadsPerWF = scalarMemReads / completedWfs; 1519 1520 tlbCycles 1521 .name(name() + ".tlb_cycles") 1522 .desc("total number of cycles for all uncoalesced requests") 1523 ; 1524 1525 tlbRequests 1526 .name(name() + ".tlb_requests") 1527 .desc("number of uncoalesced requests") 1528 ; 1529 1530 tlbLatency 1531 .name(name() + ".avg_translation_latency") 1532 .desc("Avg. translation latency for data translations") 1533 ; 1534 1535 tlbLatency = tlbCycles / tlbRequests; 1536 1537 hitsPerTLBLevel 1538 .init(4) 1539 .name(name() + ".TLB_hits_distribution") 1540 .desc("TLB hits distribution (0 for page table, x for Lx-TLB") 1541 ; 1542 1543 // fixed number of TLB levels 1544 for (int i = 0; i < 4; ++i) { 1545 if (!i) 1546 hitsPerTLBLevel.subname(i,"page_table"); 1547 else 1548 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i)); 1549 } 1550 1551 execRateDist 1552 .init(0, 10, 2) 1553 .name(name() + ".inst_exec_rate") 1554 .desc("Instruction Execution Rate: Number of executed vector " 1555 "instructions per cycle") 1556 ; 1557 1558 ldsBankConflictDist 1559 .init(0, wfSize(), 2) 1560 .name(name() + ".lds_bank_conflicts") 1561 .desc("Number of bank conflicts per LDS memory packet") 1562 ; 1563 1564 ldsBankAccesses 1565 .name(name() + ".lds_bank_access_cnt") 1566 .desc("Total number of LDS bank accesses") 1567 ; 1568 1569 pageDivergenceDist 1570 // A wavefront can touch up to N pages per memory instruction where 1571 // N is equal to the wavefront size 1572 // The number of pages per bin can be configured (here it's 4). 1573 .init(1, wfSize(), 4) 1574 .name(name() + ".page_divergence_dist") 1575 .desc("pages touched per wf (over all mem. instr.)") 1576 ; 1577 1578 controlFlowDivergenceDist 1579 .init(1, wfSize(), 4) 1580 .name(name() + ".warp_execution_dist") 1581 .desc("number of lanes active per instruction (oval all instructions)") 1582 ; 1583 1584 activeLanesPerGMemInstrDist 1585 .init(1, wfSize(), 4) 1586 .name(name() + ".gmem_lanes_execution_dist") 1587 .desc("number of active lanes per global memory instruction") 1588 ; 1589 1590 activeLanesPerLMemInstrDist 1591 .init(1, wfSize(), 4) 1592 .name(name() + ".lmem_lanes_execution_dist") 1593 .desc("number of active lanes per local memory instruction") 1594 ; 1595 1596 numInstrExecuted 1597 .name(name() + ".num_instr_executed") 1598 .desc("number of instructions executed") 1599 ; 1600 1601 numVecOpsExecuted 1602 .name(name() + ".num_vec_ops_executed") 1603 .desc("number of vec ops executed (e.g. WF size/inst)") 1604 ; 1605 1606 totalCycles 1607 .name(name() + ".num_total_cycles") 1608 .desc("number of cycles the CU ran for") 1609 ; 1610 1611 ipc 1612 .name(name() + ".ipc") 1613 .desc("Instructions per cycle (this CU only)") 1614 ; 1615 1616 vpc 1617 .name(name() + ".vpc") 1618 .desc("Vector Operations per cycle (this CU only)") 1619 ; 1620 1621 numALUInstsExecuted 1622 .name(name() + ".num_alu_insts_executed") 1623 .desc("Number of dynamic non-GM memory insts executed") 1624 ; 1625 1626 wgBlockedDueLdsAllocation 1627 .name(name() + ".wg_blocked_due_lds_alloc") 1628 .desc("Workgroup blocked due to LDS capacity") 1629 ; 1630 1631 ipc = numInstrExecuted / totalCycles; 1632 vpc = numVecOpsExecuted / totalCycles; 1633 1634 numTimesWgBlockedDueVgprAlloc 1635 .name(name() + ".times_wg_blocked_due_vgpr_alloc") 1636 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD") 1637 ; 1638 1639 dynamicGMemInstrCnt 1640 .name(name() + ".global_mem_instr_cnt") 1641 .desc("dynamic global memory instructions count") 1642 ; 1643 1644 dynamicLMemInstrCnt 1645 .name(name() + ".local_mem_instr_cnt") 1646 .desc("dynamic local memory intruction count") 1647 ; 1648 1649 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt - 1650 dynamicLMemInstrCnt; 1651 1652 completedWfs 1653 .name(name() + ".num_completed_wfs") 1654 .desc("number of completed wavefronts") 1655 ; 1656 1657 numCASOps 1658 .name(name() + ".num_CAS_ops") 1659 .desc("number of compare and swap operations") 1660 ; 1661 1662 numFailedCASOps 1663 .name(name() + ".num_failed_CAS_ops") 1664 .desc("number of compare and swap operations that failed") 1665 ; 1666 1667 // register stats of pipeline stages 1668 fetchStage.regStats(); 1669 scoreboardCheckStage.regStats(); 1670 scheduleStage.regStats(); 1671 execStage.regStats(); 1672 1673 // register stats of memory pipeline 1674 globalMemoryPipe.regStats(); 1675 localMemoryPipe.regStats(); 1676} 1677 1678void 1679ComputeUnit::updateInstStats(GPUDynInstPtr gpuDynInst) 1680{ 1681 if (gpuDynInst->isScalar()) { 1682 if (gpuDynInst->isALU() && !gpuDynInst->isWaitcnt()) { 1683 sALUInsts++; 1684 instCyclesSALU++; 1685 } else if (gpuDynInst->isLoad()) { 1686 scalarMemReads++; 1687 } else if (gpuDynInst->isStore()) { 1688 scalarMemWrites++; 1689 } 1690 } else { 1691 if (gpuDynInst->isALU()) { 1692 vALUInsts++; 1693 instCyclesVALU++; 1694 threadCyclesVALU += gpuDynInst->wavefront()->execMask().count(); 1695 } else if (gpuDynInst->isFlat()) { 1696 if (gpuDynInst->isLocalMem()) { 1697 flatLDSInsts++; 1698 } else { 1699 flatVMemInsts++; 1700 } 1701 } else if (gpuDynInst->isLocalMem()) { 1702 ldsNoFlatInsts++; 1703 } else if (gpuDynInst->isLoad()) { 1704 vectorMemReads++; 1705 } else if (gpuDynInst->isStore()) { 1706 vectorMemWrites++; 1707 } 1708 } 1709} 1710 1711void 1712ComputeUnit::updatePageDivergenceDist(Addr addr) 1713{ 1714 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes); 1715 1716 if (!pagesTouched.count(virt_page_addr)) 1717 pagesTouched[virt_page_addr] = 1; 1718 else 1719 pagesTouched[virt_page_addr]++; 1720} 1721 1722void 1723ComputeUnit::CUExitCallback::process() 1724{ 1725 if (computeUnit->countPages) { 1726 std::ostream *page_stat_file = 1727 simout.create(computeUnit->name().c_str())->stream(); 1728 1729 *page_stat_file << "page, wavefront accesses, workitem accesses" << 1730 std::endl; 1731 1732 for (auto iter : computeUnit->pageAccesses) { 1733 *page_stat_file << std::hex << iter.first << ","; 1734 *page_stat_file << std::dec << iter.second.first << ","; 1735 *page_stat_file << std::dec << iter.second.second << std::endl; 1736 } 1737 } 1738 } 1739 1740bool 1741ComputeUnit::isDone() const 1742{ 1743 for (int i = 0; i < numSIMDs; ++i) { 1744 if (!isSimdDone(i)) { 1745 return false; 1746 } 1747 } 1748 1749 bool glbMemBusRdy = true; 1750 for (int j = 0; j < numGlbMemUnits; ++j) { 1751 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy(); 1752 } 1753 bool locMemBusRdy = true; 1754 for (int j = 0; j < numLocMemUnits; ++j) { 1755 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy(); 1756 } 1757 1758 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() || 1759 !globalMemoryPipe.isGMStRespFIFOWrRdy() || 1760 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy() 1761 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() || 1762 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) { 1763 return false; 1764 } 1765 1766 return true; 1767} 1768 1769int32_t 1770ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const 1771{ 1772 return lds.getRefCounter(dispatchId, wgId); 1773} 1774 1775bool 1776ComputeUnit::isSimdDone(uint32_t simdId) const 1777{ 1778 assert(simdId < numSIMDs); 1779 1780 for (int i=0; i < numGlbMemUnits; ++i) { 1781 if (!vrfToGlobalMemPipeBus[i].rdy()) 1782 return false; 1783 } 1784 for (int i=0; i < numLocMemUnits; ++i) { 1785 if (!vrfToLocalMemPipeBus[i].rdy()) 1786 return false; 1787 } 1788 if (!aluPipe[simdId].rdy()) { 1789 return false; 1790 } 1791 1792 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){ 1793 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) { 1794 return false; 1795 } 1796 } 1797 1798 return true; 1799} 1800 1801/** 1802 * send a general request to the LDS 1803 * make sure to look at the return value here as your request might be 1804 * NACK'd and returning false means that you have to have some backup plan 1805 */ 1806bool 1807ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst) 1808{ 1809 // this is just a request to carry the GPUDynInstPtr 1810 // back and forth 1811 Request *newRequest = new Request(); 1812 newRequest->setPaddr(0x0); 1813 1814 // ReadReq is not evaluted by the LDS but the Packet ctor requires this 1815 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq); 1816 1817 // This is the SenderState needed upon return 1818 newPacket->senderState = new LDSPort::SenderState(gpuDynInst); 1819 1820 return ldsPort->sendTimingReq(newPacket); 1821} 1822 1823/** 1824 * get the result of packets sent to the LDS when they return 1825 */ 1826bool 1827ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet) 1828{ 1829 const ComputeUnit::LDSPort::SenderState *senderState = 1830 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState); 1831 1832 fatal_if(!senderState, "did not get the right sort of sender state"); 1833 1834 GPUDynInstPtr gpuDynInst = senderState->getMemInst(); 1835 1836 delete packet->senderState; 1837 delete packet->req; 1838 delete packet; 1839 1840 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst); 1841 return true; 1842} 1843 1844/** 1845 * attempt to send this packet, either the port is already stalled, the request 1846 * is nack'd and must stall or the request goes through 1847 * when a request cannot be sent, add it to the retries queue 1848 */ 1849bool 1850ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt) 1851{ 1852 ComputeUnit::LDSPort::SenderState *sender_state = 1853 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState); 1854 fatal_if(!sender_state, "packet without a valid sender state"); 1855 1856 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst(); 1857 1858 if (isStalled()) { 1859 fatal_if(retries.empty(), "must have retries waiting to be stalled"); 1860 1861 retries.push(pkt); 1862 1863 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n", 1864 computeUnit->cu_id, gpuDynInst->simdId, 1865 gpuDynInst->wfSlotId); 1866 return false; 1867 } else if (!MasterPort::sendTimingReq(pkt)) { 1868 // need to stall the LDS port until a recvReqRetry() is received 1869 // this indicates that there is more space 1870 stallPort(); 1871 retries.push(pkt); 1872 1873 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n", 1874 computeUnit->cu_id, gpuDynInst->simdId, 1875 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1876 return false; 1877 } else { 1878 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n", 1879 computeUnit->cu_id, gpuDynInst->simdId, 1880 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1881 return true; 1882 } 1883} 1884 1885/** 1886 * the bus is telling the port that there is now space so retrying stalled 1887 * requests should work now 1888 * this allows the port to have a request be nack'd and then have the receiver 1889 * say when there is space, rather than simply retrying the send every cycle 1890 */ 1891void 1892ComputeUnit::LDSPort::recvReqRetry() 1893{ 1894 auto queueSize = retries.size(); 1895 1896 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n", 1897 computeUnit->cu_id, queueSize); 1898 1899 fatal_if(queueSize < 1, 1900 "why was there a recvReqRetry() with no pending reqs?"); 1901 fatal_if(!isStalled(), 1902 "recvReqRetry() happened when the port was not stalled"); 1903 1904 unstallPort(); 1905 1906 while (!retries.empty()) { 1907 PacketPtr packet = retries.front(); 1908 1909 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id); 1910 1911 if (!MasterPort::sendTimingReq(packet)) { 1912 // Stall port 1913 stallPort(); 1914 DPRINTF(GPUPort, ": LDS send failed again\n"); 1915 break; 1916 } else { 1917 DPRINTF(GPUTLB, ": LDS send successful\n"); 1918 retries.pop(); 1919 } 1920 } 1921} 1922