412 ++cnt; 413 } 414 } 415 ++barrier_id; 416} 417 418int 419ComputeUnit::ReadyWorkgroup(NDRange *ndr) 420{ 421 // Get true size of workgroup (after clamping to grid size) 422 int trueWgSize[3]; 423 int trueWgSizeTotal = 1; 424 425 for (int d = 0; d < 3; ++d) { 426 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] - 427 ndr->wgId[d] * ndr->q.wgSize[d]); 428 429 trueWgSizeTotal *= trueWgSize[d]; 430 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]); 431 } 432 433 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal); 434 435 // calculate the number of 32-bit vector registers required by each 436 // work item of the work group 437 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 438 bool vregAvail = true; 439 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize(); 440 int freeWfSlots = 0; 441 // check if the total number of VGPRs required by all WFs of the WG 442 // fit in the VRFs of all SIMD units 443 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd)); 444 int numMappedWfs = 0; 445 std::vector<int> numWfsPerSimd; 446 numWfsPerSimd.resize(numSIMDs, 0); 447 // find how many free WF slots we have across all SIMDs 448 for (int j = 0; j < shader->n_wf; ++j) { 449 for (int i = 0; i < numSIMDs; ++i) { 450 if (wfList[i][j]->status == Wavefront::S_STOPPED) { 451 // count the number of free WF slots 452 ++freeWfSlots; 453 if (numMappedWfs < numWfs) { 454 // count the WFs to be assigned per SIMD 455 numWfsPerSimd[i]++; 456 } 457 numMappedWfs++; 458 } 459 } 460 } 461 462 // if there are enough free WF slots then find if there are enough 463 // free VGPRs per SIMD based on the WF->SIMD mapping 464 if (freeWfSlots >= numWfs) { 465 for (int j = 0; j < numSIMDs; ++j) { 466 // find if there are enough free VGPR regions in the SIMD's VRF 467 // to accommodate the WFs of the new WG that would be mapped to 468 // this SIMD unit 469 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j], 470 vregDemandPerWI); 471 472 // stop searching if there is at least one SIMD 473 // whose VRF does not have enough free VGPR pools. 474 // This is because a WG is scheduled only if ALL 475 // of its WFs can be scheduled 476 if (!vregAvail) 477 break; 478 } 479 } 480 481 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n", 482 freeWfSlots, vregAvail); 483 484 if (!vregAvail) { 485 ++numTimesWgBlockedDueVgprAlloc; 486 } 487 488 // Return true if enough WF slots to submit workgroup and if there are 489 // enough VGPRs to schedule all WFs to their SIMD units 490 if (!lds.canReserve(ndr->q.ldsSize)) { 491 wgBlockedDueLdsAllocation++; 492 } 493 494 // Return true if (a) there are enough free WF slots to submit 495 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their 496 // SIMD units and (c) if there is enough space in LDS 497 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize); 498} 499 500int 501ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots) 502{ 503 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id); 504 int ccnt = 0; 505 506 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) { 507 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) { 508 Wavefront *w = wfList[i_simd][i_wf]; 509 510 if (w->status == Wavefront::S_RUNNING) { 511 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf); 512 513 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n", 514 w->barrier_id, _barrier_id); 515 516 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n", 517 w->barrier_cnt, bcnt); 518 } 519 520 if (w->status == Wavefront::S_RUNNING && 521 w->barrier_id == _barrier_id && w->barrier_cnt == bcnt && 522 !w->outstanding_reqs) { 523 ++ccnt; 524 525 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to " 526 "%d\n", i_simd, i_wf, ccnt); 527 } 528 } 529 } 530 531 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n", 532 cu_id, ccnt, bslots); 533 534 return ccnt == bslots; 535} 536 537// Check if the current wavefront is blocked on additional resources. 538bool 539ComputeUnit::cedeSIMD(int simdId, int wfSlotId) 540{ 541 bool cede = false; 542 543 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld 544 // magic instructions will impact the scheduling of wavefronts 545 if (xact_cas_mode) { 546 /* 547 * When a wavefront calls xact_cas_ld, it adds itself to a per address 548 * queue. All per address queues are managed by the xactCasLoadMap. 549 * 550 * A wavefront is not blocked if: it is not in ANY per address queue or 551 * if it is at the head of a per address queue. 552 */ 553 for (auto itMap : xactCasLoadMap) { 554 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue; 555 556 if (!curWaveIDQueue.empty()) { 557 for (auto it : curWaveIDQueue) { 558 waveIdentifier cur_wave = it; 559 560 if (cur_wave.simdId == simdId && 561 cur_wave.wfSlotId == wfSlotId) { 562 // 2 possibilities 563 // 1: this WF has a green light 564 // 2: another WF has a green light 565 waveIdentifier owner_wave = curWaveIDQueue.front(); 566 567 if (owner_wave.simdId != cur_wave.simdId || 568 owner_wave.wfSlotId != cur_wave.wfSlotId) { 569 // possibility 2 570 cede = true; 571 break; 572 } else { 573 // possibility 1 574 break; 575 } 576 } 577 } 578 } 579 } 580 } 581 582 return cede; 583} 584 585// Execute one clock worth of work on the ComputeUnit. 586void 587ComputeUnit::exec() 588{ 589 updateEvents(); 590 // Execute pipeline stages in reverse order to simulate 591 // the pipeline latency 592 globalMemoryPipe.exec(); 593 localMemoryPipe.exec(); 594 execStage.exec(); 595 scheduleStage.exec(); 596 scoreboardCheckStage.exec(); 597 fetchStage.exec(); 598 599 totalCycles++; 600} 601 602void 603ComputeUnit::init() 604{ 605 // Initialize CU Bus models 606 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 607 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 608 nextGlbMemBus = 0; 609 nextLocMemBus = 0; 610 fatal_if(numGlbMemUnits > 1, 611 "No support for multiple Global Memory Pipelines exists!!!"); 612 vrfToGlobalMemPipeBus.resize(numGlbMemUnits); 613 for (int j = 0; j < numGlbMemUnits; ++j) { 614 vrfToGlobalMemPipeBus[j] = WaitClass(); 615 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 616 } 617 618 fatal_if(numLocMemUnits > 1, 619 "No support for multiple Local Memory Pipelines exists!!!"); 620 vrfToLocalMemPipeBus.resize(numLocMemUnits); 621 for (int j = 0; j < numLocMemUnits; ++j) { 622 vrfToLocalMemPipeBus[j] = WaitClass(); 623 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 624 } 625 vectorRegsReserved.resize(numSIMDs, 0); 626 aluPipe.resize(numSIMDs); 627 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits); 628 629 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) { 630 wfWait[i] = WaitClass(); 631 wfWait[i].init(&shader->tick_cnt, shader->ticks(1)); 632 } 633 634 for (int i = 0; i < numSIMDs; ++i) { 635 aluPipe[i] = WaitClass(); 636 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1)); 637 } 638 639 // Setup space for call args 640 for (int j = 0; j < numSIMDs; ++j) { 641 for (int i = 0; i < shader->n_wf; ++i) { 642 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize); 643 } 644 } 645 646 // Initializing pipeline resources 647 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits); 648 waveStatusList.resize(numSIMDs); 649 650 for (int j = 0; j < numSIMDs; ++j) { 651 for (int i = 0; i < shader->n_wf; ++i) { 652 waveStatusList[j].push_back( 653 std::make_pair(wfList[j][i], BLOCKED)); 654 } 655 } 656 657 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) { 658 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY)); 659 } 660 661 fetchStage.init(this); 662 scoreboardCheckStage.init(this); 663 scheduleStage.init(this); 664 execStage.init(this); 665 globalMemoryPipe.init(this); 666 localMemoryPipe.init(this); 667 // initialize state for statistics calculation 668 vectorAluInstAvail.resize(numSIMDs, false); 669 shrMemInstAvail = 0; 670 glbMemInstAvail = 0; 671} 672 673bool 674ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt) 675{ 676 // Ruby has completed the memory op. Schedule the mem_resp_event at the 677 // appropriate cycle to process the timing memory response 678 // This delay represents the pipeline delay 679 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 680 int index = sender_state->port_index; 681 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 682 683 // Is the packet returned a Kernel End or Barrier 684 if (pkt->req->isKernel() && pkt->req->isRelease()) { 685 Wavefront *w = 686 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId]; 687 688 // Check if we are waiting on Kernel End Release 689 if (w->status == Wavefront::S_RETURNING) { 690 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n", 691 computeUnit->cu_id, w->simdId, w->wfSlotId, 692 w->wfDynId, w->kern_id); 693 694 computeUnit->shader->dispatcher->notifyWgCompl(w); 695 w->status = Wavefront::S_STOPPED; 696 } else { 697 w->outstanding_reqs--; 698 } 699 700 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n", 701 computeUnit->cu_id, gpuDynInst->simdId, 702 gpuDynInst->wfSlotId, w->barrier_cnt); 703 704 if (gpuDynInst->useContinuation) { 705 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 706 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 707 gpuDynInst); 708 } 709 710 delete pkt->senderState; 711 delete pkt->req; 712 delete pkt; 713 return true; 714 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) { 715 if (gpuDynInst->useContinuation) { 716 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 717 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 718 gpuDynInst); 719 } 720 721 delete pkt->senderState; 722 delete pkt->req; 723 delete pkt; 724 return true; 725 } 726 727 ComputeUnit::DataPort::MemRespEvent *mem_resp_event = 728 new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index], 729 pkt); 730 731 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n", 732 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 733 index, pkt->req->getPaddr()); 734 735 computeUnit->schedule(mem_resp_event, 736 curTick() + computeUnit->resp_tick_latency); 737 return true; 738} 739 740void 741ComputeUnit::DataPort::recvReqRetry() 742{ 743 int len = retries.size(); 744 745 assert(len > 0); 746 747 for (int i = 0; i < len; ++i) { 748 PacketPtr pkt = retries.front().first; 749 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second; 750 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n", 751 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 752 pkt->req->getPaddr()); 753 754 /** Currently Ruby can return false due to conflicts for the particular 755 * cache block or address. Thus other requests should be allowed to 756 * pass and the data port should expect multiple retries. */ 757 if (!sendTimingReq(pkt)) { 758 DPRINTF(GPUMem, "failed again!\n"); 759 break; 760 } else { 761 DPRINTF(GPUMem, "successful!\n"); 762 retries.pop_front(); 763 } 764 } 765} 766 767bool 768ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt) 769{ 770 computeUnit->fetchStage.processFetchReturn(pkt); 771 772 return true; 773} 774 775void 776ComputeUnit::SQCPort::recvReqRetry() 777{ 778 int len = retries.size(); 779 780 assert(len > 0); 781 782 for (int i = 0; i < len; ++i) { 783 PacketPtr pkt = retries.front().first; 784 Wavefront *wavefront M5_VAR_USED = retries.front().second; 785 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n", 786 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId, 787 pkt->req->getPaddr()); 788 if (!sendTimingReq(pkt)) { 789 DPRINTF(GPUFetch, "failed again!\n"); 790 break; 791 } else { 792 DPRINTF(GPUFetch, "successful!\n"); 793 retries.pop_front(); 794 } 795 } 796} 797 798void 799ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 800{ 801 // There must be a way around this check to do the globalMemStart... 802 Addr tmp_vaddr = pkt->req->getVaddr(); 803 804 updatePageDivergenceDist(tmp_vaddr); 805 806 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(), 807 pkt->req->getFlags(), pkt->req->masterId(), 808 pkt->req->getPC()); 809 810 // figure out the type of the request to set read/write 811 BaseTLB::Mode TLB_mode; 812 assert(pkt->isRead() || pkt->isWrite()); 813 814 // Check write before read for atomic operations 815 // since atomic operations should use BaseTLB::Write 816 if (pkt->isWrite()){ 817 TLB_mode = BaseTLB::Write; 818 } else if (pkt->isRead()) { 819 TLB_mode = BaseTLB::Read; 820 } else { 821 fatal("pkt is not a read nor a write\n"); 822 } 823 824 tlbCycles -= curTick(); 825 ++tlbRequests; 826 827 int tlbPort_index = perLaneTLB ? index : 0; 828 829 if (shader->timingSim) { 830 if (debugSegFault) { 831 Process *p = shader->gpuTc->getProcessPtr(); 832 Addr vaddr = pkt->req->getVaddr(); 833 unsigned size = pkt->getSize(); 834 835 if ((vaddr + size - 1) % 64 < vaddr % 64) { 836 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n", 837 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr); 838 } 839 840 Addr paddr; 841 842 if (!p->pTable->translate(vaddr, paddr)) { 843 if (!p->fixupStackFault(vaddr)) { 844 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n", 845 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 846 vaddr); 847 } 848 } 849 } 850 851 // This is the SenderState needed upon return 852 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index); 853 854 // This is the senderState needed by the TLB hierarchy to function 855 TheISA::GpuTLB::TranslationState *translation_state = 856 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false, 857 pkt->senderState); 858 859 pkt->senderState = translation_state; 860 861 if (functionalTLB) { 862 tlbPort[tlbPort_index]->sendFunctional(pkt); 863 864 // update the hitLevel distribution 865 int hit_level = translation_state->hitLevel; 866 assert(hit_level != -1); 867 hitsPerTLBLevel[hit_level]++; 868 869 // New SenderState for the memory access 870 X86ISA::GpuTLB::TranslationState *sender_state = 871 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState); 872 873 delete sender_state->tlbEntry; 874 delete sender_state->saved; 875 delete sender_state; 876 877 assert(pkt->req->hasPaddr()); 878 assert(pkt->req->hasSize()); 879 880 uint8_t *tmpData = pkt->getPtr<uint8_t>(); 881 882 // this is necessary because the GPU TLB receives packets instead 883 // of requests. when the translation is complete, all relevent 884 // fields in the request will be populated, but not in the packet. 885 // here we create the new packet so we can set the size, addr, 886 // and proper flags. 887 PacketPtr oldPkt = pkt; 888 pkt = new Packet(oldPkt->req, oldPkt->cmd); 889 delete oldPkt; 890 pkt->dataStatic(tmpData); 891 892 893 // New SenderState for the memory access 894 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, 895 index, nullptr); 896 897 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index); 898 gpuDynInst->tlbHitLevel[index] = hit_level; 899 900 901 // translation is done. Schedule the mem_req_event at the 902 // appropriate cycle to send the timing memory request to ruby 903 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 904 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 905 906 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data " 907 "scheduled\n", cu_id, gpuDynInst->simdId, 908 gpuDynInst->wfSlotId, index, pkt->req->getPaddr()); 909 910 schedule(mem_req_event, curTick() + req_tick_latency); 911 } else if (tlbPort[tlbPort_index]->isStalled()) { 912 assert(tlbPort[tlbPort_index]->retries.size() > 0); 913 914 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 915 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 916 tmp_vaddr); 917 918 tlbPort[tlbPort_index]->retries.push_back(pkt); 919 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) { 920 // Stall the data port; 921 // No more packet will be issued till 922 // ruby indicates resources are freed by 923 // a recvReqRetry() call back on this port. 924 tlbPort[tlbPort_index]->stallPort(); 925 926 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 927 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 928 tmp_vaddr); 929 930 tlbPort[tlbPort_index]->retries.push_back(pkt); 931 } else { 932 DPRINTF(GPUTLB, 933 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n", 934 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr); 935 } 936 } else { 937 if (pkt->cmd == MemCmd::MemFenceReq) { 938 gpuDynInst->statusBitVector = VectorMask(0); 939 } else { 940 gpuDynInst->statusBitVector &= (~(1ll << index)); 941 } 942 943 // New SenderState for the memory access 944 delete pkt->senderState; 945 946 // Because it's atomic operation, only need TLB translation state 947 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode, 948 shader->gpuTc); 949 950 tlbPort[tlbPort_index]->sendFunctional(pkt); 951 952 // the addr of the packet is not modified, so we need to create a new 953 // packet, or otherwise the memory access will have the old virtual 954 // address sent in the translation packet, instead of the physical 955 // address returned by the translation. 956 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd); 957 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 958 959 // Translation is done. It is safe to send the packet to memory. 960 memPort[0]->sendFunctional(new_pkt); 961 962 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id, 963 gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 964 new_pkt->req->getPaddr()); 965 966 // safe_cast the senderState 967 TheISA::GpuTLB::TranslationState *sender_state = 968 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 969 970 delete sender_state->tlbEntry; 971 delete new_pkt; 972 delete pkt->senderState; 973 delete pkt->req; 974 delete pkt; 975 } 976} 977 978void 979ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 980{ 981 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 982 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 983 984 985 // New SenderState for the memory access 986 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index, 987 nullptr); 988 989 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n", 990 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 991 pkt->req->getPaddr()); 992 993 schedule(mem_req_event, curTick() + req_tick_latency); 994} 995 996void 997ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch, 998 Request* req) 999{ 1000 if (!req) { 1001 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId); 1002 } 1003 req->setPaddr(0); 1004 if (kernelLaunch) { 1005 req->setFlags(Request::KERNEL); 1006 } 1007 1008 gpuDynInst->s_type = SEG_GLOBAL; 1009 1010 // for non-kernel MemFence operations, memorder flags are set depending 1011 // on which type of request is currently being sent, so this 1012 // should be set by the caller (e.g. if an inst has acq-rel 1013 // semantics, it will send one acquire req an one release req) 1014 gpuDynInst->setRequestFlags(req, kernelLaunch); 1015 1016 // a mem fence must correspond to an acquire/release request 1017 assert(req->isAcquire() || req->isRelease()); 1018 1019 // create packet 1020 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq); 1021 1022 // set packet's sender state 1023 pkt->senderState = 1024 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr); 1025 1026 // send the packet 1027 sendSyncRequest(gpuDynInst, 0, pkt); 1028} 1029 1030const char* 1031ComputeUnit::DataPort::MemRespEvent::description() const 1032{ 1033 return "ComputeUnit memory response event"; 1034} 1035 1036void 1037ComputeUnit::DataPort::MemRespEvent::process() 1038{ 1039 DataPort::SenderState *sender_state = 1040 safe_cast<DataPort::SenderState*>(pkt->senderState); 1041 1042 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1043 ComputeUnit *compute_unit = dataPort->computeUnit; 1044 1045 assert(gpuDynInst); 1046 1047 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n", 1048 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 1049 pkt->req->getPaddr(), dataPort->index); 1050 1051 Addr paddr = pkt->req->getPaddr(); 1052 1053 if (pkt->cmd != MemCmd::MemFenceResp) { 1054 int index = gpuDynInst->memStatusVector[paddr].back(); 1055 1056 DPRINTF(GPUMem, "Response for addr %#x, index %d\n", 1057 pkt->req->getPaddr(), index); 1058 1059 gpuDynInst->memStatusVector[paddr].pop_back(); 1060 gpuDynInst->pAddr = pkt->req->getPaddr(); 1061 1062 if (pkt->isRead() || pkt->isWrite()) { 1063 1064 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) { 1065 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1066 } else { 1067 assert(gpuDynInst->statusVector[index] > 0); 1068 gpuDynInst->statusVector[index]--; 1069 1070 if (!gpuDynInst->statusVector[index]) 1071 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1072 } 1073 1074 DPRINTF(GPUMem, "bitvector is now %#x\n", 1075 gpuDynInst->statusBitVector); 1076 1077 if (gpuDynInst->statusBitVector == VectorMask(0)) { 1078 auto iter = gpuDynInst->memStatusVector.begin(); 1079 auto end = gpuDynInst->memStatusVector.end(); 1080 1081 while (iter != end) { 1082 assert(iter->second.empty()); 1083 ++iter; 1084 } 1085 1086 gpuDynInst->memStatusVector.clear(); 1087 1088 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST) 1089 gpuDynInst->statusVector.clear(); 1090 1091 if (gpuDynInst->m_op == Enums::MO_LD || MO_A(gpuDynInst->m_op) 1092 || MO_ANR(gpuDynInst->m_op)) { 1093 assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy()); 1094 1095 compute_unit->globalMemoryPipe.getGMLdRespFIFO() 1096 .push(gpuDynInst); 1097 } else { 1098 assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy()); 1099 1100 compute_unit->globalMemoryPipe.getGMStRespFIFO() 1101 .push(gpuDynInst); 1102 } 1103 1104 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n", 1105 compute_unit->cu_id, gpuDynInst->simdId, 1106 gpuDynInst->wfSlotId); 1107 1108 // after clearing the status vectors, 1109 // see if there is a continuation to perform 1110 // the continuation may generate more work for 1111 // this memory request 1112 if (gpuDynInst->useContinuation) { 1113 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 1114 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1115 gpuDynInst); 1116 } 1117 } 1118 } 1119 } else { 1120 gpuDynInst->statusBitVector = VectorMask(0); 1121 1122 if (gpuDynInst->useContinuation) { 1123 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 1124 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1125 gpuDynInst); 1126 } 1127 } 1128 1129 delete pkt->senderState; 1130 delete pkt->req; 1131 delete pkt; 1132} 1133 1134ComputeUnit* 1135ComputeUnitParams::create() 1136{ 1137 return new ComputeUnit(this); 1138} 1139 1140bool 1141ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt) 1142{ 1143 Addr line = pkt->req->getPaddr(); 1144 1145 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id, 1146 pkt->req->getVaddr(), line); 1147 1148 assert(pkt->senderState); 1149 computeUnit->tlbCycles += curTick(); 1150 1151 // pop off the TLB translation state 1152 TheISA::GpuTLB::TranslationState *translation_state = 1153 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1154 1155 // no PageFaults are permitted for data accesses 1156 if (!translation_state->tlbEntry->valid) { 1157 DTLBPort::SenderState *sender_state = 1158 safe_cast<DTLBPort::SenderState*>(translation_state->saved); 1159 1160 Wavefront *w M5_VAR_USED = 1161 computeUnit->wfList[sender_state->_gpuDynInst->simdId] 1162 [sender_state->_gpuDynInst->wfSlotId]; 1163 1164 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId, 1165 pkt->req->getVaddr()); 1166 } 1167 1168 assert(translation_state->tlbEntry->valid); 1169 1170 // update the hitLevel distribution 1171 int hit_level = translation_state->hitLevel; 1172 computeUnit->hitsPerTLBLevel[hit_level]++; 1173 1174 delete translation_state->tlbEntry; 1175 assert(!translation_state->ports.size()); 1176 pkt->senderState = translation_state->saved; 1177 1178 // for prefetch pkt 1179 BaseTLB::Mode TLB_mode = translation_state->tlbMode; 1180 1181 delete translation_state; 1182 1183 // use the original sender state to know how to close this transaction 1184 DTLBPort::SenderState *sender_state = 1185 safe_cast<DTLBPort::SenderState*>(pkt->senderState); 1186 1187 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1188 int mp_index = sender_state->portIndex; 1189 Addr vaddr = pkt->req->getVaddr(); 1190 gpuDynInst->memStatusVector[line].push_back(mp_index); 1191 gpuDynInst->tlbHitLevel[mp_index] = hit_level; 1192 1193 MemCmd requestCmd; 1194 1195 if (pkt->cmd == MemCmd::ReadResp) { 1196 requestCmd = MemCmd::ReadReq; 1197 } else if (pkt->cmd == MemCmd::WriteResp) { 1198 requestCmd = MemCmd::WriteReq; 1199 } else if (pkt->cmd == MemCmd::SwapResp) { 1200 requestCmd = MemCmd::SwapReq; 1201 } else { 1202 panic("unsupported response to request conversion %s\n", 1203 pkt->cmd.toString()); 1204 } 1205 1206 if (computeUnit->prefetchDepth) { 1207 int simdId = gpuDynInst->simdId; 1208 int wfSlotId = gpuDynInst->wfSlotId; 1209 Addr last = 0; 1210 1211 switch(computeUnit->prefetchType) { 1212 case Enums::PF_CU: 1213 last = computeUnit->lastVaddrCU[mp_index]; 1214 break; 1215 case Enums::PF_PHASE: 1216 last = computeUnit->lastVaddrSimd[simdId][mp_index]; 1217 break; 1218 case Enums::PF_WF: 1219 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index]; 1220 default: 1221 break; 1222 } 1223 1224 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n", 1225 computeUnit->cu_id, simdId, wfSlotId, mp_index, last); 1226 1227 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) - 1228 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift 1229 : 0; 1230 1231 DPRINTF(GPUPrefetch, "Stride is %d\n", stride); 1232 1233 computeUnit->lastVaddrCU[mp_index] = vaddr; 1234 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr; 1235 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr; 1236 1237 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ? 1238 computeUnit->prefetchStride: stride; 1239 1240 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr, 1241 computeUnit->cu_id, simdId, wfSlotId, mp_index); 1242 1243 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr); 1244 1245 // Prefetch Next few pages atomically 1246 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) { 1247 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride, 1248 vaddr+stride*pf*TheISA::PageBytes); 1249 1250 if (!stride) 1251 break; 1252 1253 Request *prefetch_req = new Request(0, vaddr + stride * pf * 1254 TheISA::PageBytes, 1255 sizeof(uint8_t), 0, 1256 computeUnit->masterId(), 1257 0, 0, 0); 1258 1259 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd); 1260 uint8_t foo = 0; 1261 prefetch_pkt->dataStatic(&foo); 1262 1263 // Because it's atomic operation, only need TLB translation state 1264 prefetch_pkt->senderState = 1265 new TheISA::GpuTLB::TranslationState(TLB_mode, 1266 computeUnit->shader->gpuTc, 1267 true); 1268 1269 // Currently prefetches are zero-latency, hence the sendFunctional 1270 sendFunctional(prefetch_pkt); 1271 1272 /* safe_cast the senderState */ 1273 TheISA::GpuTLB::TranslationState *tlb_state = 1274 safe_cast<TheISA::GpuTLB::TranslationState*>( 1275 prefetch_pkt->senderState); 1276 1277 1278 delete tlb_state->tlbEntry; 1279 delete tlb_state; 1280 delete prefetch_pkt->req; 1281 delete prefetch_pkt; 1282 } 1283 } 1284 1285 // First we must convert the response cmd back to a request cmd so that 1286 // the request can be sent through the cu's master port 1287 PacketPtr new_pkt = new Packet(pkt->req, requestCmd); 1288 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 1289 delete pkt->senderState; 1290 delete pkt; 1291 1292 // New SenderState for the memory access 1293 new_pkt->senderState = 1294 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index, 1295 nullptr); 1296 1297 // translation is done. Schedule the mem_req_event at the appropriate 1298 // cycle to send the timing memory request to ruby 1299 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 1300 new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index], 1301 new_pkt); 1302 1303 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n", 1304 computeUnit->cu_id, gpuDynInst->simdId, 1305 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr()); 1306 1307 computeUnit->schedule(mem_req_event, curTick() + 1308 computeUnit->req_tick_latency); 1309 1310 return true; 1311} 1312 1313const char* 1314ComputeUnit::DataPort::MemReqEvent::description() const 1315{ 1316 return "ComputeUnit memory request event"; 1317} 1318 1319void 1320ComputeUnit::DataPort::MemReqEvent::process() 1321{ 1322 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 1323 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1324 ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit; 1325 1326 if (!(dataPort->sendTimingReq(pkt))) { 1327 dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst)); 1328 1329 DPRINTF(GPUPort, 1330 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n", 1331 compute_unit->cu_id, gpuDynInst->simdId, 1332 gpuDynInst->wfSlotId, dataPort->index, 1333 pkt->req->getPaddr()); 1334 } else { 1335 DPRINTF(GPUPort, 1336 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n", 1337 compute_unit->cu_id, gpuDynInst->simdId, 1338 gpuDynInst->wfSlotId, dataPort->index, 1339 pkt->req->getPaddr()); 1340 } 1341} 1342 1343/* 1344 * The initial translation request could have been rejected, 1345 * if <retries> queue is not Retry sending the translation 1346 * request. sendRetry() is called from the peer port whenever 1347 * a translation completes. 1348 */ 1349void 1350ComputeUnit::DTLBPort::recvReqRetry() 1351{ 1352 int len = retries.size(); 1353 1354 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n", 1355 computeUnit->cu_id, len); 1356 1357 assert(len > 0); 1358 assert(isStalled()); 1359 // recvReqRetry is an indication that the resource on which this 1360 // port was stalling on is freed. So, remove the stall first 1361 unstallPort(); 1362 1363 for (int i = 0; i < len; ++i) { 1364 PacketPtr pkt = retries.front(); 1365 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1366 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr); 1367 1368 if (!sendTimingReq(pkt)) { 1369 // Stall port 1370 stallPort(); 1371 DPRINTF(GPUTLB, ": failed again\n"); 1372 break; 1373 } else { 1374 DPRINTF(GPUTLB, ": successful\n"); 1375 retries.pop_front(); 1376 } 1377 } 1378} 1379 1380bool 1381ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt) 1382{ 1383 Addr line M5_VAR_USED = pkt->req->getPaddr(); 1384 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n", 1385 computeUnit->cu_id, pkt->req->getVaddr(), line); 1386 1387 assert(pkt->senderState); 1388 1389 // pop off the TLB translation state 1390 TheISA::GpuTLB::TranslationState *translation_state = 1391 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1392 1393 bool success = translation_state->tlbEntry->valid; 1394 delete translation_state->tlbEntry; 1395 assert(!translation_state->ports.size()); 1396 pkt->senderState = translation_state->saved; 1397 delete translation_state; 1398 1399 // use the original sender state to know how to close this transaction 1400 ITLBPort::SenderState *sender_state = 1401 safe_cast<ITLBPort::SenderState*>(pkt->senderState); 1402 1403 // get the wavefront associated with this translation request 1404 Wavefront *wavefront = sender_state->wavefront; 1405 delete pkt->senderState; 1406 1407 if (success) { 1408 // pkt is reused in fetch(), don't delete it here. However, we must 1409 // reset the command to be a request so that it can be sent through 1410 // the cu's master port 1411 assert(pkt->cmd == MemCmd::ReadResp); 1412 pkt->cmd = MemCmd::ReadReq; 1413 1414 computeUnit->fetchStage.fetch(pkt, wavefront); 1415 } else { 1416 if (wavefront->dropFetch) { 1417 assert(wavefront->instructionBuffer.empty()); 1418 wavefront->dropFetch = false; 1419 } 1420 1421 wavefront->pendingFetch = 0; 1422 } 1423 1424 return true; 1425} 1426 1427/* 1428 * The initial translation request could have been rejected, if 1429 * <retries> queue is not empty. Retry sending the translation 1430 * request. sendRetry() is called from the peer port whenever 1431 * a translation completes. 1432 */ 1433void 1434ComputeUnit::ITLBPort::recvReqRetry() 1435{ 1436 1437 int len = retries.size(); 1438 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len); 1439 1440 assert(len > 0); 1441 assert(isStalled()); 1442 1443 // recvReqRetry is an indication that the resource on which this 1444 // port was stalling on is freed. So, remove the stall first 1445 unstallPort(); 1446 1447 for (int i = 0; i < len; ++i) { 1448 PacketPtr pkt = retries.front(); 1449 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1450 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr); 1451 1452 if (!sendTimingReq(pkt)) { 1453 stallPort(); // Stall port 1454 DPRINTF(GPUTLB, ": failed again\n"); 1455 break; 1456 } else { 1457 DPRINTF(GPUTLB, ": successful\n"); 1458 retries.pop_front(); 1459 } 1460 } 1461} 1462 1463void 1464ComputeUnit::regStats() 1465{ 1466 MemObject::regStats(); 1467 1468 tlbCycles 1469 .name(name() + ".tlb_cycles") 1470 .desc("total number of cycles for all uncoalesced requests") 1471 ; 1472 1473 tlbRequests 1474 .name(name() + ".tlb_requests") 1475 .desc("number of uncoalesced requests") 1476 ; 1477 1478 tlbLatency 1479 .name(name() + ".avg_translation_latency") 1480 .desc("Avg. translation latency for data translations") 1481 ; 1482 1483 tlbLatency = tlbCycles / tlbRequests; 1484 1485 hitsPerTLBLevel 1486 .init(4) 1487 .name(name() + ".TLB_hits_distribution") 1488 .desc("TLB hits distribution (0 for page table, x for Lx-TLB") 1489 ; 1490 1491 // fixed number of TLB levels 1492 for (int i = 0; i < 4; ++i) { 1493 if (!i) 1494 hitsPerTLBLevel.subname(i,"page_table"); 1495 else 1496 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i)); 1497 } 1498 1499 execRateDist 1500 .init(0, 10, 2) 1501 .name(name() + ".inst_exec_rate") 1502 .desc("Instruction Execution Rate: Number of executed vector " 1503 "instructions per cycle") 1504 ; 1505 1506 ldsBankConflictDist 1507 .init(0, wfSize(), 2) 1508 .name(name() + ".lds_bank_conflicts") 1509 .desc("Number of bank conflicts per LDS memory packet") 1510 ; 1511 1512 ldsBankAccesses 1513 .name(name() + ".lds_bank_access_cnt") 1514 .desc("Total number of LDS bank accesses") 1515 ; 1516 1517 pageDivergenceDist 1518 // A wavefront can touch up to N pages per memory instruction where 1519 // N is equal to the wavefront size 1520 // The number of pages per bin can be configured (here it's 4). 1521 .init(1, wfSize(), 4) 1522 .name(name() + ".page_divergence_dist") 1523 .desc("pages touched per wf (over all mem. instr.)") 1524 ; 1525 1526 controlFlowDivergenceDist 1527 .init(1, wfSize(), 4) 1528 .name(name() + ".warp_execution_dist") 1529 .desc("number of lanes active per instruction (oval all instructions)") 1530 ; 1531 1532 activeLanesPerGMemInstrDist 1533 .init(1, wfSize(), 4) 1534 .name(name() + ".gmem_lanes_execution_dist") 1535 .desc("number of active lanes per global memory instruction") 1536 ; 1537 1538 activeLanesPerLMemInstrDist 1539 .init(1, wfSize(), 4) 1540 .name(name() + ".lmem_lanes_execution_dist") 1541 .desc("number of active lanes per local memory instruction") 1542 ; 1543 1544 numInstrExecuted 1545 .name(name() + ".num_instr_executed") 1546 .desc("number of instructions executed") 1547 ; 1548 1549 numVecOpsExecuted 1550 .name(name() + ".num_vec_ops_executed") 1551 .desc("number of vec ops executed (e.g. WF size/inst)") 1552 ; 1553 1554 totalCycles 1555 .name(name() + ".num_total_cycles") 1556 .desc("number of cycles the CU ran for") 1557 ; 1558 1559 ipc 1560 .name(name() + ".ipc") 1561 .desc("Instructions per cycle (this CU only)") 1562 ; 1563 1564 vpc 1565 .name(name() + ".vpc") 1566 .desc("Vector Operations per cycle (this CU only)") 1567 ; 1568 1569 numALUInstsExecuted 1570 .name(name() + ".num_alu_insts_executed") 1571 .desc("Number of dynamic non-GM memory insts executed") 1572 ; 1573 1574 wgBlockedDueLdsAllocation 1575 .name(name() + ".wg_blocked_due_lds_alloc") 1576 .desc("Workgroup blocked due to LDS capacity") 1577 ; 1578 1579 ipc = numInstrExecuted / totalCycles; 1580 vpc = numVecOpsExecuted / totalCycles; 1581 1582 numTimesWgBlockedDueVgprAlloc 1583 .name(name() + ".times_wg_blocked_due_vgpr_alloc") 1584 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD") 1585 ; 1586 1587 dynamicGMemInstrCnt 1588 .name(name() + ".global_mem_instr_cnt") 1589 .desc("dynamic global memory instructions count") 1590 ; 1591 1592 dynamicLMemInstrCnt 1593 .name(name() + ".local_mem_instr_cnt") 1594 .desc("dynamic local memory intruction count") 1595 ; 1596 1597 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt - 1598 dynamicLMemInstrCnt; 1599 1600 completedWfs 1601 .name(name() + ".num_completed_wfs") 1602 .desc("number of completed wavefronts") 1603 ; 1604 1605 numCASOps 1606 .name(name() + ".num_CAS_ops") 1607 .desc("number of compare and swap operations") 1608 ; 1609 1610 numFailedCASOps 1611 .name(name() + ".num_failed_CAS_ops") 1612 .desc("number of compare and swap operations that failed") 1613 ; 1614 1615 // register stats of pipeline stages 1616 fetchStage.regStats(); 1617 scoreboardCheckStage.regStats(); 1618 scheduleStage.regStats(); 1619 execStage.regStats(); 1620 1621 // register stats of memory pipeline 1622 globalMemoryPipe.regStats(); 1623 localMemoryPipe.regStats(); 1624} 1625 1626void 1627ComputeUnit::updatePageDivergenceDist(Addr addr) 1628{ 1629 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes); 1630 1631 if (!pagesTouched.count(virt_page_addr)) 1632 pagesTouched[virt_page_addr] = 1; 1633 else 1634 pagesTouched[virt_page_addr]++; 1635} 1636 1637void 1638ComputeUnit::CUExitCallback::process() 1639{ 1640 if (computeUnit->countPages) { 1641 std::ostream *page_stat_file = 1642 simout.create(computeUnit->name().c_str())->stream(); 1643 1644 *page_stat_file << "page, wavefront accesses, workitem accesses" << 1645 std::endl; 1646 1647 for (auto iter : computeUnit->pageAccesses) { 1648 *page_stat_file << std::hex << iter.first << ","; 1649 *page_stat_file << std::dec << iter.second.first << ","; 1650 *page_stat_file << std::dec << iter.second.second << std::endl; 1651 } 1652 } 1653 } 1654 1655bool 1656ComputeUnit::isDone() const 1657{ 1658 for (int i = 0; i < numSIMDs; ++i) { 1659 if (!isSimdDone(i)) { 1660 return false; 1661 } 1662 } 1663 1664 bool glbMemBusRdy = true; 1665 for (int j = 0; j < numGlbMemUnits; ++j) { 1666 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy(); 1667 } 1668 bool locMemBusRdy = true; 1669 for (int j = 0; j < numLocMemUnits; ++j) { 1670 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy(); 1671 } 1672 1673 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() || 1674 !globalMemoryPipe.isGMStRespFIFOWrRdy() || 1675 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy() 1676 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() || 1677 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) { 1678 return false; 1679 } 1680 1681 return true; 1682} 1683 1684int32_t 1685ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const 1686{ 1687 return lds.getRefCounter(dispatchId, wgId); 1688} 1689 1690bool 1691ComputeUnit::isSimdDone(uint32_t simdId) const 1692{ 1693 assert(simdId < numSIMDs); 1694 1695 for (int i=0; i < numGlbMemUnits; ++i) { 1696 if (!vrfToGlobalMemPipeBus[i].rdy()) 1697 return false; 1698 } 1699 for (int i=0; i < numLocMemUnits; ++i) { 1700 if (!vrfToLocalMemPipeBus[i].rdy()) 1701 return false; 1702 } 1703 if (!aluPipe[simdId].rdy()) { 1704 return false; 1705 } 1706 1707 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){ 1708 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) { 1709 return false; 1710 } 1711 } 1712 1713 return true; 1714} 1715 1716/** 1717 * send a general request to the LDS 1718 * make sure to look at the return value here as your request might be 1719 * NACK'd and returning false means that you have to have some backup plan 1720 */ 1721bool 1722ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst) 1723{ 1724 // this is just a request to carry the GPUDynInstPtr 1725 // back and forth 1726 Request *newRequest = new Request(); 1727 newRequest->setPaddr(0x0); 1728 1729 // ReadReq is not evaluted by the LDS but the Packet ctor requires this 1730 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq); 1731 1732 // This is the SenderState needed upon return 1733 newPacket->senderState = new LDSPort::SenderState(gpuDynInst); 1734 1735 return ldsPort->sendTimingReq(newPacket); 1736} 1737 1738/** 1739 * get the result of packets sent to the LDS when they return 1740 */ 1741bool 1742ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet) 1743{ 1744 const ComputeUnit::LDSPort::SenderState *senderState = 1745 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState); 1746 1747 fatal_if(!senderState, "did not get the right sort of sender state"); 1748 1749 GPUDynInstPtr gpuDynInst = senderState->getMemInst(); 1750 1751 delete packet->senderState; 1752 delete packet->req; 1753 delete packet; 1754 1755 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst); 1756 return true; 1757} 1758 1759/** 1760 * attempt to send this packet, either the port is already stalled, the request 1761 * is nack'd and must stall or the request goes through 1762 * when a request cannot be sent, add it to the retries queue 1763 */ 1764bool 1765ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt) 1766{ 1767 ComputeUnit::LDSPort::SenderState *sender_state = 1768 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState); 1769 fatal_if(!sender_state, "packet without a valid sender state"); 1770 1771 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst(); 1772 1773 if (isStalled()) { 1774 fatal_if(retries.empty(), "must have retries waiting to be stalled"); 1775 1776 retries.push(pkt); 1777 1778 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n", 1779 computeUnit->cu_id, gpuDynInst->simdId, 1780 gpuDynInst->wfSlotId); 1781 return false; 1782 } else if (!MasterPort::sendTimingReq(pkt)) { 1783 // need to stall the LDS port until a recvReqRetry() is received 1784 // this indicates that there is more space 1785 stallPort(); 1786 retries.push(pkt); 1787 1788 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n", 1789 computeUnit->cu_id, gpuDynInst->simdId, 1790 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1791 return false; 1792 } else { 1793 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n", 1794 computeUnit->cu_id, gpuDynInst->simdId, 1795 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1796 return true; 1797 } 1798} 1799 1800/** 1801 * the bus is telling the port that there is now space so retrying stalled 1802 * requests should work now 1803 * this allows the port to have a request be nack'd and then have the receiver 1804 * say when there is space, rather than simply retrying the send every cycle 1805 */ 1806void 1807ComputeUnit::LDSPort::recvReqRetry() 1808{ 1809 auto queueSize = retries.size(); 1810 1811 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n", 1812 computeUnit->cu_id, queueSize); 1813 1814 fatal_if(queueSize < 1, 1815 "why was there a recvReqRetry() with no pending reqs?"); 1816 fatal_if(!isStalled(), 1817 "recvReqRetry() happened when the port was not stalled"); 1818 1819 unstallPort(); 1820 1821 while (!retries.empty()) { 1822 PacketPtr packet = retries.front(); 1823 1824 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id); 1825 1826 if (!MasterPort::sendTimingReq(packet)) { 1827 // Stall port 1828 stallPort(); 1829 DPRINTF(GPUPort, ": LDS send failed again\n"); 1830 break; 1831 } else { 1832 DPRINTF(GPUTLB, ": LDS send successful\n"); 1833 retries.pop(); 1834 } 1835 } 1836}
| 369 ++cnt; 370 } 371 } 372 ++barrier_id; 373} 374 375int 376ComputeUnit::ReadyWorkgroup(NDRange *ndr) 377{ 378 // Get true size of workgroup (after clamping to grid size) 379 int trueWgSize[3]; 380 int trueWgSizeTotal = 1; 381 382 for (int d = 0; d < 3; ++d) { 383 trueWgSize[d] = std::min(ndr->q.wgSize[d], ndr->q.gdSize[d] - 384 ndr->wgId[d] * ndr->q.wgSize[d]); 385 386 trueWgSizeTotal *= trueWgSize[d]; 387 DPRINTF(GPUDisp, "trueWgSize[%d] = %d\n", d, trueWgSize[d]); 388 } 389 390 DPRINTF(GPUDisp, "trueWgSizeTotal = %d\n", trueWgSizeTotal); 391 392 // calculate the number of 32-bit vector registers required by each 393 // work item of the work group 394 int vregDemandPerWI = ndr->q.sRegCount + (2 * ndr->q.dRegCount); 395 bool vregAvail = true; 396 int numWfs = (trueWgSizeTotal + wfSize() - 1) / wfSize(); 397 int freeWfSlots = 0; 398 // check if the total number of VGPRs required by all WFs of the WG 399 // fit in the VRFs of all SIMD units 400 assert((numWfs * vregDemandPerWI) <= (numSIMDs * numVecRegsPerSimd)); 401 int numMappedWfs = 0; 402 std::vector<int> numWfsPerSimd; 403 numWfsPerSimd.resize(numSIMDs, 0); 404 // find how many free WF slots we have across all SIMDs 405 for (int j = 0; j < shader->n_wf; ++j) { 406 for (int i = 0; i < numSIMDs; ++i) { 407 if (wfList[i][j]->status == Wavefront::S_STOPPED) { 408 // count the number of free WF slots 409 ++freeWfSlots; 410 if (numMappedWfs < numWfs) { 411 // count the WFs to be assigned per SIMD 412 numWfsPerSimd[i]++; 413 } 414 numMappedWfs++; 415 } 416 } 417 } 418 419 // if there are enough free WF slots then find if there are enough 420 // free VGPRs per SIMD based on the WF->SIMD mapping 421 if (freeWfSlots >= numWfs) { 422 for (int j = 0; j < numSIMDs; ++j) { 423 // find if there are enough free VGPR regions in the SIMD's VRF 424 // to accommodate the WFs of the new WG that would be mapped to 425 // this SIMD unit 426 vregAvail = vrf[j]->manager->canAllocate(numWfsPerSimd[j], 427 vregDemandPerWI); 428 429 // stop searching if there is at least one SIMD 430 // whose VRF does not have enough free VGPR pools. 431 // This is because a WG is scheduled only if ALL 432 // of its WFs can be scheduled 433 if (!vregAvail) 434 break; 435 } 436 } 437 438 DPRINTF(GPUDisp, "Free WF slots = %d, VGPR Availability = %d\n", 439 freeWfSlots, vregAvail); 440 441 if (!vregAvail) { 442 ++numTimesWgBlockedDueVgprAlloc; 443 } 444 445 // Return true if enough WF slots to submit workgroup and if there are 446 // enough VGPRs to schedule all WFs to their SIMD units 447 if (!lds.canReserve(ndr->q.ldsSize)) { 448 wgBlockedDueLdsAllocation++; 449 } 450 451 // Return true if (a) there are enough free WF slots to submit 452 // workgrounp and (b) if there are enough VGPRs to schedule all WFs to their 453 // SIMD units and (c) if there is enough space in LDS 454 return freeWfSlots >= numWfs && vregAvail && lds.canReserve(ndr->q.ldsSize); 455} 456 457int 458ComputeUnit::AllAtBarrier(uint32_t _barrier_id, uint32_t bcnt, uint32_t bslots) 459{ 460 DPRINTF(GPUSync, "CU%d: Checking for All At Barrier\n", cu_id); 461 int ccnt = 0; 462 463 for (int i_simd = 0; i_simd < numSIMDs; ++i_simd) { 464 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf) { 465 Wavefront *w = wfList[i_simd][i_wf]; 466 467 if (w->status == Wavefront::S_RUNNING) { 468 DPRINTF(GPUSync, "Checking WF[%d][%d]\n", i_simd, i_wf); 469 470 DPRINTF(GPUSync, "wf->barrier_id = %d, _barrier_id = %d\n", 471 w->barrier_id, _barrier_id); 472 473 DPRINTF(GPUSync, "wf->barrier_cnt %d, bcnt = %d\n", 474 w->barrier_cnt, bcnt); 475 } 476 477 if (w->status == Wavefront::S_RUNNING && 478 w->barrier_id == _barrier_id && w->barrier_cnt == bcnt && 479 !w->outstanding_reqs) { 480 ++ccnt; 481 482 DPRINTF(GPUSync, "WF[%d][%d] at barrier, increment ccnt to " 483 "%d\n", i_simd, i_wf, ccnt); 484 } 485 } 486 } 487 488 DPRINTF(GPUSync, "CU%d: returning allAtBarrier ccnt = %d, bslots = %d\n", 489 cu_id, ccnt, bslots); 490 491 return ccnt == bslots; 492} 493 494// Check if the current wavefront is blocked on additional resources. 495bool 496ComputeUnit::cedeSIMD(int simdId, int wfSlotId) 497{ 498 bool cede = false; 499 500 // If --xact-cas-mode option is enabled in run.py, then xact_cas_ld 501 // magic instructions will impact the scheduling of wavefronts 502 if (xact_cas_mode) { 503 /* 504 * When a wavefront calls xact_cas_ld, it adds itself to a per address 505 * queue. All per address queues are managed by the xactCasLoadMap. 506 * 507 * A wavefront is not blocked if: it is not in ANY per address queue or 508 * if it is at the head of a per address queue. 509 */ 510 for (auto itMap : xactCasLoadMap) { 511 std::list<waveIdentifier> curWaveIDQueue = itMap.second.waveIDQueue; 512 513 if (!curWaveIDQueue.empty()) { 514 for (auto it : curWaveIDQueue) { 515 waveIdentifier cur_wave = it; 516 517 if (cur_wave.simdId == simdId && 518 cur_wave.wfSlotId == wfSlotId) { 519 // 2 possibilities 520 // 1: this WF has a green light 521 // 2: another WF has a green light 522 waveIdentifier owner_wave = curWaveIDQueue.front(); 523 524 if (owner_wave.simdId != cur_wave.simdId || 525 owner_wave.wfSlotId != cur_wave.wfSlotId) { 526 // possibility 2 527 cede = true; 528 break; 529 } else { 530 // possibility 1 531 break; 532 } 533 } 534 } 535 } 536 } 537 } 538 539 return cede; 540} 541 542// Execute one clock worth of work on the ComputeUnit. 543void 544ComputeUnit::exec() 545{ 546 updateEvents(); 547 // Execute pipeline stages in reverse order to simulate 548 // the pipeline latency 549 globalMemoryPipe.exec(); 550 localMemoryPipe.exec(); 551 execStage.exec(); 552 scheduleStage.exec(); 553 scoreboardCheckStage.exec(); 554 fetchStage.exec(); 555 556 totalCycles++; 557} 558 559void 560ComputeUnit::init() 561{ 562 // Initialize CU Bus models 563 glbMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 564 locMemToVrfBus.init(&shader->tick_cnt, shader->ticks(1)); 565 nextGlbMemBus = 0; 566 nextLocMemBus = 0; 567 fatal_if(numGlbMemUnits > 1, 568 "No support for multiple Global Memory Pipelines exists!!!"); 569 vrfToGlobalMemPipeBus.resize(numGlbMemUnits); 570 for (int j = 0; j < numGlbMemUnits; ++j) { 571 vrfToGlobalMemPipeBus[j] = WaitClass(); 572 vrfToGlobalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 573 } 574 575 fatal_if(numLocMemUnits > 1, 576 "No support for multiple Local Memory Pipelines exists!!!"); 577 vrfToLocalMemPipeBus.resize(numLocMemUnits); 578 for (int j = 0; j < numLocMemUnits; ++j) { 579 vrfToLocalMemPipeBus[j] = WaitClass(); 580 vrfToLocalMemPipeBus[j].init(&shader->tick_cnt, shader->ticks(1)); 581 } 582 vectorRegsReserved.resize(numSIMDs, 0); 583 aluPipe.resize(numSIMDs); 584 wfWait.resize(numSIMDs + numLocMemUnits + numGlbMemUnits); 585 586 for (int i = 0; i < numSIMDs + numLocMemUnits + numGlbMemUnits; ++i) { 587 wfWait[i] = WaitClass(); 588 wfWait[i].init(&shader->tick_cnt, shader->ticks(1)); 589 } 590 591 for (int i = 0; i < numSIMDs; ++i) { 592 aluPipe[i] = WaitClass(); 593 aluPipe[i].init(&shader->tick_cnt, shader->ticks(1)); 594 } 595 596 // Setup space for call args 597 for (int j = 0; j < numSIMDs; ++j) { 598 for (int i = 0; i < shader->n_wf; ++i) { 599 wfList[j][i]->initCallArgMem(shader->funcargs_size, wavefrontSize); 600 } 601 } 602 603 // Initializing pipeline resources 604 readyList.resize(numSIMDs + numGlbMemUnits + numLocMemUnits); 605 waveStatusList.resize(numSIMDs); 606 607 for (int j = 0; j < numSIMDs; ++j) { 608 for (int i = 0; i < shader->n_wf; ++i) { 609 waveStatusList[j].push_back( 610 std::make_pair(wfList[j][i], BLOCKED)); 611 } 612 } 613 614 for (int j = 0; j < (numSIMDs + numGlbMemUnits + numLocMemUnits); ++j) { 615 dispatchList.push_back(std::make_pair((Wavefront*)nullptr, EMPTY)); 616 } 617 618 fetchStage.init(this); 619 scoreboardCheckStage.init(this); 620 scheduleStage.init(this); 621 execStage.init(this); 622 globalMemoryPipe.init(this); 623 localMemoryPipe.init(this); 624 // initialize state for statistics calculation 625 vectorAluInstAvail.resize(numSIMDs, false); 626 shrMemInstAvail = 0; 627 glbMemInstAvail = 0; 628} 629 630bool 631ComputeUnit::DataPort::recvTimingResp(PacketPtr pkt) 632{ 633 // Ruby has completed the memory op. Schedule the mem_resp_event at the 634 // appropriate cycle to process the timing memory response 635 // This delay represents the pipeline delay 636 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 637 int index = sender_state->port_index; 638 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 639 640 // Is the packet returned a Kernel End or Barrier 641 if (pkt->req->isKernel() && pkt->req->isRelease()) { 642 Wavefront *w = 643 computeUnit->wfList[gpuDynInst->simdId][gpuDynInst->wfSlotId]; 644 645 // Check if we are waiting on Kernel End Release 646 if (w->status == Wavefront::S_RETURNING) { 647 DPRINTF(GPUDisp, "CU%d: WF[%d][%d][wv=%d]: WG id completed %d\n", 648 computeUnit->cu_id, w->simdId, w->wfSlotId, 649 w->wfDynId, w->kern_id); 650 651 computeUnit->shader->dispatcher->notifyWgCompl(w); 652 w->status = Wavefront::S_STOPPED; 653 } else { 654 w->outstanding_reqs--; 655 } 656 657 DPRINTF(GPUSync, "CU%d: WF[%d][%d]: barrier_cnt = %d\n", 658 computeUnit->cu_id, gpuDynInst->simdId, 659 gpuDynInst->wfSlotId, w->barrier_cnt); 660 661 if (gpuDynInst->useContinuation) { 662 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 663 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 664 gpuDynInst); 665 } 666 667 delete pkt->senderState; 668 delete pkt->req; 669 delete pkt; 670 return true; 671 } else if (pkt->req->isKernel() && pkt->req->isAcquire()) { 672 if (gpuDynInst->useContinuation) { 673 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 674 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 675 gpuDynInst); 676 } 677 678 delete pkt->senderState; 679 delete pkt->req; 680 delete pkt; 681 return true; 682 } 683 684 ComputeUnit::DataPort::MemRespEvent *mem_resp_event = 685 new ComputeUnit::DataPort::MemRespEvent(computeUnit->memPort[index], 686 pkt); 687 688 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x received!\n", 689 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 690 index, pkt->req->getPaddr()); 691 692 computeUnit->schedule(mem_resp_event, 693 curTick() + computeUnit->resp_tick_latency); 694 return true; 695} 696 697void 698ComputeUnit::DataPort::recvReqRetry() 699{ 700 int len = retries.size(); 701 702 assert(len > 0); 703 704 for (int i = 0; i < len; ++i) { 705 PacketPtr pkt = retries.front().first; 706 GPUDynInstPtr gpuDynInst M5_VAR_USED = retries.front().second; 707 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: retry mem inst addr %#x\n", 708 computeUnit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 709 pkt->req->getPaddr()); 710 711 /** Currently Ruby can return false due to conflicts for the particular 712 * cache block or address. Thus other requests should be allowed to 713 * pass and the data port should expect multiple retries. */ 714 if (!sendTimingReq(pkt)) { 715 DPRINTF(GPUMem, "failed again!\n"); 716 break; 717 } else { 718 DPRINTF(GPUMem, "successful!\n"); 719 retries.pop_front(); 720 } 721 } 722} 723 724bool 725ComputeUnit::SQCPort::recvTimingResp(PacketPtr pkt) 726{ 727 computeUnit->fetchStage.processFetchReturn(pkt); 728 729 return true; 730} 731 732void 733ComputeUnit::SQCPort::recvReqRetry() 734{ 735 int len = retries.size(); 736 737 assert(len > 0); 738 739 for (int i = 0; i < len; ++i) { 740 PacketPtr pkt = retries.front().first; 741 Wavefront *wavefront M5_VAR_USED = retries.front().second; 742 DPRINTF(GPUFetch, "CU%d: WF[%d][%d]: retrying FETCH addr %#x\n", 743 computeUnit->cu_id, wavefront->simdId, wavefront->wfSlotId, 744 pkt->req->getPaddr()); 745 if (!sendTimingReq(pkt)) { 746 DPRINTF(GPUFetch, "failed again!\n"); 747 break; 748 } else { 749 DPRINTF(GPUFetch, "successful!\n"); 750 retries.pop_front(); 751 } 752 } 753} 754 755void 756ComputeUnit::sendRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 757{ 758 // There must be a way around this check to do the globalMemStart... 759 Addr tmp_vaddr = pkt->req->getVaddr(); 760 761 updatePageDivergenceDist(tmp_vaddr); 762 763 pkt->req->setVirt(pkt->req->getAsid(), tmp_vaddr, pkt->req->getSize(), 764 pkt->req->getFlags(), pkt->req->masterId(), 765 pkt->req->getPC()); 766 767 // figure out the type of the request to set read/write 768 BaseTLB::Mode TLB_mode; 769 assert(pkt->isRead() || pkt->isWrite()); 770 771 // Check write before read for atomic operations 772 // since atomic operations should use BaseTLB::Write 773 if (pkt->isWrite()){ 774 TLB_mode = BaseTLB::Write; 775 } else if (pkt->isRead()) { 776 TLB_mode = BaseTLB::Read; 777 } else { 778 fatal("pkt is not a read nor a write\n"); 779 } 780 781 tlbCycles -= curTick(); 782 ++tlbRequests; 783 784 int tlbPort_index = perLaneTLB ? index : 0; 785 786 if (shader->timingSim) { 787 if (debugSegFault) { 788 Process *p = shader->gpuTc->getProcessPtr(); 789 Addr vaddr = pkt->req->getVaddr(); 790 unsigned size = pkt->getSize(); 791 792 if ((vaddr + size - 1) % 64 < vaddr % 64) { 793 panic("CU%d: WF[%d][%d]: Access to addr %#x is unaligned!\n", 794 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, vaddr); 795 } 796 797 Addr paddr; 798 799 if (!p->pTable->translate(vaddr, paddr)) { 800 if (!p->fixupStackFault(vaddr)) { 801 panic("CU%d: WF[%d][%d]: Fault on addr %#x!\n", 802 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 803 vaddr); 804 } 805 } 806 } 807 808 // This is the SenderState needed upon return 809 pkt->senderState = new DTLBPort::SenderState(gpuDynInst, index); 810 811 // This is the senderState needed by the TLB hierarchy to function 812 TheISA::GpuTLB::TranslationState *translation_state = 813 new TheISA::GpuTLB::TranslationState(TLB_mode, shader->gpuTc, false, 814 pkt->senderState); 815 816 pkt->senderState = translation_state; 817 818 if (functionalTLB) { 819 tlbPort[tlbPort_index]->sendFunctional(pkt); 820 821 // update the hitLevel distribution 822 int hit_level = translation_state->hitLevel; 823 assert(hit_level != -1); 824 hitsPerTLBLevel[hit_level]++; 825 826 // New SenderState for the memory access 827 X86ISA::GpuTLB::TranslationState *sender_state = 828 safe_cast<X86ISA::GpuTLB::TranslationState*>(pkt->senderState); 829 830 delete sender_state->tlbEntry; 831 delete sender_state->saved; 832 delete sender_state; 833 834 assert(pkt->req->hasPaddr()); 835 assert(pkt->req->hasSize()); 836 837 uint8_t *tmpData = pkt->getPtr<uint8_t>(); 838 839 // this is necessary because the GPU TLB receives packets instead 840 // of requests. when the translation is complete, all relevent 841 // fields in the request will be populated, but not in the packet. 842 // here we create the new packet so we can set the size, addr, 843 // and proper flags. 844 PacketPtr oldPkt = pkt; 845 pkt = new Packet(oldPkt->req, oldPkt->cmd); 846 delete oldPkt; 847 pkt->dataStatic(tmpData); 848 849 850 // New SenderState for the memory access 851 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, 852 index, nullptr); 853 854 gpuDynInst->memStatusVector[pkt->getAddr()].push_back(index); 855 gpuDynInst->tlbHitLevel[index] = hit_level; 856 857 858 // translation is done. Schedule the mem_req_event at the 859 // appropriate cycle to send the timing memory request to ruby 860 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 861 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 862 863 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data " 864 "scheduled\n", cu_id, gpuDynInst->simdId, 865 gpuDynInst->wfSlotId, index, pkt->req->getPaddr()); 866 867 schedule(mem_req_event, curTick() + req_tick_latency); 868 } else if (tlbPort[tlbPort_index]->isStalled()) { 869 assert(tlbPort[tlbPort_index]->retries.size() > 0); 870 871 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 872 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 873 tmp_vaddr); 874 875 tlbPort[tlbPort_index]->retries.push_back(pkt); 876 } else if (!tlbPort[tlbPort_index]->sendTimingReq(pkt)) { 877 // Stall the data port; 878 // No more packet will be issued till 879 // ruby indicates resources are freed by 880 // a recvReqRetry() call back on this port. 881 tlbPort[tlbPort_index]->stallPort(); 882 883 DPRINTF(GPUTLB, "CU%d: WF[%d][%d]: Translation for addr %#x " 884 "failed!\n", cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 885 tmp_vaddr); 886 887 tlbPort[tlbPort_index]->retries.push_back(pkt); 888 } else { 889 DPRINTF(GPUTLB, 890 "CU%d: WF[%d][%d]: Translation for addr %#x sent!\n", 891 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, tmp_vaddr); 892 } 893 } else { 894 if (pkt->cmd == MemCmd::MemFenceReq) { 895 gpuDynInst->statusBitVector = VectorMask(0); 896 } else { 897 gpuDynInst->statusBitVector &= (~(1ll << index)); 898 } 899 900 // New SenderState for the memory access 901 delete pkt->senderState; 902 903 // Because it's atomic operation, only need TLB translation state 904 pkt->senderState = new TheISA::GpuTLB::TranslationState(TLB_mode, 905 shader->gpuTc); 906 907 tlbPort[tlbPort_index]->sendFunctional(pkt); 908 909 // the addr of the packet is not modified, so we need to create a new 910 // packet, or otherwise the memory access will have the old virtual 911 // address sent in the translation packet, instead of the physical 912 // address returned by the translation. 913 PacketPtr new_pkt = new Packet(pkt->req, pkt->cmd); 914 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 915 916 // Translation is done. It is safe to send the packet to memory. 917 memPort[0]->sendFunctional(new_pkt); 918 919 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: index %d: addr %#x\n", cu_id, 920 gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 921 new_pkt->req->getPaddr()); 922 923 // safe_cast the senderState 924 TheISA::GpuTLB::TranslationState *sender_state = 925 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 926 927 delete sender_state->tlbEntry; 928 delete new_pkt; 929 delete pkt->senderState; 930 delete pkt->req; 931 delete pkt; 932 } 933} 934 935void 936ComputeUnit::sendSyncRequest(GPUDynInstPtr gpuDynInst, int index, PacketPtr pkt) 937{ 938 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 939 new ComputeUnit::DataPort::MemReqEvent(memPort[index], pkt); 940 941 942 // New SenderState for the memory access 943 pkt->senderState = new ComputeUnit::DataPort::SenderState(gpuDynInst, index, 944 nullptr); 945 946 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x sync scheduled\n", 947 cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, index, 948 pkt->req->getPaddr()); 949 950 schedule(mem_req_event, curTick() + req_tick_latency); 951} 952 953void 954ComputeUnit::injectGlobalMemFence(GPUDynInstPtr gpuDynInst, bool kernelLaunch, 955 Request* req) 956{ 957 if (!req) { 958 req = new Request(0, 0, 0, 0, masterId(), 0, gpuDynInst->wfDynId); 959 } 960 req->setPaddr(0); 961 if (kernelLaunch) { 962 req->setFlags(Request::KERNEL); 963 } 964 965 gpuDynInst->s_type = SEG_GLOBAL; 966 967 // for non-kernel MemFence operations, memorder flags are set depending 968 // on which type of request is currently being sent, so this 969 // should be set by the caller (e.g. if an inst has acq-rel 970 // semantics, it will send one acquire req an one release req) 971 gpuDynInst->setRequestFlags(req, kernelLaunch); 972 973 // a mem fence must correspond to an acquire/release request 974 assert(req->isAcquire() || req->isRelease()); 975 976 // create packet 977 PacketPtr pkt = new Packet(req, MemCmd::MemFenceReq); 978 979 // set packet's sender state 980 pkt->senderState = 981 new ComputeUnit::DataPort::SenderState(gpuDynInst, 0, nullptr); 982 983 // send the packet 984 sendSyncRequest(gpuDynInst, 0, pkt); 985} 986 987const char* 988ComputeUnit::DataPort::MemRespEvent::description() const 989{ 990 return "ComputeUnit memory response event"; 991} 992 993void 994ComputeUnit::DataPort::MemRespEvent::process() 995{ 996 DataPort::SenderState *sender_state = 997 safe_cast<DataPort::SenderState*>(pkt->senderState); 998 999 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1000 ComputeUnit *compute_unit = dataPort->computeUnit; 1001 1002 assert(gpuDynInst); 1003 1004 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: Response for addr %#x, index %d\n", 1005 compute_unit->cu_id, gpuDynInst->simdId, gpuDynInst->wfSlotId, 1006 pkt->req->getPaddr(), dataPort->index); 1007 1008 Addr paddr = pkt->req->getPaddr(); 1009 1010 if (pkt->cmd != MemCmd::MemFenceResp) { 1011 int index = gpuDynInst->memStatusVector[paddr].back(); 1012 1013 DPRINTF(GPUMem, "Response for addr %#x, index %d\n", 1014 pkt->req->getPaddr(), index); 1015 1016 gpuDynInst->memStatusVector[paddr].pop_back(); 1017 gpuDynInst->pAddr = pkt->req->getPaddr(); 1018 1019 if (pkt->isRead() || pkt->isWrite()) { 1020 1021 if (gpuDynInst->n_reg <= MAX_REGS_FOR_NON_VEC_MEM_INST) { 1022 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1023 } else { 1024 assert(gpuDynInst->statusVector[index] > 0); 1025 gpuDynInst->statusVector[index]--; 1026 1027 if (!gpuDynInst->statusVector[index]) 1028 gpuDynInst->statusBitVector &= (~(1ULL << index)); 1029 } 1030 1031 DPRINTF(GPUMem, "bitvector is now %#x\n", 1032 gpuDynInst->statusBitVector); 1033 1034 if (gpuDynInst->statusBitVector == VectorMask(0)) { 1035 auto iter = gpuDynInst->memStatusVector.begin(); 1036 auto end = gpuDynInst->memStatusVector.end(); 1037 1038 while (iter != end) { 1039 assert(iter->second.empty()); 1040 ++iter; 1041 } 1042 1043 gpuDynInst->memStatusVector.clear(); 1044 1045 if (gpuDynInst->n_reg > MAX_REGS_FOR_NON_VEC_MEM_INST) 1046 gpuDynInst->statusVector.clear(); 1047 1048 if (gpuDynInst->m_op == Enums::MO_LD || MO_A(gpuDynInst->m_op) 1049 || MO_ANR(gpuDynInst->m_op)) { 1050 assert(compute_unit->globalMemoryPipe.isGMLdRespFIFOWrRdy()); 1051 1052 compute_unit->globalMemoryPipe.getGMLdRespFIFO() 1053 .push(gpuDynInst); 1054 } else { 1055 assert(compute_unit->globalMemoryPipe.isGMStRespFIFOWrRdy()); 1056 1057 compute_unit->globalMemoryPipe.getGMStRespFIFO() 1058 .push(gpuDynInst); 1059 } 1060 1061 DPRINTF(GPUMem, "CU%d: WF[%d][%d]: packet totally complete\n", 1062 compute_unit->cu_id, gpuDynInst->simdId, 1063 gpuDynInst->wfSlotId); 1064 1065 // after clearing the status vectors, 1066 // see if there is a continuation to perform 1067 // the continuation may generate more work for 1068 // this memory request 1069 if (gpuDynInst->useContinuation) { 1070 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 1071 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1072 gpuDynInst); 1073 } 1074 } 1075 } 1076 } else { 1077 gpuDynInst->statusBitVector = VectorMask(0); 1078 1079 if (gpuDynInst->useContinuation) { 1080 assert(gpuDynInst->scope != Enums::MEMORY_SCOPE_NONE); 1081 gpuDynInst->execContinuation(gpuDynInst->staticInstruction(), 1082 gpuDynInst); 1083 } 1084 } 1085 1086 delete pkt->senderState; 1087 delete pkt->req; 1088 delete pkt; 1089} 1090 1091ComputeUnit* 1092ComputeUnitParams::create() 1093{ 1094 return new ComputeUnit(this); 1095} 1096 1097bool 1098ComputeUnit::DTLBPort::recvTimingResp(PacketPtr pkt) 1099{ 1100 Addr line = pkt->req->getPaddr(); 1101 1102 DPRINTF(GPUTLB, "CU%d: DTLBPort received %#x->%#x\n", computeUnit->cu_id, 1103 pkt->req->getVaddr(), line); 1104 1105 assert(pkt->senderState); 1106 computeUnit->tlbCycles += curTick(); 1107 1108 // pop off the TLB translation state 1109 TheISA::GpuTLB::TranslationState *translation_state = 1110 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1111 1112 // no PageFaults are permitted for data accesses 1113 if (!translation_state->tlbEntry->valid) { 1114 DTLBPort::SenderState *sender_state = 1115 safe_cast<DTLBPort::SenderState*>(translation_state->saved); 1116 1117 Wavefront *w M5_VAR_USED = 1118 computeUnit->wfList[sender_state->_gpuDynInst->simdId] 1119 [sender_state->_gpuDynInst->wfSlotId]; 1120 1121 DPRINTFN("Wave %d couldn't tranlate vaddr %#x\n", w->wfDynId, 1122 pkt->req->getVaddr()); 1123 } 1124 1125 assert(translation_state->tlbEntry->valid); 1126 1127 // update the hitLevel distribution 1128 int hit_level = translation_state->hitLevel; 1129 computeUnit->hitsPerTLBLevel[hit_level]++; 1130 1131 delete translation_state->tlbEntry; 1132 assert(!translation_state->ports.size()); 1133 pkt->senderState = translation_state->saved; 1134 1135 // for prefetch pkt 1136 BaseTLB::Mode TLB_mode = translation_state->tlbMode; 1137 1138 delete translation_state; 1139 1140 // use the original sender state to know how to close this transaction 1141 DTLBPort::SenderState *sender_state = 1142 safe_cast<DTLBPort::SenderState*>(pkt->senderState); 1143 1144 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1145 int mp_index = sender_state->portIndex; 1146 Addr vaddr = pkt->req->getVaddr(); 1147 gpuDynInst->memStatusVector[line].push_back(mp_index); 1148 gpuDynInst->tlbHitLevel[mp_index] = hit_level; 1149 1150 MemCmd requestCmd; 1151 1152 if (pkt->cmd == MemCmd::ReadResp) { 1153 requestCmd = MemCmd::ReadReq; 1154 } else if (pkt->cmd == MemCmd::WriteResp) { 1155 requestCmd = MemCmd::WriteReq; 1156 } else if (pkt->cmd == MemCmd::SwapResp) { 1157 requestCmd = MemCmd::SwapReq; 1158 } else { 1159 panic("unsupported response to request conversion %s\n", 1160 pkt->cmd.toString()); 1161 } 1162 1163 if (computeUnit->prefetchDepth) { 1164 int simdId = gpuDynInst->simdId; 1165 int wfSlotId = gpuDynInst->wfSlotId; 1166 Addr last = 0; 1167 1168 switch(computeUnit->prefetchType) { 1169 case Enums::PF_CU: 1170 last = computeUnit->lastVaddrCU[mp_index]; 1171 break; 1172 case Enums::PF_PHASE: 1173 last = computeUnit->lastVaddrSimd[simdId][mp_index]; 1174 break; 1175 case Enums::PF_WF: 1176 last = computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index]; 1177 default: 1178 break; 1179 } 1180 1181 DPRINTF(GPUPrefetch, "CU[%d][%d][%d][%d]: %#x was last\n", 1182 computeUnit->cu_id, simdId, wfSlotId, mp_index, last); 1183 1184 int stride = last ? (roundDown(vaddr, TheISA::PageBytes) - 1185 roundDown(last, TheISA::PageBytes)) >> TheISA::PageShift 1186 : 0; 1187 1188 DPRINTF(GPUPrefetch, "Stride is %d\n", stride); 1189 1190 computeUnit->lastVaddrCU[mp_index] = vaddr; 1191 computeUnit->lastVaddrSimd[simdId][mp_index] = vaddr; 1192 computeUnit->lastVaddrWF[simdId][wfSlotId][mp_index] = vaddr; 1193 1194 stride = (computeUnit->prefetchType == Enums::PF_STRIDE) ? 1195 computeUnit->prefetchStride: stride; 1196 1197 DPRINTF(GPUPrefetch, "%#x to: CU[%d][%d][%d][%d]\n", vaddr, 1198 computeUnit->cu_id, simdId, wfSlotId, mp_index); 1199 1200 DPRINTF(GPUPrefetch, "Prefetching from %#x:", vaddr); 1201 1202 // Prefetch Next few pages atomically 1203 for (int pf = 1; pf <= computeUnit->prefetchDepth; ++pf) { 1204 DPRINTF(GPUPrefetch, "%d * %d: %#x\n", pf, stride, 1205 vaddr+stride*pf*TheISA::PageBytes); 1206 1207 if (!stride) 1208 break; 1209 1210 Request *prefetch_req = new Request(0, vaddr + stride * pf * 1211 TheISA::PageBytes, 1212 sizeof(uint8_t), 0, 1213 computeUnit->masterId(), 1214 0, 0, 0); 1215 1216 PacketPtr prefetch_pkt = new Packet(prefetch_req, requestCmd); 1217 uint8_t foo = 0; 1218 prefetch_pkt->dataStatic(&foo); 1219 1220 // Because it's atomic operation, only need TLB translation state 1221 prefetch_pkt->senderState = 1222 new TheISA::GpuTLB::TranslationState(TLB_mode, 1223 computeUnit->shader->gpuTc, 1224 true); 1225 1226 // Currently prefetches are zero-latency, hence the sendFunctional 1227 sendFunctional(prefetch_pkt); 1228 1229 /* safe_cast the senderState */ 1230 TheISA::GpuTLB::TranslationState *tlb_state = 1231 safe_cast<TheISA::GpuTLB::TranslationState*>( 1232 prefetch_pkt->senderState); 1233 1234 1235 delete tlb_state->tlbEntry; 1236 delete tlb_state; 1237 delete prefetch_pkt->req; 1238 delete prefetch_pkt; 1239 } 1240 } 1241 1242 // First we must convert the response cmd back to a request cmd so that 1243 // the request can be sent through the cu's master port 1244 PacketPtr new_pkt = new Packet(pkt->req, requestCmd); 1245 new_pkt->dataStatic(pkt->getPtr<uint8_t>()); 1246 delete pkt->senderState; 1247 delete pkt; 1248 1249 // New SenderState for the memory access 1250 new_pkt->senderState = 1251 new ComputeUnit::DataPort::SenderState(gpuDynInst, mp_index, 1252 nullptr); 1253 1254 // translation is done. Schedule the mem_req_event at the appropriate 1255 // cycle to send the timing memory request to ruby 1256 ComputeUnit::DataPort::MemReqEvent *mem_req_event = 1257 new ComputeUnit::DataPort::MemReqEvent(computeUnit->memPort[mp_index], 1258 new_pkt); 1259 1260 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: index %d, addr %#x data scheduled\n", 1261 computeUnit->cu_id, gpuDynInst->simdId, 1262 gpuDynInst->wfSlotId, mp_index, new_pkt->req->getPaddr()); 1263 1264 computeUnit->schedule(mem_req_event, curTick() + 1265 computeUnit->req_tick_latency); 1266 1267 return true; 1268} 1269 1270const char* 1271ComputeUnit::DataPort::MemReqEvent::description() const 1272{ 1273 return "ComputeUnit memory request event"; 1274} 1275 1276void 1277ComputeUnit::DataPort::MemReqEvent::process() 1278{ 1279 SenderState *sender_state = safe_cast<SenderState*>(pkt->senderState); 1280 GPUDynInstPtr gpuDynInst = sender_state->_gpuDynInst; 1281 ComputeUnit *compute_unit M5_VAR_USED = dataPort->computeUnit; 1282 1283 if (!(dataPort->sendTimingReq(pkt))) { 1284 dataPort->retries.push_back(std::make_pair(pkt, gpuDynInst)); 1285 1286 DPRINTF(GPUPort, 1287 "CU%d: WF[%d][%d]: index %d, addr %#x data req failed!\n", 1288 compute_unit->cu_id, gpuDynInst->simdId, 1289 gpuDynInst->wfSlotId, dataPort->index, 1290 pkt->req->getPaddr()); 1291 } else { 1292 DPRINTF(GPUPort, 1293 "CU%d: WF[%d][%d]: index %d, addr %#x data req sent!\n", 1294 compute_unit->cu_id, gpuDynInst->simdId, 1295 gpuDynInst->wfSlotId, dataPort->index, 1296 pkt->req->getPaddr()); 1297 } 1298} 1299 1300/* 1301 * The initial translation request could have been rejected, 1302 * if <retries> queue is not Retry sending the translation 1303 * request. sendRetry() is called from the peer port whenever 1304 * a translation completes. 1305 */ 1306void 1307ComputeUnit::DTLBPort::recvReqRetry() 1308{ 1309 int len = retries.size(); 1310 1311 DPRINTF(GPUTLB, "CU%d: DTLB recvReqRetry - %d pending requests\n", 1312 computeUnit->cu_id, len); 1313 1314 assert(len > 0); 1315 assert(isStalled()); 1316 // recvReqRetry is an indication that the resource on which this 1317 // port was stalling on is freed. So, remove the stall first 1318 unstallPort(); 1319 1320 for (int i = 0; i < len; ++i) { 1321 PacketPtr pkt = retries.front(); 1322 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1323 DPRINTF(GPUTLB, "CU%d: retrying D-translaton for address%#x", vaddr); 1324 1325 if (!sendTimingReq(pkt)) { 1326 // Stall port 1327 stallPort(); 1328 DPRINTF(GPUTLB, ": failed again\n"); 1329 break; 1330 } else { 1331 DPRINTF(GPUTLB, ": successful\n"); 1332 retries.pop_front(); 1333 } 1334 } 1335} 1336 1337bool 1338ComputeUnit::ITLBPort::recvTimingResp(PacketPtr pkt) 1339{ 1340 Addr line M5_VAR_USED = pkt->req->getPaddr(); 1341 DPRINTF(GPUTLB, "CU%d: ITLBPort received %#x->%#x\n", 1342 computeUnit->cu_id, pkt->req->getVaddr(), line); 1343 1344 assert(pkt->senderState); 1345 1346 // pop off the TLB translation state 1347 TheISA::GpuTLB::TranslationState *translation_state = 1348 safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 1349 1350 bool success = translation_state->tlbEntry->valid; 1351 delete translation_state->tlbEntry; 1352 assert(!translation_state->ports.size()); 1353 pkt->senderState = translation_state->saved; 1354 delete translation_state; 1355 1356 // use the original sender state to know how to close this transaction 1357 ITLBPort::SenderState *sender_state = 1358 safe_cast<ITLBPort::SenderState*>(pkt->senderState); 1359 1360 // get the wavefront associated with this translation request 1361 Wavefront *wavefront = sender_state->wavefront; 1362 delete pkt->senderState; 1363 1364 if (success) { 1365 // pkt is reused in fetch(), don't delete it here. However, we must 1366 // reset the command to be a request so that it can be sent through 1367 // the cu's master port 1368 assert(pkt->cmd == MemCmd::ReadResp); 1369 pkt->cmd = MemCmd::ReadReq; 1370 1371 computeUnit->fetchStage.fetch(pkt, wavefront); 1372 } else { 1373 if (wavefront->dropFetch) { 1374 assert(wavefront->instructionBuffer.empty()); 1375 wavefront->dropFetch = false; 1376 } 1377 1378 wavefront->pendingFetch = 0; 1379 } 1380 1381 return true; 1382} 1383 1384/* 1385 * The initial translation request could have been rejected, if 1386 * <retries> queue is not empty. Retry sending the translation 1387 * request. sendRetry() is called from the peer port whenever 1388 * a translation completes. 1389 */ 1390void 1391ComputeUnit::ITLBPort::recvReqRetry() 1392{ 1393 1394 int len = retries.size(); 1395 DPRINTF(GPUTLB, "CU%d: ITLB recvReqRetry - %d pending requests\n", len); 1396 1397 assert(len > 0); 1398 assert(isStalled()); 1399 1400 // recvReqRetry is an indication that the resource on which this 1401 // port was stalling on is freed. So, remove the stall first 1402 unstallPort(); 1403 1404 for (int i = 0; i < len; ++i) { 1405 PacketPtr pkt = retries.front(); 1406 Addr vaddr M5_VAR_USED = pkt->req->getVaddr(); 1407 DPRINTF(GPUTLB, "CU%d: retrying I-translaton for address%#x", vaddr); 1408 1409 if (!sendTimingReq(pkt)) { 1410 stallPort(); // Stall port 1411 DPRINTF(GPUTLB, ": failed again\n"); 1412 break; 1413 } else { 1414 DPRINTF(GPUTLB, ": successful\n"); 1415 retries.pop_front(); 1416 } 1417 } 1418} 1419 1420void 1421ComputeUnit::regStats() 1422{ 1423 MemObject::regStats(); 1424 1425 tlbCycles 1426 .name(name() + ".tlb_cycles") 1427 .desc("total number of cycles for all uncoalesced requests") 1428 ; 1429 1430 tlbRequests 1431 .name(name() + ".tlb_requests") 1432 .desc("number of uncoalesced requests") 1433 ; 1434 1435 tlbLatency 1436 .name(name() + ".avg_translation_latency") 1437 .desc("Avg. translation latency for data translations") 1438 ; 1439 1440 tlbLatency = tlbCycles / tlbRequests; 1441 1442 hitsPerTLBLevel 1443 .init(4) 1444 .name(name() + ".TLB_hits_distribution") 1445 .desc("TLB hits distribution (0 for page table, x for Lx-TLB") 1446 ; 1447 1448 // fixed number of TLB levels 1449 for (int i = 0; i < 4; ++i) { 1450 if (!i) 1451 hitsPerTLBLevel.subname(i,"page_table"); 1452 else 1453 hitsPerTLBLevel.subname(i, csprintf("L%d_TLB",i)); 1454 } 1455 1456 execRateDist 1457 .init(0, 10, 2) 1458 .name(name() + ".inst_exec_rate") 1459 .desc("Instruction Execution Rate: Number of executed vector " 1460 "instructions per cycle") 1461 ; 1462 1463 ldsBankConflictDist 1464 .init(0, wfSize(), 2) 1465 .name(name() + ".lds_bank_conflicts") 1466 .desc("Number of bank conflicts per LDS memory packet") 1467 ; 1468 1469 ldsBankAccesses 1470 .name(name() + ".lds_bank_access_cnt") 1471 .desc("Total number of LDS bank accesses") 1472 ; 1473 1474 pageDivergenceDist 1475 // A wavefront can touch up to N pages per memory instruction where 1476 // N is equal to the wavefront size 1477 // The number of pages per bin can be configured (here it's 4). 1478 .init(1, wfSize(), 4) 1479 .name(name() + ".page_divergence_dist") 1480 .desc("pages touched per wf (over all mem. instr.)") 1481 ; 1482 1483 controlFlowDivergenceDist 1484 .init(1, wfSize(), 4) 1485 .name(name() + ".warp_execution_dist") 1486 .desc("number of lanes active per instruction (oval all instructions)") 1487 ; 1488 1489 activeLanesPerGMemInstrDist 1490 .init(1, wfSize(), 4) 1491 .name(name() + ".gmem_lanes_execution_dist") 1492 .desc("number of active lanes per global memory instruction") 1493 ; 1494 1495 activeLanesPerLMemInstrDist 1496 .init(1, wfSize(), 4) 1497 .name(name() + ".lmem_lanes_execution_dist") 1498 .desc("number of active lanes per local memory instruction") 1499 ; 1500 1501 numInstrExecuted 1502 .name(name() + ".num_instr_executed") 1503 .desc("number of instructions executed") 1504 ; 1505 1506 numVecOpsExecuted 1507 .name(name() + ".num_vec_ops_executed") 1508 .desc("number of vec ops executed (e.g. WF size/inst)") 1509 ; 1510 1511 totalCycles 1512 .name(name() + ".num_total_cycles") 1513 .desc("number of cycles the CU ran for") 1514 ; 1515 1516 ipc 1517 .name(name() + ".ipc") 1518 .desc("Instructions per cycle (this CU only)") 1519 ; 1520 1521 vpc 1522 .name(name() + ".vpc") 1523 .desc("Vector Operations per cycle (this CU only)") 1524 ; 1525 1526 numALUInstsExecuted 1527 .name(name() + ".num_alu_insts_executed") 1528 .desc("Number of dynamic non-GM memory insts executed") 1529 ; 1530 1531 wgBlockedDueLdsAllocation 1532 .name(name() + ".wg_blocked_due_lds_alloc") 1533 .desc("Workgroup blocked due to LDS capacity") 1534 ; 1535 1536 ipc = numInstrExecuted / totalCycles; 1537 vpc = numVecOpsExecuted / totalCycles; 1538 1539 numTimesWgBlockedDueVgprAlloc 1540 .name(name() + ".times_wg_blocked_due_vgpr_alloc") 1541 .desc("Number of times WGs are blocked due to VGPR allocation per SIMD") 1542 ; 1543 1544 dynamicGMemInstrCnt 1545 .name(name() + ".global_mem_instr_cnt") 1546 .desc("dynamic global memory instructions count") 1547 ; 1548 1549 dynamicLMemInstrCnt 1550 .name(name() + ".local_mem_instr_cnt") 1551 .desc("dynamic local memory intruction count") 1552 ; 1553 1554 numALUInstsExecuted = numInstrExecuted - dynamicGMemInstrCnt - 1555 dynamicLMemInstrCnt; 1556 1557 completedWfs 1558 .name(name() + ".num_completed_wfs") 1559 .desc("number of completed wavefronts") 1560 ; 1561 1562 numCASOps 1563 .name(name() + ".num_CAS_ops") 1564 .desc("number of compare and swap operations") 1565 ; 1566 1567 numFailedCASOps 1568 .name(name() + ".num_failed_CAS_ops") 1569 .desc("number of compare and swap operations that failed") 1570 ; 1571 1572 // register stats of pipeline stages 1573 fetchStage.regStats(); 1574 scoreboardCheckStage.regStats(); 1575 scheduleStage.regStats(); 1576 execStage.regStats(); 1577 1578 // register stats of memory pipeline 1579 globalMemoryPipe.regStats(); 1580 localMemoryPipe.regStats(); 1581} 1582 1583void 1584ComputeUnit::updatePageDivergenceDist(Addr addr) 1585{ 1586 Addr virt_page_addr = roundDown(addr, TheISA::PageBytes); 1587 1588 if (!pagesTouched.count(virt_page_addr)) 1589 pagesTouched[virt_page_addr] = 1; 1590 else 1591 pagesTouched[virt_page_addr]++; 1592} 1593 1594void 1595ComputeUnit::CUExitCallback::process() 1596{ 1597 if (computeUnit->countPages) { 1598 std::ostream *page_stat_file = 1599 simout.create(computeUnit->name().c_str())->stream(); 1600 1601 *page_stat_file << "page, wavefront accesses, workitem accesses" << 1602 std::endl; 1603 1604 for (auto iter : computeUnit->pageAccesses) { 1605 *page_stat_file << std::hex << iter.first << ","; 1606 *page_stat_file << std::dec << iter.second.first << ","; 1607 *page_stat_file << std::dec << iter.second.second << std::endl; 1608 } 1609 } 1610 } 1611 1612bool 1613ComputeUnit::isDone() const 1614{ 1615 for (int i = 0; i < numSIMDs; ++i) { 1616 if (!isSimdDone(i)) { 1617 return false; 1618 } 1619 } 1620 1621 bool glbMemBusRdy = true; 1622 for (int j = 0; j < numGlbMemUnits; ++j) { 1623 glbMemBusRdy &= vrfToGlobalMemPipeBus[j].rdy(); 1624 } 1625 bool locMemBusRdy = true; 1626 for (int j = 0; j < numLocMemUnits; ++j) { 1627 locMemBusRdy &= vrfToLocalMemPipeBus[j].rdy(); 1628 } 1629 1630 if (!globalMemoryPipe.isGMLdRespFIFOWrRdy() || 1631 !globalMemoryPipe.isGMStRespFIFOWrRdy() || 1632 !globalMemoryPipe.isGMReqFIFOWrRdy() || !localMemoryPipe.isLMReqFIFOWrRdy() 1633 || !localMemoryPipe.isLMRespFIFOWrRdy() || !locMemToVrfBus.rdy() || 1634 !glbMemToVrfBus.rdy() || !locMemBusRdy || !glbMemBusRdy) { 1635 return false; 1636 } 1637 1638 return true; 1639} 1640 1641int32_t 1642ComputeUnit::getRefCounter(const uint32_t dispatchId, const uint32_t wgId) const 1643{ 1644 return lds.getRefCounter(dispatchId, wgId); 1645} 1646 1647bool 1648ComputeUnit::isSimdDone(uint32_t simdId) const 1649{ 1650 assert(simdId < numSIMDs); 1651 1652 for (int i=0; i < numGlbMemUnits; ++i) { 1653 if (!vrfToGlobalMemPipeBus[i].rdy()) 1654 return false; 1655 } 1656 for (int i=0; i < numLocMemUnits; ++i) { 1657 if (!vrfToLocalMemPipeBus[i].rdy()) 1658 return false; 1659 } 1660 if (!aluPipe[simdId].rdy()) { 1661 return false; 1662 } 1663 1664 for (int i_wf = 0; i_wf < shader->n_wf; ++i_wf){ 1665 if (wfList[simdId][i_wf]->status != Wavefront::S_STOPPED) { 1666 return false; 1667 } 1668 } 1669 1670 return true; 1671} 1672 1673/** 1674 * send a general request to the LDS 1675 * make sure to look at the return value here as your request might be 1676 * NACK'd and returning false means that you have to have some backup plan 1677 */ 1678bool 1679ComputeUnit::sendToLds(GPUDynInstPtr gpuDynInst) 1680{ 1681 // this is just a request to carry the GPUDynInstPtr 1682 // back and forth 1683 Request *newRequest = new Request(); 1684 newRequest->setPaddr(0x0); 1685 1686 // ReadReq is not evaluted by the LDS but the Packet ctor requires this 1687 PacketPtr newPacket = new Packet(newRequest, MemCmd::ReadReq); 1688 1689 // This is the SenderState needed upon return 1690 newPacket->senderState = new LDSPort::SenderState(gpuDynInst); 1691 1692 return ldsPort->sendTimingReq(newPacket); 1693} 1694 1695/** 1696 * get the result of packets sent to the LDS when they return 1697 */ 1698bool 1699ComputeUnit::LDSPort::recvTimingResp(PacketPtr packet) 1700{ 1701 const ComputeUnit::LDSPort::SenderState *senderState = 1702 dynamic_cast<ComputeUnit::LDSPort::SenderState *>(packet->senderState); 1703 1704 fatal_if(!senderState, "did not get the right sort of sender state"); 1705 1706 GPUDynInstPtr gpuDynInst = senderState->getMemInst(); 1707 1708 delete packet->senderState; 1709 delete packet->req; 1710 delete packet; 1711 1712 computeUnit->localMemoryPipe.getLMRespFIFO().push(gpuDynInst); 1713 return true; 1714} 1715 1716/** 1717 * attempt to send this packet, either the port is already stalled, the request 1718 * is nack'd and must stall or the request goes through 1719 * when a request cannot be sent, add it to the retries queue 1720 */ 1721bool 1722ComputeUnit::LDSPort::sendTimingReq(PacketPtr pkt) 1723{ 1724 ComputeUnit::LDSPort::SenderState *sender_state = 1725 dynamic_cast<ComputeUnit::LDSPort::SenderState*>(pkt->senderState); 1726 fatal_if(!sender_state, "packet without a valid sender state"); 1727 1728 GPUDynInstPtr gpuDynInst M5_VAR_USED = sender_state->getMemInst(); 1729 1730 if (isStalled()) { 1731 fatal_if(retries.empty(), "must have retries waiting to be stalled"); 1732 1733 retries.push(pkt); 1734 1735 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: LDS send failed!\n", 1736 computeUnit->cu_id, gpuDynInst->simdId, 1737 gpuDynInst->wfSlotId); 1738 return false; 1739 } else if (!MasterPort::sendTimingReq(pkt)) { 1740 // need to stall the LDS port until a recvReqRetry() is received 1741 // this indicates that there is more space 1742 stallPort(); 1743 retries.push(pkt); 1744 1745 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req failed!\n", 1746 computeUnit->cu_id, gpuDynInst->simdId, 1747 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1748 return false; 1749 } else { 1750 DPRINTF(GPUPort, "CU%d: WF[%d][%d]: addr %#x lds req sent!\n", 1751 computeUnit->cu_id, gpuDynInst->simdId, 1752 gpuDynInst->wfSlotId, pkt->req->getPaddr()); 1753 return true; 1754 } 1755} 1756 1757/** 1758 * the bus is telling the port that there is now space so retrying stalled 1759 * requests should work now 1760 * this allows the port to have a request be nack'd and then have the receiver 1761 * say when there is space, rather than simply retrying the send every cycle 1762 */ 1763void 1764ComputeUnit::LDSPort::recvReqRetry() 1765{ 1766 auto queueSize = retries.size(); 1767 1768 DPRINTF(GPUPort, "CU%d: LDSPort recvReqRetry - %d pending requests\n", 1769 computeUnit->cu_id, queueSize); 1770 1771 fatal_if(queueSize < 1, 1772 "why was there a recvReqRetry() with no pending reqs?"); 1773 fatal_if(!isStalled(), 1774 "recvReqRetry() happened when the port was not stalled"); 1775 1776 unstallPort(); 1777 1778 while (!retries.empty()) { 1779 PacketPtr packet = retries.front(); 1780 1781 DPRINTF(GPUPort, "CU%d: retrying LDS send\n", computeUnit->cu_id); 1782 1783 if (!MasterPort::sendTimingReq(packet)) { 1784 // Stall port 1785 stallPort(); 1786 DPRINTF(GPUPort, ": LDS send failed again\n"); 1787 break; 1788 } else { 1789 DPRINTF(GPUTLB, ": LDS send successful\n"); 1790 retries.pop(); 1791 } 1792 } 1793}
|