tlb_coalescer.cc revision 13892
111308Santhony.gutierrez@amd.com/* 211308Santhony.gutierrez@amd.com * Copyright (c) 2011-2015 Advanced Micro Devices, Inc. 311308Santhony.gutierrez@amd.com * All rights reserved. 411308Santhony.gutierrez@amd.com * 511308Santhony.gutierrez@amd.com * For use for simulation and test purposes only 611308Santhony.gutierrez@amd.com * 711308Santhony.gutierrez@amd.com * Redistribution and use in source and binary forms, with or without 811308Santhony.gutierrez@amd.com * modification, are permitted provided that the following conditions are met: 911308Santhony.gutierrez@amd.com * 1011308Santhony.gutierrez@amd.com * 1. Redistributions of source code must retain the above copyright notice, 1111308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer. 1211308Santhony.gutierrez@amd.com * 1311308Santhony.gutierrez@amd.com * 2. Redistributions in binary form must reproduce the above copyright notice, 1411308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer in the documentation 1511308Santhony.gutierrez@amd.com * and/or other materials provided with the distribution. 1611308Santhony.gutierrez@amd.com * 1712697Santhony.gutierrez@amd.com * 3. Neither the name of the copyright holder nor the names of its 1812697Santhony.gutierrez@amd.com * contributors may be used to endorse or promote products derived from this 1912697Santhony.gutierrez@amd.com * software without specific prior written permission. 2011308Santhony.gutierrez@amd.com * 2111308Santhony.gutierrez@amd.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2211308Santhony.gutierrez@amd.com * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2311308Santhony.gutierrez@amd.com * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2411308Santhony.gutierrez@amd.com * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 2511308Santhony.gutierrez@amd.com * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2611308Santhony.gutierrez@amd.com * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2711308Santhony.gutierrez@amd.com * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2811308Santhony.gutierrez@amd.com * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2911308Santhony.gutierrez@amd.com * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 3011308Santhony.gutierrez@amd.com * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3111308Santhony.gutierrez@amd.com * POSSIBILITY OF SUCH DAMAGE. 3211308Santhony.gutierrez@amd.com * 3312697Santhony.gutierrez@amd.com * Authors: Lisa Hsu 3411308Santhony.gutierrez@amd.com */ 3511308Santhony.gutierrez@amd.com 3611308Santhony.gutierrez@amd.com#include "gpu-compute/tlb_coalescer.hh" 3711308Santhony.gutierrez@amd.com 3811308Santhony.gutierrez@amd.com#include <cstring> 3911308Santhony.gutierrez@amd.com 4013449Sgabeblack@google.com#include "base/logging.hh" 4111308Santhony.gutierrez@amd.com#include "debug/GPUTLB.hh" 4212717Sbrandon.potter@amd.com#include "sim/process.hh" 4311308Santhony.gutierrez@amd.com 4412126Sspwilson2@wisc.eduTLBCoalescer::TLBCoalescer(const Params *p) 4513892Sgabeblack@google.com : ClockedObject(p), 4612126Sspwilson2@wisc.edu clock(p->clk_domain->clockPeriod()), 4712126Sspwilson2@wisc.edu TLBProbesPerCycle(p->probesPerCycle), 4812126Sspwilson2@wisc.edu coalescingWindow(p->coalescingWindow), 4912126Sspwilson2@wisc.edu disableCoalescing(p->disableCoalescing), 5012126Sspwilson2@wisc.edu probeTLBEvent([this]{ processProbeTLBEvent(); }, 5112126Sspwilson2@wisc.edu "Probe the TLB below", 5212126Sspwilson2@wisc.edu false, Event::CPU_Tick_Pri), 5312126Sspwilson2@wisc.edu cleanupEvent([this]{ processCleanupEvent(); }, 5412126Sspwilson2@wisc.edu "Cleanup issuedTranslationsTable hashmap", 5512126Sspwilson2@wisc.edu false, Event::Maximum_Pri) 5611308Santhony.gutierrez@amd.com{ 5711308Santhony.gutierrez@amd.com // create the slave ports based on the number of connected ports 5811308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_slave_connection_count; ++i) { 5911308Santhony.gutierrez@amd.com cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d", name(), i), 6011308Santhony.gutierrez@amd.com this, i)); 6111308Santhony.gutierrez@amd.com } 6211308Santhony.gutierrez@amd.com 6311308Santhony.gutierrez@amd.com // create the master ports based on the number of connected ports 6411308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_master_connection_count; ++i) { 6511308Santhony.gutierrez@amd.com memSidePort.push_back(new MemSidePort(csprintf("%s-port%d", name(), i), 6611308Santhony.gutierrez@amd.com this, i)); 6711308Santhony.gutierrez@amd.com } 6811308Santhony.gutierrez@amd.com} 6911308Santhony.gutierrez@amd.com 7013784Sgabeblack@google.comPort & 7113784Sgabeblack@google.comTLBCoalescer::getPort(const std::string &if_name, PortID idx) 7211308Santhony.gutierrez@amd.com{ 7311308Santhony.gutierrez@amd.com if (if_name == "slave") { 7411308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(cpuSidePort.size())) { 7513784Sgabeblack@google.com panic("TLBCoalescer::getPort: unknown index %d\n", idx); 7611308Santhony.gutierrez@amd.com } 7711308Santhony.gutierrez@amd.com 7811308Santhony.gutierrez@amd.com return *cpuSidePort[idx]; 7913784Sgabeblack@google.com } else if (if_name == "master") { 8011308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(memSidePort.size())) { 8113784Sgabeblack@google.com panic("TLBCoalescer::getPort: unknown index %d\n", idx); 8211308Santhony.gutierrez@amd.com } 8311308Santhony.gutierrez@amd.com 8411308Santhony.gutierrez@amd.com return *memSidePort[idx]; 8511308Santhony.gutierrez@amd.com } else { 8613784Sgabeblack@google.com panic("TLBCoalescer::getPort: unknown port %s\n", if_name); 8711308Santhony.gutierrez@amd.com } 8811308Santhony.gutierrez@amd.com} 8911308Santhony.gutierrez@amd.com 9011308Santhony.gutierrez@amd.com/* 9111308Santhony.gutierrez@amd.com * This method returns true if the <incoming_pkt> 9211308Santhony.gutierrez@amd.com * can be coalesced with <coalesced_pkt> and false otherwise. 9311308Santhony.gutierrez@amd.com * A given set of rules is checked. 9411308Santhony.gutierrez@amd.com * The rules can potentially be modified based on the TLB level. 9511308Santhony.gutierrez@amd.com */ 9611308Santhony.gutierrez@amd.combool 9711308Santhony.gutierrez@amd.comTLBCoalescer::canCoalesce(PacketPtr incoming_pkt, PacketPtr coalesced_pkt) 9811308Santhony.gutierrez@amd.com{ 9911308Santhony.gutierrez@amd.com if (disableCoalescing) 10011308Santhony.gutierrez@amd.com return false; 10111308Santhony.gutierrez@amd.com 10211308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *incoming_state = 10311308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(incoming_pkt->senderState); 10411308Santhony.gutierrez@amd.com 10511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *coalesced_state = 10611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(coalesced_pkt->senderState); 10711308Santhony.gutierrez@amd.com 10811308Santhony.gutierrez@amd.com // Rule 1: Coalesce requests only if they 10911308Santhony.gutierrez@amd.com // fall within the same virtual page 11011308Santhony.gutierrez@amd.com Addr incoming_virt_page_addr = roundDown(incoming_pkt->req->getVaddr(), 11111308Santhony.gutierrez@amd.com TheISA::PageBytes); 11211308Santhony.gutierrez@amd.com 11311308Santhony.gutierrez@amd.com Addr coalesced_virt_page_addr = roundDown(coalesced_pkt->req->getVaddr(), 11411308Santhony.gutierrez@amd.com TheISA::PageBytes); 11511308Santhony.gutierrez@amd.com 11611308Santhony.gutierrez@amd.com if (incoming_virt_page_addr != coalesced_virt_page_addr) 11711308Santhony.gutierrez@amd.com return false; 11811308Santhony.gutierrez@amd.com 11911308Santhony.gutierrez@amd.com //* Rule 2: Coalesce requests only if they 12011308Santhony.gutierrez@amd.com // share a TLB Mode, i.e. they are both read 12111308Santhony.gutierrez@amd.com // or write requests. 12211308Santhony.gutierrez@amd.com BaseTLB::Mode incoming_mode = incoming_state->tlbMode; 12311308Santhony.gutierrez@amd.com BaseTLB::Mode coalesced_mode = coalesced_state->tlbMode; 12411308Santhony.gutierrez@amd.com 12511308Santhony.gutierrez@amd.com if (incoming_mode != coalesced_mode) 12611308Santhony.gutierrez@amd.com return false; 12711308Santhony.gutierrez@amd.com 12811308Santhony.gutierrez@amd.com // when we can coalesce a packet update the reqCnt 12911308Santhony.gutierrez@amd.com // that is the number of packets represented by 13011308Santhony.gutierrez@amd.com // this coalesced packet 13111308Santhony.gutierrez@amd.com if (!incoming_state->prefetch) 13211308Santhony.gutierrez@amd.com coalesced_state->reqCnt.back() += incoming_state->reqCnt.back(); 13311308Santhony.gutierrez@amd.com 13411308Santhony.gutierrez@amd.com return true; 13511308Santhony.gutierrez@amd.com} 13611308Santhony.gutierrez@amd.com 13711308Santhony.gutierrez@amd.com/* 13811308Santhony.gutierrez@amd.com * We need to update the physical addresses of all the translation requests 13911308Santhony.gutierrez@amd.com * that were coalesced into the one that just returned. 14011308Santhony.gutierrez@amd.com */ 14111308Santhony.gutierrez@amd.comvoid 14211308Santhony.gutierrez@amd.comTLBCoalescer::updatePhysAddresses(PacketPtr pkt) 14311308Santhony.gutierrez@amd.com{ 14411308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 14511308Santhony.gutierrez@amd.com 14611308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Update phys. addr. for %d coalesced reqs for page %#x\n", 14711308Santhony.gutierrez@amd.com issuedTranslationsTable[virt_page_addr].size(), virt_page_addr); 14811308Santhony.gutierrez@amd.com 14911308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 15011308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 15111308Santhony.gutierrez@amd.com 15212717Sbrandon.potter@amd.com TheISA::TlbEntry *tlb_entry = sender_state->tlbEntry; 15311308Santhony.gutierrez@amd.com assert(tlb_entry); 15411308Santhony.gutierrez@amd.com Addr first_entry_vaddr = tlb_entry->vaddr; 15511308Santhony.gutierrez@amd.com Addr first_entry_paddr = tlb_entry->paddr; 15611308Santhony.gutierrez@amd.com int page_size = tlb_entry->size(); 15711308Santhony.gutierrez@amd.com bool uncacheable = tlb_entry->uncacheable; 15811308Santhony.gutierrez@amd.com int first_hit_level = sender_state->hitLevel; 15911308Santhony.gutierrez@amd.com 16011308Santhony.gutierrez@amd.com // Get the physical page address of the translated request 16111308Santhony.gutierrez@amd.com // Using the page_size specified in the TLBEntry allows us 16211308Santhony.gutierrez@amd.com // to support different page sizes. 16311308Santhony.gutierrez@amd.com Addr phys_page_paddr = pkt->req->getPaddr(); 16411308Santhony.gutierrez@amd.com phys_page_paddr &= ~(page_size - 1); 16511308Santhony.gutierrez@amd.com 16611308Santhony.gutierrez@amd.com for (int i = 0; i < issuedTranslationsTable[virt_page_addr].size(); ++i) { 16711308Santhony.gutierrez@amd.com PacketPtr local_pkt = issuedTranslationsTable[virt_page_addr][i]; 16811308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 16911308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>( 17011308Santhony.gutierrez@amd.com local_pkt->senderState); 17111308Santhony.gutierrez@amd.com 17211308Santhony.gutierrez@amd.com // we are sending the packet back, so pop the reqCnt associated 17311308Santhony.gutierrez@amd.com // with this level in the TLB hiearchy 17411308Santhony.gutierrez@amd.com if (!sender_state->prefetch) 17511308Santhony.gutierrez@amd.com sender_state->reqCnt.pop_back(); 17611308Santhony.gutierrez@amd.com 17711308Santhony.gutierrez@amd.com /* 17811308Santhony.gutierrez@amd.com * Only the first packet from this coalesced request has been 17911308Santhony.gutierrez@amd.com * translated. Grab the translated phys. page addr and update the 18011308Santhony.gutierrez@amd.com * physical addresses of the remaining packets with the appropriate 18111308Santhony.gutierrez@amd.com * page offsets. 18211308Santhony.gutierrez@amd.com */ 18311308Santhony.gutierrez@amd.com if (i) { 18411308Santhony.gutierrez@amd.com Addr paddr = phys_page_paddr; 18511308Santhony.gutierrez@amd.com paddr |= (local_pkt->req->getVaddr() & (page_size - 1)); 18611308Santhony.gutierrez@amd.com local_pkt->req->setPaddr(paddr); 18711308Santhony.gutierrez@amd.com 18811308Santhony.gutierrez@amd.com if (uncacheable) 18911308Santhony.gutierrez@amd.com local_pkt->req->setFlags(Request::UNCACHEABLE); 19011308Santhony.gutierrez@amd.com 19111308Santhony.gutierrez@amd.com // update senderState->tlbEntry, so we can insert 19211308Santhony.gutierrez@amd.com // the correct TLBEentry in the TLBs above. 19312717Sbrandon.potter@amd.com auto p = sender_state->tc->getProcessPtr(); 19411308Santhony.gutierrez@amd.com sender_state->tlbEntry = 19512717Sbrandon.potter@amd.com new TheISA::TlbEntry(p->pid(), first_entry_vaddr, 19612717Sbrandon.potter@amd.com first_entry_paddr, false, false); 19711308Santhony.gutierrez@amd.com 19811308Santhony.gutierrez@amd.com // update the hitLevel for all uncoalesced reqs 19911308Santhony.gutierrez@amd.com // so that each packet knows where it hit 20011308Santhony.gutierrez@amd.com // (used for statistics in the CUs) 20111308Santhony.gutierrez@amd.com sender_state->hitLevel = first_hit_level; 20211308Santhony.gutierrez@amd.com } 20311308Santhony.gutierrez@amd.com 20411308Santhony.gutierrez@amd.com SlavePort *return_port = sender_state->ports.back(); 20511308Santhony.gutierrez@amd.com sender_state->ports.pop_back(); 20611308Santhony.gutierrez@amd.com 20711308Santhony.gutierrez@amd.com // Translation is done - Convert to a response pkt if necessary and 20811308Santhony.gutierrez@amd.com // send the translation back 20911308Santhony.gutierrez@amd.com if (local_pkt->isRequest()) { 21011308Santhony.gutierrez@amd.com local_pkt->makeTimingResponse(); 21111308Santhony.gutierrez@amd.com } 21211308Santhony.gutierrez@amd.com 21311308Santhony.gutierrez@amd.com return_port->sendTimingResp(local_pkt); 21411308Santhony.gutierrez@amd.com } 21511308Santhony.gutierrez@amd.com 21611308Santhony.gutierrez@amd.com // schedule clean up for end of this cycle 21711308Santhony.gutierrez@amd.com // This is a maximum priority event and must be on 21811308Santhony.gutierrez@amd.com // the same cycle as GPUTLB cleanup event to prevent 21911308Santhony.gutierrez@amd.com // race conditions with an IssueProbeEvent caused by 22011308Santhony.gutierrez@amd.com // MemSidePort::recvReqRetry 22111308Santhony.gutierrez@amd.com cleanupQueue.push(virt_page_addr); 22211308Santhony.gutierrez@amd.com 22311308Santhony.gutierrez@amd.com if (!cleanupEvent.scheduled()) 22411308Santhony.gutierrez@amd.com schedule(cleanupEvent, curTick()); 22511308Santhony.gutierrez@amd.com} 22611308Santhony.gutierrez@amd.com 22711308Santhony.gutierrez@amd.com// Receive translation requests, create a coalesced request, 22811308Santhony.gutierrez@amd.com// and send them to the TLB (TLBProbesPerCycle) 22911308Santhony.gutierrez@amd.combool 23011308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvTimingReq(PacketPtr pkt) 23111308Santhony.gutierrez@amd.com{ 23211308Santhony.gutierrez@amd.com // first packet of a coalesced request 23311308Santhony.gutierrez@amd.com PacketPtr first_packet = nullptr; 23411308Santhony.gutierrez@amd.com // true if we are able to do coalescing 23511308Santhony.gutierrez@amd.com bool didCoalesce = false; 23611308Santhony.gutierrez@amd.com // number of coalesced reqs for a given window 23711308Santhony.gutierrez@amd.com int coalescedReq_cnt = 0; 23811308Santhony.gutierrez@amd.com 23911308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 24011308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 24111308Santhony.gutierrez@amd.com 24211308Santhony.gutierrez@amd.com // push back the port to remember the path back 24311308Santhony.gutierrez@amd.com sender_state->ports.push_back(this); 24411308Santhony.gutierrez@amd.com 24511308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 24611308Santhony.gutierrez@amd.com 24711308Santhony.gutierrez@amd.com if (update_stats) { 24811308Santhony.gutierrez@amd.com // if reqCnt is empty then this packet does not represent 24911308Santhony.gutierrez@amd.com // multiple uncoalesced reqs(pkts) but just a single pkt. 25011308Santhony.gutierrez@amd.com // If it does though then the reqCnt for each level in the 25111308Santhony.gutierrez@amd.com // hierarchy accumulates the total number of reqs this packet 25211308Santhony.gutierrez@amd.com // represents 25311308Santhony.gutierrez@amd.com int req_cnt = 1; 25411308Santhony.gutierrez@amd.com 25511308Santhony.gutierrez@amd.com if (!sender_state->reqCnt.empty()) 25611308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 25711308Santhony.gutierrez@amd.com 25811308Santhony.gutierrez@amd.com sender_state->reqCnt.push_back(req_cnt); 25911308Santhony.gutierrez@amd.com 26011308Santhony.gutierrez@amd.com // update statistics 26111308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 26211308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 26311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "receiving pkt w/ req_cnt %d\n", req_cnt); 26411308Santhony.gutierrez@amd.com coalescer->queuingCycles -= (curTick() * req_cnt); 26511308Santhony.gutierrez@amd.com coalescer->localqueuingCycles -= curTick(); 26611308Santhony.gutierrez@amd.com } 26711308Santhony.gutierrez@amd.com 26811308Santhony.gutierrez@amd.com // FIXME if you want to coalesce not based on the issueTime 26911308Santhony.gutierrez@amd.com // of the packets (i.e., from the compute unit's perspective) 27011308Santhony.gutierrez@amd.com // but based on when they reached this coalescer then 27111308Santhony.gutierrez@amd.com // remove the following if statement and use curTick() or 27211308Santhony.gutierrez@amd.com // coalescingWindow for the tick_index. 27311308Santhony.gutierrez@amd.com if (!sender_state->issueTime) 27411308Santhony.gutierrez@amd.com sender_state->issueTime = curTick(); 27511308Santhony.gutierrez@amd.com 27611308Santhony.gutierrez@amd.com // The tick index is used as a key to the coalescerFIFO hashmap. 27711308Santhony.gutierrez@amd.com // It is shared by all candidates that fall within the 27811308Santhony.gutierrez@amd.com // given coalescingWindow. 27911308Santhony.gutierrez@amd.com int64_t tick_index = sender_state->issueTime / coalescer->coalescingWindow; 28011308Santhony.gutierrez@amd.com 28111308Santhony.gutierrez@amd.com if (coalescer->coalescerFIFO.count(tick_index)) { 28211308Santhony.gutierrez@amd.com coalescedReq_cnt = coalescer->coalescerFIFO[tick_index].size(); 28311308Santhony.gutierrez@amd.com } 28411308Santhony.gutierrez@amd.com 28511308Santhony.gutierrez@amd.com // see if we can coalesce the incoming pkt with another 28611308Santhony.gutierrez@amd.com // coalesced request with the same tick_index 28711308Santhony.gutierrez@amd.com for (int i = 0; i < coalescedReq_cnt; ++i) { 28811308Santhony.gutierrez@amd.com first_packet = coalescer->coalescerFIFO[tick_index][i][0]; 28911308Santhony.gutierrez@amd.com 29011308Santhony.gutierrez@amd.com if (coalescer->canCoalesce(pkt, first_packet)) { 29111308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].push_back(pkt); 29211308Santhony.gutierrez@amd.com 29311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Coalesced req %i w/ tick_index %d has %d reqs\n", 29411308Santhony.gutierrez@amd.com i, tick_index, 29511308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].size()); 29611308Santhony.gutierrez@amd.com 29711308Santhony.gutierrez@amd.com didCoalesce = true; 29811308Santhony.gutierrez@amd.com break; 29911308Santhony.gutierrez@amd.com } 30011308Santhony.gutierrez@amd.com } 30111308Santhony.gutierrez@amd.com 30211308Santhony.gutierrez@amd.com // if this is the first request for this tick_index 30311308Santhony.gutierrez@amd.com // or we did not manage to coalesce, update stats 30411308Santhony.gutierrez@amd.com // and make necessary allocations. 30511308Santhony.gutierrez@amd.com if (!coalescedReq_cnt || !didCoalesce) { 30611308Santhony.gutierrez@amd.com if (update_stats) 30711308Santhony.gutierrez@amd.com coalescer->coalescedAccesses++; 30811308Santhony.gutierrez@amd.com 30911308Santhony.gutierrez@amd.com std::vector<PacketPtr> new_array; 31011308Santhony.gutierrez@amd.com new_array.push_back(pkt); 31111308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].push_back(new_array); 31211308Santhony.gutierrez@amd.com 31311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescerFIFO[%d] now has %d coalesced reqs after " 31411308Santhony.gutierrez@amd.com "push\n", tick_index, 31511308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].size()); 31611308Santhony.gutierrez@amd.com } 31711308Santhony.gutierrez@amd.com 31811308Santhony.gutierrez@amd.com //schedule probeTLBEvent next cycle to send the 31911308Santhony.gutierrez@amd.com //coalesced requests to the TLB 32011308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) { 32111308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 32211308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 32311308Santhony.gutierrez@amd.com } 32411308Santhony.gutierrez@amd.com 32511308Santhony.gutierrez@amd.com return true; 32611308Santhony.gutierrez@amd.com} 32711308Santhony.gutierrez@amd.com 32811308Santhony.gutierrez@amd.comvoid 32911308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvReqRetry() 33011308Santhony.gutierrez@amd.com{ 33113449Sgabeblack@google.com panic("recvReqRetry called"); 33211308Santhony.gutierrez@amd.com} 33311308Santhony.gutierrez@amd.com 33411308Santhony.gutierrez@amd.comvoid 33511308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvFunctional(PacketPtr pkt) 33611308Santhony.gutierrez@amd.com{ 33711308Santhony.gutierrez@amd.com 33811308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 33911308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 34011308Santhony.gutierrez@amd.com 34111308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 34211308Santhony.gutierrez@amd.com 34311308Santhony.gutierrez@amd.com if (update_stats) 34411308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 34511308Santhony.gutierrez@amd.com 34611308Santhony.gutierrez@amd.com // If there is a pending timing request for this virtual address 34711308Santhony.gutierrez@amd.com // print a warning message. This is a temporary caveat of 34811308Santhony.gutierrez@amd.com // the current simulator where atomic and timing requests can 34911308Santhony.gutierrez@amd.com // coexist. FIXME remove this check/warning in the future. 35011308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 35111308Santhony.gutierrez@amd.com int map_count = coalescer->issuedTranslationsTable.count(virt_page_addr); 35211308Santhony.gutierrez@amd.com 35311308Santhony.gutierrez@amd.com if (map_count) { 35411308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Warning! Functional access to addr %#x sees timing " 35511308Santhony.gutierrez@amd.com "req. pending\n", virt_page_addr); 35611308Santhony.gutierrez@amd.com } 35711308Santhony.gutierrez@amd.com 35811308Santhony.gutierrez@amd.com coalescer->memSidePort[0]->sendFunctional(pkt); 35911308Santhony.gutierrez@amd.com} 36011308Santhony.gutierrez@amd.com 36111308Santhony.gutierrez@amd.comAddrRangeList 36211308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::getAddrRanges() const 36311308Santhony.gutierrez@amd.com{ 36411308Santhony.gutierrez@amd.com // currently not checked by the master 36511308Santhony.gutierrez@amd.com AddrRangeList ranges; 36611308Santhony.gutierrez@amd.com 36711308Santhony.gutierrez@amd.com return ranges; 36811308Santhony.gutierrez@amd.com} 36911308Santhony.gutierrez@amd.com 37011308Santhony.gutierrez@amd.combool 37111308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvTimingResp(PacketPtr pkt) 37211308Santhony.gutierrez@amd.com{ 37311308Santhony.gutierrez@amd.com // a translation completed and returned 37411308Santhony.gutierrez@amd.com coalescer->updatePhysAddresses(pkt); 37511308Santhony.gutierrez@amd.com 37611308Santhony.gutierrez@amd.com return true; 37711308Santhony.gutierrez@amd.com} 37811308Santhony.gutierrez@amd.com 37911308Santhony.gutierrez@amd.comvoid 38011308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvReqRetry() 38111308Santhony.gutierrez@amd.com{ 38211308Santhony.gutierrez@amd.com //we've receeived a retry. Schedule a probeTLBEvent 38311308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) 38411308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 38511308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 38611308Santhony.gutierrez@amd.com} 38711308Santhony.gutierrez@amd.com 38811308Santhony.gutierrez@amd.comvoid 38911308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvFunctional(PacketPtr pkt) 39011308Santhony.gutierrez@amd.com{ 39111308Santhony.gutierrez@amd.com fatal("Memory side recvFunctional() not implemented in TLB coalescer.\n"); 39211308Santhony.gutierrez@amd.com} 39311308Santhony.gutierrez@amd.com 39411308Santhony.gutierrez@amd.com/* 39511308Santhony.gutierrez@amd.com * Here we scan the coalescer FIFO and issue the max 39611308Santhony.gutierrez@amd.com * number of permitted probes to the TLB below. We 39711308Santhony.gutierrez@amd.com * permit bypassing of coalesced requests for the same 39811308Santhony.gutierrez@amd.com * tick_index. 39911308Santhony.gutierrez@amd.com * 40011308Santhony.gutierrez@amd.com * We do not access the next tick_index unless we've 40111308Santhony.gutierrez@amd.com * drained the previous one. The coalesced requests 40211308Santhony.gutierrez@amd.com * that are successfully sent are moved to the 40311308Santhony.gutierrez@amd.com * issuedTranslationsTable table (the table which keeps 40411308Santhony.gutierrez@amd.com * track of the outstanding reqs) 40511308Santhony.gutierrez@amd.com */ 40611308Santhony.gutierrez@amd.comvoid 40712126Sspwilson2@wisc.eduTLBCoalescer::processProbeTLBEvent() 40811308Santhony.gutierrez@amd.com{ 40911308Santhony.gutierrez@amd.com // number of TLB probes sent so far 41011308Santhony.gutierrez@amd.com int sent_probes = 0; 41111308Santhony.gutierrez@amd.com // rejected denotes a blocking event 41211308Santhony.gutierrez@amd.com bool rejected = false; 41311308Santhony.gutierrez@amd.com 41411308Santhony.gutierrez@amd.com // It is set to true either when the recvTiming of the TLB below 41511308Santhony.gutierrez@amd.com // returns false or when there is another outstanding request for the 41611308Santhony.gutierrez@amd.com // same virt. page. 41711308Santhony.gutierrez@amd.com 41812126Sspwilson2@wisc.edu DPRINTF(GPUTLB, "triggered TLBCoalescer %s\n", __func__); 41911308Santhony.gutierrez@amd.com 42012126Sspwilson2@wisc.edu for (auto iter = coalescerFIFO.begin(); 42112126Sspwilson2@wisc.edu iter != coalescerFIFO.end() && !rejected; ) { 42211308Santhony.gutierrez@amd.com int coalescedReq_cnt = iter->second.size(); 42311308Santhony.gutierrez@amd.com int i = 0; 42411308Santhony.gutierrez@amd.com int vector_index = 0; 42511308Santhony.gutierrez@amd.com 42611308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescedReq_cnt is %d for tick_index %d\n", 42711308Santhony.gutierrez@amd.com coalescedReq_cnt, iter->first); 42811308Santhony.gutierrez@amd.com 42911308Santhony.gutierrez@amd.com while (i < coalescedReq_cnt) { 43011308Santhony.gutierrez@amd.com ++i; 43111308Santhony.gutierrez@amd.com PacketPtr first_packet = iter->second[vector_index][0]; 43211308Santhony.gutierrez@amd.com 43311308Santhony.gutierrez@amd.com // compute virtual page address for this request 43411308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(first_packet->req->getVaddr(), 43511308Santhony.gutierrez@amd.com TheISA::PageBytes); 43611308Santhony.gutierrez@amd.com 43711308Santhony.gutierrez@amd.com // is there another outstanding request for the same page addr? 43811308Santhony.gutierrez@amd.com int pending_reqs = 43912126Sspwilson2@wisc.edu issuedTranslationsTable.count(virt_page_addr); 44011308Santhony.gutierrez@amd.com 44111308Santhony.gutierrez@amd.com if (pending_reqs) { 44211308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cannot issue - There are pending reqs for " 44311308Santhony.gutierrez@amd.com "page %#x\n", virt_page_addr); 44411308Santhony.gutierrez@amd.com 44511308Santhony.gutierrez@amd.com ++vector_index; 44611308Santhony.gutierrez@amd.com rejected = true; 44711308Santhony.gutierrez@amd.com 44811308Santhony.gutierrez@amd.com continue; 44911308Santhony.gutierrez@amd.com } 45011308Santhony.gutierrez@amd.com 45111308Santhony.gutierrez@amd.com // send the coalesced request for virt_page_addr 45212126Sspwilson2@wisc.edu if (!memSidePort[0]->sendTimingReq(first_packet)) { 45311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Failed to send TLB request for page %#x", 45411308Santhony.gutierrez@amd.com virt_page_addr); 45511308Santhony.gutierrez@amd.com 45611308Santhony.gutierrez@amd.com // No need for a retries queue since we are already buffering 45711308Santhony.gutierrez@amd.com // the coalesced request in coalescerFIFO. 45811308Santhony.gutierrez@amd.com rejected = true; 45911308Santhony.gutierrez@amd.com ++vector_index; 46011308Santhony.gutierrez@amd.com } else { 46111308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *tmp_sender_state = 46211308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*> 46311308Santhony.gutierrez@amd.com (first_packet->senderState); 46411308Santhony.gutierrez@amd.com 46511308Santhony.gutierrez@amd.com bool update_stats = !tmp_sender_state->prefetch; 46611308Santhony.gutierrez@amd.com 46711308Santhony.gutierrez@amd.com if (update_stats) { 46811308Santhony.gutierrez@amd.com // req_cnt is total number of packets represented 46911308Santhony.gutierrez@amd.com // by the one we just sent counting all the way from 47011308Santhony.gutierrez@amd.com // the top of TLB hiearchy (i.e., from the CU) 47111308Santhony.gutierrez@amd.com int req_cnt = tmp_sender_state->reqCnt.back(); 47212126Sspwilson2@wisc.edu queuingCycles += (curTick() * req_cnt); 47311308Santhony.gutierrez@amd.com 47411308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "%s sending pkt w/ req_cnt %d\n", 47512126Sspwilson2@wisc.edu name(), req_cnt); 47611308Santhony.gutierrez@amd.com 47711308Santhony.gutierrez@amd.com // pkt_cnt is number of packets we coalesced into the one 47811308Santhony.gutierrez@amd.com // we just sent but only at this coalescer level 47911308Santhony.gutierrez@amd.com int pkt_cnt = iter->second[vector_index].size(); 48012126Sspwilson2@wisc.edu localqueuingCycles += (curTick() * pkt_cnt); 48111308Santhony.gutierrez@amd.com } 48211308Santhony.gutierrez@amd.com 48311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Successfully sent TLB request for page %#x", 48411308Santhony.gutierrez@amd.com virt_page_addr); 48511308Santhony.gutierrez@amd.com 48611308Santhony.gutierrez@amd.com //copy coalescedReq to issuedTranslationsTable 48712126Sspwilson2@wisc.edu issuedTranslationsTable[virt_page_addr] 48811308Santhony.gutierrez@amd.com = iter->second[vector_index]; 48911308Santhony.gutierrez@amd.com 49011308Santhony.gutierrez@amd.com //erase the entry of this coalesced req 49111308Santhony.gutierrez@amd.com iter->second.erase(iter->second.begin() + vector_index); 49211308Santhony.gutierrez@amd.com 49311308Santhony.gutierrez@amd.com if (iter->second.empty()) 49411308Santhony.gutierrez@amd.com assert(i == coalescedReq_cnt); 49511308Santhony.gutierrez@amd.com 49611308Santhony.gutierrez@amd.com sent_probes++; 49712126Sspwilson2@wisc.edu if (sent_probes == TLBProbesPerCycle) 49811308Santhony.gutierrez@amd.com return; 49911308Santhony.gutierrez@amd.com } 50011308Santhony.gutierrez@amd.com } 50111308Santhony.gutierrez@amd.com 50211308Santhony.gutierrez@amd.com //if there are no more coalesced reqs for this tick_index 50311308Santhony.gutierrez@amd.com //erase the hash_map with the first iterator 50411308Santhony.gutierrez@amd.com if (iter->second.empty()) { 50512126Sspwilson2@wisc.edu coalescerFIFO.erase(iter++); 50611308Santhony.gutierrez@amd.com } else { 50711308Santhony.gutierrez@amd.com ++iter; 50811308Santhony.gutierrez@amd.com } 50911308Santhony.gutierrez@amd.com } 51011308Santhony.gutierrez@amd.com} 51111308Santhony.gutierrez@amd.com 51212126Sspwilson2@wisc.eduvoid 51312126Sspwilson2@wisc.eduTLBCoalescer::processCleanupEvent() 51411308Santhony.gutierrez@amd.com{ 51512126Sspwilson2@wisc.edu while (!cleanupQueue.empty()) { 51612126Sspwilson2@wisc.edu Addr cleanup_addr = cleanupQueue.front(); 51712126Sspwilson2@wisc.edu cleanupQueue.pop(); 51812126Sspwilson2@wisc.edu issuedTranslationsTable.erase(cleanup_addr); 51911308Santhony.gutierrez@amd.com 52011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cleanup - Delete coalescer entry with key %#x\n", 52111308Santhony.gutierrez@amd.com cleanup_addr); 52211308Santhony.gutierrez@amd.com } 52311308Santhony.gutierrez@amd.com} 52411308Santhony.gutierrez@amd.com 52511308Santhony.gutierrez@amd.comvoid 52611308Santhony.gutierrez@amd.comTLBCoalescer::regStats() 52711308Santhony.gutierrez@amd.com{ 52813892Sgabeblack@google.com ClockedObject::regStats(); 52911523Sdavid.guillen@arm.com 53011308Santhony.gutierrez@amd.com uncoalescedAccesses 53111308Santhony.gutierrez@amd.com .name(name() + ".uncoalesced_accesses") 53211308Santhony.gutierrez@amd.com .desc("Number of uncoalesced TLB accesses") 53311308Santhony.gutierrez@amd.com ; 53411308Santhony.gutierrez@amd.com 53511308Santhony.gutierrez@amd.com coalescedAccesses 53611308Santhony.gutierrez@amd.com .name(name() + ".coalesced_accesses") 53711308Santhony.gutierrez@amd.com .desc("Number of coalesced TLB accesses") 53811308Santhony.gutierrez@amd.com ; 53911308Santhony.gutierrez@amd.com 54011308Santhony.gutierrez@amd.com queuingCycles 54111308Santhony.gutierrez@amd.com .name(name() + ".queuing_cycles") 54211308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue") 54311308Santhony.gutierrez@amd.com ; 54411308Santhony.gutierrez@amd.com 54511308Santhony.gutierrez@amd.com localqueuingCycles 54611308Santhony.gutierrez@amd.com .name(name() + ".local_queuing_cycles") 54711308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue for all incoming reqs") 54811308Santhony.gutierrez@amd.com ; 54911308Santhony.gutierrez@amd.com 55011308Santhony.gutierrez@amd.com localLatency 55111308Santhony.gutierrez@amd.com .name(name() + ".local_latency") 55211308Santhony.gutierrez@amd.com .desc("Avg. latency over all incoming pkts") 55311308Santhony.gutierrez@amd.com ; 55411308Santhony.gutierrez@amd.com 55511308Santhony.gutierrez@amd.com localLatency = localqueuingCycles / uncoalescedAccesses; 55611308Santhony.gutierrez@amd.com} 55711308Santhony.gutierrez@amd.com 55811308Santhony.gutierrez@amd.com 55911308Santhony.gutierrez@amd.comTLBCoalescer* 56011308Santhony.gutierrez@amd.comTLBCoalescerParams::create() 56111308Santhony.gutierrez@amd.com{ 56211308Santhony.gutierrez@amd.com return new TLBCoalescer(this); 56311308Santhony.gutierrez@amd.com} 56411308Santhony.gutierrez@amd.com 565