tlb_coalescer.cc revision 12717
111308Santhony.gutierrez@amd.com/* 211308Santhony.gutierrez@amd.com * Copyright (c) 2011-2015 Advanced Micro Devices, Inc. 311308Santhony.gutierrez@amd.com * All rights reserved. 411308Santhony.gutierrez@amd.com * 511308Santhony.gutierrez@amd.com * For use for simulation and test purposes only 611308Santhony.gutierrez@amd.com * 711308Santhony.gutierrez@amd.com * Redistribution and use in source and binary forms, with or without 811308Santhony.gutierrez@amd.com * modification, are permitted provided that the following conditions are met: 911308Santhony.gutierrez@amd.com * 1011308Santhony.gutierrez@amd.com * 1. Redistributions of source code must retain the above copyright notice, 1111308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer. 1211308Santhony.gutierrez@amd.com * 1311308Santhony.gutierrez@amd.com * 2. Redistributions in binary form must reproduce the above copyright notice, 1411308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer in the documentation 1511308Santhony.gutierrez@amd.com * and/or other materials provided with the distribution. 1611308Santhony.gutierrez@amd.com * 1712697Santhony.gutierrez@amd.com * 3. Neither the name of the copyright holder nor the names of its 1812697Santhony.gutierrez@amd.com * contributors may be used to endorse or promote products derived from this 1912697Santhony.gutierrez@amd.com * software without specific prior written permission. 2011308Santhony.gutierrez@amd.com * 2111308Santhony.gutierrez@amd.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2211308Santhony.gutierrez@amd.com * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2311308Santhony.gutierrez@amd.com * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2411308Santhony.gutierrez@amd.com * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 2511308Santhony.gutierrez@amd.com * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2611308Santhony.gutierrez@amd.com * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2711308Santhony.gutierrez@amd.com * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2811308Santhony.gutierrez@amd.com * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2911308Santhony.gutierrez@amd.com * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 3011308Santhony.gutierrez@amd.com * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3111308Santhony.gutierrez@amd.com * POSSIBILITY OF SUCH DAMAGE. 3211308Santhony.gutierrez@amd.com * 3312697Santhony.gutierrez@amd.com * Authors: Lisa Hsu 3411308Santhony.gutierrez@amd.com */ 3511308Santhony.gutierrez@amd.com 3611308Santhony.gutierrez@amd.com#include "gpu-compute/tlb_coalescer.hh" 3711308Santhony.gutierrez@amd.com 3811308Santhony.gutierrez@amd.com#include <cstring> 3911308Santhony.gutierrez@amd.com 4011308Santhony.gutierrez@amd.com#include "debug/GPUTLB.hh" 4112717Sbrandon.potter@amd.com#include "sim/process.hh" 4211308Santhony.gutierrez@amd.com 4312126Sspwilson2@wisc.eduTLBCoalescer::TLBCoalescer(const Params *p) 4412126Sspwilson2@wisc.edu : MemObject(p), 4512126Sspwilson2@wisc.edu clock(p->clk_domain->clockPeriod()), 4612126Sspwilson2@wisc.edu TLBProbesPerCycle(p->probesPerCycle), 4712126Sspwilson2@wisc.edu coalescingWindow(p->coalescingWindow), 4812126Sspwilson2@wisc.edu disableCoalescing(p->disableCoalescing), 4912126Sspwilson2@wisc.edu probeTLBEvent([this]{ processProbeTLBEvent(); }, 5012126Sspwilson2@wisc.edu "Probe the TLB below", 5112126Sspwilson2@wisc.edu false, Event::CPU_Tick_Pri), 5212126Sspwilson2@wisc.edu cleanupEvent([this]{ processCleanupEvent(); }, 5312126Sspwilson2@wisc.edu "Cleanup issuedTranslationsTable hashmap", 5412126Sspwilson2@wisc.edu false, Event::Maximum_Pri) 5511308Santhony.gutierrez@amd.com{ 5611308Santhony.gutierrez@amd.com // create the slave ports based on the number of connected ports 5711308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_slave_connection_count; ++i) { 5811308Santhony.gutierrez@amd.com cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d", name(), i), 5911308Santhony.gutierrez@amd.com this, i)); 6011308Santhony.gutierrez@amd.com } 6111308Santhony.gutierrez@amd.com 6211308Santhony.gutierrez@amd.com // create the master ports based on the number of connected ports 6311308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_master_connection_count; ++i) { 6411308Santhony.gutierrez@amd.com memSidePort.push_back(new MemSidePort(csprintf("%s-port%d", name(), i), 6511308Santhony.gutierrez@amd.com this, i)); 6611308Santhony.gutierrez@amd.com } 6711308Santhony.gutierrez@amd.com} 6811308Santhony.gutierrez@amd.com 6911308Santhony.gutierrez@amd.comBaseSlavePort& 7011308Santhony.gutierrez@amd.comTLBCoalescer::getSlavePort(const std::string &if_name, PortID idx) 7111308Santhony.gutierrez@amd.com{ 7211308Santhony.gutierrez@amd.com if (if_name == "slave") { 7311308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(cpuSidePort.size())) { 7411308Santhony.gutierrez@amd.com panic("TLBCoalescer::getSlavePort: unknown index %d\n", idx); 7511308Santhony.gutierrez@amd.com } 7611308Santhony.gutierrez@amd.com 7711308Santhony.gutierrez@amd.com return *cpuSidePort[idx]; 7811308Santhony.gutierrez@amd.com } else { 7911308Santhony.gutierrez@amd.com panic("TLBCoalescer::getSlavePort: unknown port %s\n", if_name); 8011308Santhony.gutierrez@amd.com } 8111308Santhony.gutierrez@amd.com} 8211308Santhony.gutierrez@amd.com 8311308Santhony.gutierrez@amd.comBaseMasterPort& 8411308Santhony.gutierrez@amd.comTLBCoalescer::getMasterPort(const std::string &if_name, PortID idx) 8511308Santhony.gutierrez@amd.com{ 8611308Santhony.gutierrez@amd.com if (if_name == "master") { 8711308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(memSidePort.size())) { 8811308Santhony.gutierrez@amd.com panic("TLBCoalescer::getMasterPort: unknown index %d\n", idx); 8911308Santhony.gutierrez@amd.com } 9011308Santhony.gutierrez@amd.com 9111308Santhony.gutierrez@amd.com return *memSidePort[idx]; 9211308Santhony.gutierrez@amd.com } else { 9311308Santhony.gutierrez@amd.com panic("TLBCoalescer::getMasterPort: unknown port %s\n", if_name); 9411308Santhony.gutierrez@amd.com } 9511308Santhony.gutierrez@amd.com} 9611308Santhony.gutierrez@amd.com 9711308Santhony.gutierrez@amd.com/* 9811308Santhony.gutierrez@amd.com * This method returns true if the <incoming_pkt> 9911308Santhony.gutierrez@amd.com * can be coalesced with <coalesced_pkt> and false otherwise. 10011308Santhony.gutierrez@amd.com * A given set of rules is checked. 10111308Santhony.gutierrez@amd.com * The rules can potentially be modified based on the TLB level. 10211308Santhony.gutierrez@amd.com */ 10311308Santhony.gutierrez@amd.combool 10411308Santhony.gutierrez@amd.comTLBCoalescer::canCoalesce(PacketPtr incoming_pkt, PacketPtr coalesced_pkt) 10511308Santhony.gutierrez@amd.com{ 10611308Santhony.gutierrez@amd.com if (disableCoalescing) 10711308Santhony.gutierrez@amd.com return false; 10811308Santhony.gutierrez@amd.com 10911308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *incoming_state = 11011308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(incoming_pkt->senderState); 11111308Santhony.gutierrez@amd.com 11211308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *coalesced_state = 11311308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(coalesced_pkt->senderState); 11411308Santhony.gutierrez@amd.com 11511308Santhony.gutierrez@amd.com // Rule 1: Coalesce requests only if they 11611308Santhony.gutierrez@amd.com // fall within the same virtual page 11711308Santhony.gutierrez@amd.com Addr incoming_virt_page_addr = roundDown(incoming_pkt->req->getVaddr(), 11811308Santhony.gutierrez@amd.com TheISA::PageBytes); 11911308Santhony.gutierrez@amd.com 12011308Santhony.gutierrez@amd.com Addr coalesced_virt_page_addr = roundDown(coalesced_pkt->req->getVaddr(), 12111308Santhony.gutierrez@amd.com TheISA::PageBytes); 12211308Santhony.gutierrez@amd.com 12311308Santhony.gutierrez@amd.com if (incoming_virt_page_addr != coalesced_virt_page_addr) 12411308Santhony.gutierrez@amd.com return false; 12511308Santhony.gutierrez@amd.com 12611308Santhony.gutierrez@amd.com //* Rule 2: Coalesce requests only if they 12711308Santhony.gutierrez@amd.com // share a TLB Mode, i.e. they are both read 12811308Santhony.gutierrez@amd.com // or write requests. 12911308Santhony.gutierrez@amd.com BaseTLB::Mode incoming_mode = incoming_state->tlbMode; 13011308Santhony.gutierrez@amd.com BaseTLB::Mode coalesced_mode = coalesced_state->tlbMode; 13111308Santhony.gutierrez@amd.com 13211308Santhony.gutierrez@amd.com if (incoming_mode != coalesced_mode) 13311308Santhony.gutierrez@amd.com return false; 13411308Santhony.gutierrez@amd.com 13511308Santhony.gutierrez@amd.com // when we can coalesce a packet update the reqCnt 13611308Santhony.gutierrez@amd.com // that is the number of packets represented by 13711308Santhony.gutierrez@amd.com // this coalesced packet 13811308Santhony.gutierrez@amd.com if (!incoming_state->prefetch) 13911308Santhony.gutierrez@amd.com coalesced_state->reqCnt.back() += incoming_state->reqCnt.back(); 14011308Santhony.gutierrez@amd.com 14111308Santhony.gutierrez@amd.com return true; 14211308Santhony.gutierrez@amd.com} 14311308Santhony.gutierrez@amd.com 14411308Santhony.gutierrez@amd.com/* 14511308Santhony.gutierrez@amd.com * We need to update the physical addresses of all the translation requests 14611308Santhony.gutierrez@amd.com * that were coalesced into the one that just returned. 14711308Santhony.gutierrez@amd.com */ 14811308Santhony.gutierrez@amd.comvoid 14911308Santhony.gutierrez@amd.comTLBCoalescer::updatePhysAddresses(PacketPtr pkt) 15011308Santhony.gutierrez@amd.com{ 15111308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 15211308Santhony.gutierrez@amd.com 15311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Update phys. addr. for %d coalesced reqs for page %#x\n", 15411308Santhony.gutierrez@amd.com issuedTranslationsTable[virt_page_addr].size(), virt_page_addr); 15511308Santhony.gutierrez@amd.com 15611308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 15711308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 15811308Santhony.gutierrez@amd.com 15912717Sbrandon.potter@amd.com TheISA::TlbEntry *tlb_entry = sender_state->tlbEntry; 16011308Santhony.gutierrez@amd.com assert(tlb_entry); 16111308Santhony.gutierrez@amd.com Addr first_entry_vaddr = tlb_entry->vaddr; 16211308Santhony.gutierrez@amd.com Addr first_entry_paddr = tlb_entry->paddr; 16311308Santhony.gutierrez@amd.com int page_size = tlb_entry->size(); 16411308Santhony.gutierrez@amd.com bool uncacheable = tlb_entry->uncacheable; 16511308Santhony.gutierrez@amd.com int first_hit_level = sender_state->hitLevel; 16611308Santhony.gutierrez@amd.com 16711308Santhony.gutierrez@amd.com // Get the physical page address of the translated request 16811308Santhony.gutierrez@amd.com // Using the page_size specified in the TLBEntry allows us 16911308Santhony.gutierrez@amd.com // to support different page sizes. 17011308Santhony.gutierrez@amd.com Addr phys_page_paddr = pkt->req->getPaddr(); 17111308Santhony.gutierrez@amd.com phys_page_paddr &= ~(page_size - 1); 17211308Santhony.gutierrez@amd.com 17311308Santhony.gutierrez@amd.com for (int i = 0; i < issuedTranslationsTable[virt_page_addr].size(); ++i) { 17411308Santhony.gutierrez@amd.com PacketPtr local_pkt = issuedTranslationsTable[virt_page_addr][i]; 17511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 17611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>( 17711308Santhony.gutierrez@amd.com local_pkt->senderState); 17811308Santhony.gutierrez@amd.com 17911308Santhony.gutierrez@amd.com // we are sending the packet back, so pop the reqCnt associated 18011308Santhony.gutierrez@amd.com // with this level in the TLB hiearchy 18111308Santhony.gutierrez@amd.com if (!sender_state->prefetch) 18211308Santhony.gutierrez@amd.com sender_state->reqCnt.pop_back(); 18311308Santhony.gutierrez@amd.com 18411308Santhony.gutierrez@amd.com /* 18511308Santhony.gutierrez@amd.com * Only the first packet from this coalesced request has been 18611308Santhony.gutierrez@amd.com * translated. Grab the translated phys. page addr and update the 18711308Santhony.gutierrez@amd.com * physical addresses of the remaining packets with the appropriate 18811308Santhony.gutierrez@amd.com * page offsets. 18911308Santhony.gutierrez@amd.com */ 19011308Santhony.gutierrez@amd.com if (i) { 19111308Santhony.gutierrez@amd.com Addr paddr = phys_page_paddr; 19211308Santhony.gutierrez@amd.com paddr |= (local_pkt->req->getVaddr() & (page_size - 1)); 19311308Santhony.gutierrez@amd.com local_pkt->req->setPaddr(paddr); 19411308Santhony.gutierrez@amd.com 19511308Santhony.gutierrez@amd.com if (uncacheable) 19611308Santhony.gutierrez@amd.com local_pkt->req->setFlags(Request::UNCACHEABLE); 19711308Santhony.gutierrez@amd.com 19811308Santhony.gutierrez@amd.com // update senderState->tlbEntry, so we can insert 19911308Santhony.gutierrez@amd.com // the correct TLBEentry in the TLBs above. 20012717Sbrandon.potter@amd.com auto p = sender_state->tc->getProcessPtr(); 20111308Santhony.gutierrez@amd.com sender_state->tlbEntry = 20212717Sbrandon.potter@amd.com new TheISA::TlbEntry(p->pid(), first_entry_vaddr, 20312717Sbrandon.potter@amd.com first_entry_paddr, false, false); 20411308Santhony.gutierrez@amd.com 20511308Santhony.gutierrez@amd.com // update the hitLevel for all uncoalesced reqs 20611308Santhony.gutierrez@amd.com // so that each packet knows where it hit 20711308Santhony.gutierrez@amd.com // (used for statistics in the CUs) 20811308Santhony.gutierrez@amd.com sender_state->hitLevel = first_hit_level; 20911308Santhony.gutierrez@amd.com } 21011308Santhony.gutierrez@amd.com 21111308Santhony.gutierrez@amd.com SlavePort *return_port = sender_state->ports.back(); 21211308Santhony.gutierrez@amd.com sender_state->ports.pop_back(); 21311308Santhony.gutierrez@amd.com 21411308Santhony.gutierrez@amd.com // Translation is done - Convert to a response pkt if necessary and 21511308Santhony.gutierrez@amd.com // send the translation back 21611308Santhony.gutierrez@amd.com if (local_pkt->isRequest()) { 21711308Santhony.gutierrez@amd.com local_pkt->makeTimingResponse(); 21811308Santhony.gutierrez@amd.com } 21911308Santhony.gutierrez@amd.com 22011308Santhony.gutierrez@amd.com return_port->sendTimingResp(local_pkt); 22111308Santhony.gutierrez@amd.com } 22211308Santhony.gutierrez@amd.com 22311308Santhony.gutierrez@amd.com // schedule clean up for end of this cycle 22411308Santhony.gutierrez@amd.com // This is a maximum priority event and must be on 22511308Santhony.gutierrez@amd.com // the same cycle as GPUTLB cleanup event to prevent 22611308Santhony.gutierrez@amd.com // race conditions with an IssueProbeEvent caused by 22711308Santhony.gutierrez@amd.com // MemSidePort::recvReqRetry 22811308Santhony.gutierrez@amd.com cleanupQueue.push(virt_page_addr); 22911308Santhony.gutierrez@amd.com 23011308Santhony.gutierrez@amd.com if (!cleanupEvent.scheduled()) 23111308Santhony.gutierrez@amd.com schedule(cleanupEvent, curTick()); 23211308Santhony.gutierrez@amd.com} 23311308Santhony.gutierrez@amd.com 23411308Santhony.gutierrez@amd.com// Receive translation requests, create a coalesced request, 23511308Santhony.gutierrez@amd.com// and send them to the TLB (TLBProbesPerCycle) 23611308Santhony.gutierrez@amd.combool 23711308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvTimingReq(PacketPtr pkt) 23811308Santhony.gutierrez@amd.com{ 23911308Santhony.gutierrez@amd.com // first packet of a coalesced request 24011308Santhony.gutierrez@amd.com PacketPtr first_packet = nullptr; 24111308Santhony.gutierrez@amd.com // true if we are able to do coalescing 24211308Santhony.gutierrez@amd.com bool didCoalesce = false; 24311308Santhony.gutierrez@amd.com // number of coalesced reqs for a given window 24411308Santhony.gutierrez@amd.com int coalescedReq_cnt = 0; 24511308Santhony.gutierrez@amd.com 24611308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 24711308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 24811308Santhony.gutierrez@amd.com 24911308Santhony.gutierrez@amd.com // push back the port to remember the path back 25011308Santhony.gutierrez@amd.com sender_state->ports.push_back(this); 25111308Santhony.gutierrez@amd.com 25211308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 25311308Santhony.gutierrez@amd.com 25411308Santhony.gutierrez@amd.com if (update_stats) { 25511308Santhony.gutierrez@amd.com // if reqCnt is empty then this packet does not represent 25611308Santhony.gutierrez@amd.com // multiple uncoalesced reqs(pkts) but just a single pkt. 25711308Santhony.gutierrez@amd.com // If it does though then the reqCnt for each level in the 25811308Santhony.gutierrez@amd.com // hierarchy accumulates the total number of reqs this packet 25911308Santhony.gutierrez@amd.com // represents 26011308Santhony.gutierrez@amd.com int req_cnt = 1; 26111308Santhony.gutierrez@amd.com 26211308Santhony.gutierrez@amd.com if (!sender_state->reqCnt.empty()) 26311308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 26411308Santhony.gutierrez@amd.com 26511308Santhony.gutierrez@amd.com sender_state->reqCnt.push_back(req_cnt); 26611308Santhony.gutierrez@amd.com 26711308Santhony.gutierrez@amd.com // update statistics 26811308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 26911308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 27011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "receiving pkt w/ req_cnt %d\n", req_cnt); 27111308Santhony.gutierrez@amd.com coalescer->queuingCycles -= (curTick() * req_cnt); 27211308Santhony.gutierrez@amd.com coalescer->localqueuingCycles -= curTick(); 27311308Santhony.gutierrez@amd.com } 27411308Santhony.gutierrez@amd.com 27511308Santhony.gutierrez@amd.com // FIXME if you want to coalesce not based on the issueTime 27611308Santhony.gutierrez@amd.com // of the packets (i.e., from the compute unit's perspective) 27711308Santhony.gutierrez@amd.com // but based on when they reached this coalescer then 27811308Santhony.gutierrez@amd.com // remove the following if statement and use curTick() or 27911308Santhony.gutierrez@amd.com // coalescingWindow for the tick_index. 28011308Santhony.gutierrez@amd.com if (!sender_state->issueTime) 28111308Santhony.gutierrez@amd.com sender_state->issueTime = curTick(); 28211308Santhony.gutierrez@amd.com 28311308Santhony.gutierrez@amd.com // The tick index is used as a key to the coalescerFIFO hashmap. 28411308Santhony.gutierrez@amd.com // It is shared by all candidates that fall within the 28511308Santhony.gutierrez@amd.com // given coalescingWindow. 28611308Santhony.gutierrez@amd.com int64_t tick_index = sender_state->issueTime / coalescer->coalescingWindow; 28711308Santhony.gutierrez@amd.com 28811308Santhony.gutierrez@amd.com if (coalescer->coalescerFIFO.count(tick_index)) { 28911308Santhony.gutierrez@amd.com coalescedReq_cnt = coalescer->coalescerFIFO[tick_index].size(); 29011308Santhony.gutierrez@amd.com } 29111308Santhony.gutierrez@amd.com 29211308Santhony.gutierrez@amd.com // see if we can coalesce the incoming pkt with another 29311308Santhony.gutierrez@amd.com // coalesced request with the same tick_index 29411308Santhony.gutierrez@amd.com for (int i = 0; i < coalescedReq_cnt; ++i) { 29511308Santhony.gutierrez@amd.com first_packet = coalescer->coalescerFIFO[tick_index][i][0]; 29611308Santhony.gutierrez@amd.com 29711308Santhony.gutierrez@amd.com if (coalescer->canCoalesce(pkt, first_packet)) { 29811308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].push_back(pkt); 29911308Santhony.gutierrez@amd.com 30011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Coalesced req %i w/ tick_index %d has %d reqs\n", 30111308Santhony.gutierrez@amd.com i, tick_index, 30211308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].size()); 30311308Santhony.gutierrez@amd.com 30411308Santhony.gutierrez@amd.com didCoalesce = true; 30511308Santhony.gutierrez@amd.com break; 30611308Santhony.gutierrez@amd.com } 30711308Santhony.gutierrez@amd.com } 30811308Santhony.gutierrez@amd.com 30911308Santhony.gutierrez@amd.com // if this is the first request for this tick_index 31011308Santhony.gutierrez@amd.com // or we did not manage to coalesce, update stats 31111308Santhony.gutierrez@amd.com // and make necessary allocations. 31211308Santhony.gutierrez@amd.com if (!coalescedReq_cnt || !didCoalesce) { 31311308Santhony.gutierrez@amd.com if (update_stats) 31411308Santhony.gutierrez@amd.com coalescer->coalescedAccesses++; 31511308Santhony.gutierrez@amd.com 31611308Santhony.gutierrez@amd.com std::vector<PacketPtr> new_array; 31711308Santhony.gutierrez@amd.com new_array.push_back(pkt); 31811308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].push_back(new_array); 31911308Santhony.gutierrez@amd.com 32011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescerFIFO[%d] now has %d coalesced reqs after " 32111308Santhony.gutierrez@amd.com "push\n", tick_index, 32211308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].size()); 32311308Santhony.gutierrez@amd.com } 32411308Santhony.gutierrez@amd.com 32511308Santhony.gutierrez@amd.com //schedule probeTLBEvent next cycle to send the 32611308Santhony.gutierrez@amd.com //coalesced requests to the TLB 32711308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) { 32811308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 32911308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 33011308Santhony.gutierrez@amd.com } 33111308Santhony.gutierrez@amd.com 33211308Santhony.gutierrez@amd.com return true; 33311308Santhony.gutierrez@amd.com} 33411308Santhony.gutierrez@amd.com 33511308Santhony.gutierrez@amd.comvoid 33611308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvReqRetry() 33711308Santhony.gutierrez@amd.com{ 33811308Santhony.gutierrez@amd.com assert(false); 33911308Santhony.gutierrez@amd.com} 34011308Santhony.gutierrez@amd.com 34111308Santhony.gutierrez@amd.comvoid 34211308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvFunctional(PacketPtr pkt) 34311308Santhony.gutierrez@amd.com{ 34411308Santhony.gutierrez@amd.com 34511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 34611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 34711308Santhony.gutierrez@amd.com 34811308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 34911308Santhony.gutierrez@amd.com 35011308Santhony.gutierrez@amd.com if (update_stats) 35111308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 35211308Santhony.gutierrez@amd.com 35311308Santhony.gutierrez@amd.com // If there is a pending timing request for this virtual address 35411308Santhony.gutierrez@amd.com // print a warning message. This is a temporary caveat of 35511308Santhony.gutierrez@amd.com // the current simulator where atomic and timing requests can 35611308Santhony.gutierrez@amd.com // coexist. FIXME remove this check/warning in the future. 35711308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 35811308Santhony.gutierrez@amd.com int map_count = coalescer->issuedTranslationsTable.count(virt_page_addr); 35911308Santhony.gutierrez@amd.com 36011308Santhony.gutierrez@amd.com if (map_count) { 36111308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Warning! Functional access to addr %#x sees timing " 36211308Santhony.gutierrez@amd.com "req. pending\n", virt_page_addr); 36311308Santhony.gutierrez@amd.com } 36411308Santhony.gutierrez@amd.com 36511308Santhony.gutierrez@amd.com coalescer->memSidePort[0]->sendFunctional(pkt); 36611308Santhony.gutierrez@amd.com} 36711308Santhony.gutierrez@amd.com 36811308Santhony.gutierrez@amd.comAddrRangeList 36911308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::getAddrRanges() const 37011308Santhony.gutierrez@amd.com{ 37111308Santhony.gutierrez@amd.com // currently not checked by the master 37211308Santhony.gutierrez@amd.com AddrRangeList ranges; 37311308Santhony.gutierrez@amd.com 37411308Santhony.gutierrez@amd.com return ranges; 37511308Santhony.gutierrez@amd.com} 37611308Santhony.gutierrez@amd.com 37711308Santhony.gutierrez@amd.combool 37811308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvTimingResp(PacketPtr pkt) 37911308Santhony.gutierrez@amd.com{ 38011308Santhony.gutierrez@amd.com // a translation completed and returned 38111308Santhony.gutierrez@amd.com coalescer->updatePhysAddresses(pkt); 38211308Santhony.gutierrez@amd.com 38311308Santhony.gutierrez@amd.com return true; 38411308Santhony.gutierrez@amd.com} 38511308Santhony.gutierrez@amd.com 38611308Santhony.gutierrez@amd.comvoid 38711308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvReqRetry() 38811308Santhony.gutierrez@amd.com{ 38911308Santhony.gutierrez@amd.com //we've receeived a retry. Schedule a probeTLBEvent 39011308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) 39111308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 39211308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 39311308Santhony.gutierrez@amd.com} 39411308Santhony.gutierrez@amd.com 39511308Santhony.gutierrez@amd.comvoid 39611308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvFunctional(PacketPtr pkt) 39711308Santhony.gutierrez@amd.com{ 39811308Santhony.gutierrez@amd.com fatal("Memory side recvFunctional() not implemented in TLB coalescer.\n"); 39911308Santhony.gutierrez@amd.com} 40011308Santhony.gutierrez@amd.com 40111308Santhony.gutierrez@amd.com/* 40211308Santhony.gutierrez@amd.com * Here we scan the coalescer FIFO and issue the max 40311308Santhony.gutierrez@amd.com * number of permitted probes to the TLB below. We 40411308Santhony.gutierrez@amd.com * permit bypassing of coalesced requests for the same 40511308Santhony.gutierrez@amd.com * tick_index. 40611308Santhony.gutierrez@amd.com * 40711308Santhony.gutierrez@amd.com * We do not access the next tick_index unless we've 40811308Santhony.gutierrez@amd.com * drained the previous one. The coalesced requests 40911308Santhony.gutierrez@amd.com * that are successfully sent are moved to the 41011308Santhony.gutierrez@amd.com * issuedTranslationsTable table (the table which keeps 41111308Santhony.gutierrez@amd.com * track of the outstanding reqs) 41211308Santhony.gutierrez@amd.com */ 41311308Santhony.gutierrez@amd.comvoid 41412126Sspwilson2@wisc.eduTLBCoalescer::processProbeTLBEvent() 41511308Santhony.gutierrez@amd.com{ 41611308Santhony.gutierrez@amd.com // number of TLB probes sent so far 41711308Santhony.gutierrez@amd.com int sent_probes = 0; 41811308Santhony.gutierrez@amd.com // rejected denotes a blocking event 41911308Santhony.gutierrez@amd.com bool rejected = false; 42011308Santhony.gutierrez@amd.com 42111308Santhony.gutierrez@amd.com // It is set to true either when the recvTiming of the TLB below 42211308Santhony.gutierrez@amd.com // returns false or when there is another outstanding request for the 42311308Santhony.gutierrez@amd.com // same virt. page. 42411308Santhony.gutierrez@amd.com 42512126Sspwilson2@wisc.edu DPRINTF(GPUTLB, "triggered TLBCoalescer %s\n", __func__); 42611308Santhony.gutierrez@amd.com 42712126Sspwilson2@wisc.edu for (auto iter = coalescerFIFO.begin(); 42812126Sspwilson2@wisc.edu iter != coalescerFIFO.end() && !rejected; ) { 42911308Santhony.gutierrez@amd.com int coalescedReq_cnt = iter->second.size(); 43011308Santhony.gutierrez@amd.com int i = 0; 43111308Santhony.gutierrez@amd.com int vector_index = 0; 43211308Santhony.gutierrez@amd.com 43311308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescedReq_cnt is %d for tick_index %d\n", 43411308Santhony.gutierrez@amd.com coalescedReq_cnt, iter->first); 43511308Santhony.gutierrez@amd.com 43611308Santhony.gutierrez@amd.com while (i < coalescedReq_cnt) { 43711308Santhony.gutierrez@amd.com ++i; 43811308Santhony.gutierrez@amd.com PacketPtr first_packet = iter->second[vector_index][0]; 43911308Santhony.gutierrez@amd.com 44011308Santhony.gutierrez@amd.com // compute virtual page address for this request 44111308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(first_packet->req->getVaddr(), 44211308Santhony.gutierrez@amd.com TheISA::PageBytes); 44311308Santhony.gutierrez@amd.com 44411308Santhony.gutierrez@amd.com // is there another outstanding request for the same page addr? 44511308Santhony.gutierrez@amd.com int pending_reqs = 44612126Sspwilson2@wisc.edu issuedTranslationsTable.count(virt_page_addr); 44711308Santhony.gutierrez@amd.com 44811308Santhony.gutierrez@amd.com if (pending_reqs) { 44911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cannot issue - There are pending reqs for " 45011308Santhony.gutierrez@amd.com "page %#x\n", virt_page_addr); 45111308Santhony.gutierrez@amd.com 45211308Santhony.gutierrez@amd.com ++vector_index; 45311308Santhony.gutierrez@amd.com rejected = true; 45411308Santhony.gutierrez@amd.com 45511308Santhony.gutierrez@amd.com continue; 45611308Santhony.gutierrez@amd.com } 45711308Santhony.gutierrez@amd.com 45811308Santhony.gutierrez@amd.com // send the coalesced request for virt_page_addr 45912126Sspwilson2@wisc.edu if (!memSidePort[0]->sendTimingReq(first_packet)) { 46011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Failed to send TLB request for page %#x", 46111308Santhony.gutierrez@amd.com virt_page_addr); 46211308Santhony.gutierrez@amd.com 46311308Santhony.gutierrez@amd.com // No need for a retries queue since we are already buffering 46411308Santhony.gutierrez@amd.com // the coalesced request in coalescerFIFO. 46511308Santhony.gutierrez@amd.com rejected = true; 46611308Santhony.gutierrez@amd.com ++vector_index; 46711308Santhony.gutierrez@amd.com } else { 46811308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *tmp_sender_state = 46911308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*> 47011308Santhony.gutierrez@amd.com (first_packet->senderState); 47111308Santhony.gutierrez@amd.com 47211308Santhony.gutierrez@amd.com bool update_stats = !tmp_sender_state->prefetch; 47311308Santhony.gutierrez@amd.com 47411308Santhony.gutierrez@amd.com if (update_stats) { 47511308Santhony.gutierrez@amd.com // req_cnt is total number of packets represented 47611308Santhony.gutierrez@amd.com // by the one we just sent counting all the way from 47711308Santhony.gutierrez@amd.com // the top of TLB hiearchy (i.e., from the CU) 47811308Santhony.gutierrez@amd.com int req_cnt = tmp_sender_state->reqCnt.back(); 47912126Sspwilson2@wisc.edu queuingCycles += (curTick() * req_cnt); 48011308Santhony.gutierrez@amd.com 48111308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "%s sending pkt w/ req_cnt %d\n", 48212126Sspwilson2@wisc.edu name(), req_cnt); 48311308Santhony.gutierrez@amd.com 48411308Santhony.gutierrez@amd.com // pkt_cnt is number of packets we coalesced into the one 48511308Santhony.gutierrez@amd.com // we just sent but only at this coalescer level 48611308Santhony.gutierrez@amd.com int pkt_cnt = iter->second[vector_index].size(); 48712126Sspwilson2@wisc.edu localqueuingCycles += (curTick() * pkt_cnt); 48811308Santhony.gutierrez@amd.com } 48911308Santhony.gutierrez@amd.com 49011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Successfully sent TLB request for page %#x", 49111308Santhony.gutierrez@amd.com virt_page_addr); 49211308Santhony.gutierrez@amd.com 49311308Santhony.gutierrez@amd.com //copy coalescedReq to issuedTranslationsTable 49412126Sspwilson2@wisc.edu issuedTranslationsTable[virt_page_addr] 49511308Santhony.gutierrez@amd.com = iter->second[vector_index]; 49611308Santhony.gutierrez@amd.com 49711308Santhony.gutierrez@amd.com //erase the entry of this coalesced req 49811308Santhony.gutierrez@amd.com iter->second.erase(iter->second.begin() + vector_index); 49911308Santhony.gutierrez@amd.com 50011308Santhony.gutierrez@amd.com if (iter->second.empty()) 50111308Santhony.gutierrez@amd.com assert(i == coalescedReq_cnt); 50211308Santhony.gutierrez@amd.com 50311308Santhony.gutierrez@amd.com sent_probes++; 50412126Sspwilson2@wisc.edu if (sent_probes == TLBProbesPerCycle) 50511308Santhony.gutierrez@amd.com return; 50611308Santhony.gutierrez@amd.com } 50711308Santhony.gutierrez@amd.com } 50811308Santhony.gutierrez@amd.com 50911308Santhony.gutierrez@amd.com //if there are no more coalesced reqs for this tick_index 51011308Santhony.gutierrez@amd.com //erase the hash_map with the first iterator 51111308Santhony.gutierrez@amd.com if (iter->second.empty()) { 51212126Sspwilson2@wisc.edu coalescerFIFO.erase(iter++); 51311308Santhony.gutierrez@amd.com } else { 51411308Santhony.gutierrez@amd.com ++iter; 51511308Santhony.gutierrez@amd.com } 51611308Santhony.gutierrez@amd.com } 51711308Santhony.gutierrez@amd.com} 51811308Santhony.gutierrez@amd.com 51912126Sspwilson2@wisc.eduvoid 52012126Sspwilson2@wisc.eduTLBCoalescer::processCleanupEvent() 52111308Santhony.gutierrez@amd.com{ 52212126Sspwilson2@wisc.edu while (!cleanupQueue.empty()) { 52312126Sspwilson2@wisc.edu Addr cleanup_addr = cleanupQueue.front(); 52412126Sspwilson2@wisc.edu cleanupQueue.pop(); 52512126Sspwilson2@wisc.edu issuedTranslationsTable.erase(cleanup_addr); 52611308Santhony.gutierrez@amd.com 52711308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cleanup - Delete coalescer entry with key %#x\n", 52811308Santhony.gutierrez@amd.com cleanup_addr); 52911308Santhony.gutierrez@amd.com } 53011308Santhony.gutierrez@amd.com} 53111308Santhony.gutierrez@amd.com 53211308Santhony.gutierrez@amd.comvoid 53311308Santhony.gutierrez@amd.comTLBCoalescer::regStats() 53411308Santhony.gutierrez@amd.com{ 53511523Sdavid.guillen@arm.com MemObject::regStats(); 53611523Sdavid.guillen@arm.com 53711308Santhony.gutierrez@amd.com uncoalescedAccesses 53811308Santhony.gutierrez@amd.com .name(name() + ".uncoalesced_accesses") 53911308Santhony.gutierrez@amd.com .desc("Number of uncoalesced TLB accesses") 54011308Santhony.gutierrez@amd.com ; 54111308Santhony.gutierrez@amd.com 54211308Santhony.gutierrez@amd.com coalescedAccesses 54311308Santhony.gutierrez@amd.com .name(name() + ".coalesced_accesses") 54411308Santhony.gutierrez@amd.com .desc("Number of coalesced TLB accesses") 54511308Santhony.gutierrez@amd.com ; 54611308Santhony.gutierrez@amd.com 54711308Santhony.gutierrez@amd.com queuingCycles 54811308Santhony.gutierrez@amd.com .name(name() + ".queuing_cycles") 54911308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue") 55011308Santhony.gutierrez@amd.com ; 55111308Santhony.gutierrez@amd.com 55211308Santhony.gutierrez@amd.com localqueuingCycles 55311308Santhony.gutierrez@amd.com .name(name() + ".local_queuing_cycles") 55411308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue for all incoming reqs") 55511308Santhony.gutierrez@amd.com ; 55611308Santhony.gutierrez@amd.com 55711308Santhony.gutierrez@amd.com localLatency 55811308Santhony.gutierrez@amd.com .name(name() + ".local_latency") 55911308Santhony.gutierrez@amd.com .desc("Avg. latency over all incoming pkts") 56011308Santhony.gutierrez@amd.com ; 56111308Santhony.gutierrez@amd.com 56211308Santhony.gutierrez@amd.com localLatency = localqueuingCycles / uncoalescedAccesses; 56311308Santhony.gutierrez@amd.com} 56411308Santhony.gutierrez@amd.com 56511308Santhony.gutierrez@amd.com 56611308Santhony.gutierrez@amd.comTLBCoalescer* 56711308Santhony.gutierrez@amd.comTLBCoalescerParams::create() 56811308Santhony.gutierrez@amd.com{ 56911308Santhony.gutierrez@amd.com return new TLBCoalescer(this); 57011308Santhony.gutierrez@amd.com} 57111308Santhony.gutierrez@amd.com 572