tlb_coalescer.cc revision 12697
111308Santhony.gutierrez@amd.com/* 211308Santhony.gutierrez@amd.com * Copyright (c) 2011-2015 Advanced Micro Devices, Inc. 311308Santhony.gutierrez@amd.com * All rights reserved. 411308Santhony.gutierrez@amd.com * 511308Santhony.gutierrez@amd.com * For use for simulation and test purposes only 611308Santhony.gutierrez@amd.com * 711308Santhony.gutierrez@amd.com * Redistribution and use in source and binary forms, with or without 811308Santhony.gutierrez@amd.com * modification, are permitted provided that the following conditions are met: 911308Santhony.gutierrez@amd.com * 1011308Santhony.gutierrez@amd.com * 1. Redistributions of source code must retain the above copyright notice, 1111308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer. 1211308Santhony.gutierrez@amd.com * 1311308Santhony.gutierrez@amd.com * 2. Redistributions in binary form must reproduce the above copyright notice, 1411308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer in the documentation 1511308Santhony.gutierrez@amd.com * and/or other materials provided with the distribution. 1611308Santhony.gutierrez@amd.com * 1712697Santhony.gutierrez@amd.com * 3. Neither the name of the copyright holder nor the names of its 1812697Santhony.gutierrez@amd.com * contributors may be used to endorse or promote products derived from this 1912697Santhony.gutierrez@amd.com * software without specific prior written permission. 2011308Santhony.gutierrez@amd.com * 2111308Santhony.gutierrez@amd.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2211308Santhony.gutierrez@amd.com * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2311308Santhony.gutierrez@amd.com * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2411308Santhony.gutierrez@amd.com * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 2511308Santhony.gutierrez@amd.com * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2611308Santhony.gutierrez@amd.com * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2711308Santhony.gutierrez@amd.com * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2811308Santhony.gutierrez@amd.com * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2911308Santhony.gutierrez@amd.com * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 3011308Santhony.gutierrez@amd.com * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3111308Santhony.gutierrez@amd.com * POSSIBILITY OF SUCH DAMAGE. 3211308Santhony.gutierrez@amd.com * 3312697Santhony.gutierrez@amd.com * Authors: Lisa Hsu 3411308Santhony.gutierrez@amd.com */ 3511308Santhony.gutierrez@amd.com 3611308Santhony.gutierrez@amd.com#include "gpu-compute/tlb_coalescer.hh" 3711308Santhony.gutierrez@amd.com 3811308Santhony.gutierrez@amd.com#include <cstring> 3911308Santhony.gutierrez@amd.com 4011308Santhony.gutierrez@amd.com#include "debug/GPUTLB.hh" 4111308Santhony.gutierrez@amd.com 4212126Sspwilson2@wisc.eduTLBCoalescer::TLBCoalescer(const Params *p) 4312126Sspwilson2@wisc.edu : MemObject(p), 4412126Sspwilson2@wisc.edu clock(p->clk_domain->clockPeriod()), 4512126Sspwilson2@wisc.edu TLBProbesPerCycle(p->probesPerCycle), 4612126Sspwilson2@wisc.edu coalescingWindow(p->coalescingWindow), 4712126Sspwilson2@wisc.edu disableCoalescing(p->disableCoalescing), 4812126Sspwilson2@wisc.edu probeTLBEvent([this]{ processProbeTLBEvent(); }, 4912126Sspwilson2@wisc.edu "Probe the TLB below", 5012126Sspwilson2@wisc.edu false, Event::CPU_Tick_Pri), 5112126Sspwilson2@wisc.edu cleanupEvent([this]{ processCleanupEvent(); }, 5212126Sspwilson2@wisc.edu "Cleanup issuedTranslationsTable hashmap", 5312126Sspwilson2@wisc.edu false, Event::Maximum_Pri) 5411308Santhony.gutierrez@amd.com{ 5511308Santhony.gutierrez@amd.com // create the slave ports based on the number of connected ports 5611308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_slave_connection_count; ++i) { 5711308Santhony.gutierrez@amd.com cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d", name(), i), 5811308Santhony.gutierrez@amd.com this, i)); 5911308Santhony.gutierrez@amd.com } 6011308Santhony.gutierrez@amd.com 6111308Santhony.gutierrez@amd.com // create the master ports based on the number of connected ports 6211308Santhony.gutierrez@amd.com for (size_t i = 0; i < p->port_master_connection_count; ++i) { 6311308Santhony.gutierrez@amd.com memSidePort.push_back(new MemSidePort(csprintf("%s-port%d", name(), i), 6411308Santhony.gutierrez@amd.com this, i)); 6511308Santhony.gutierrez@amd.com } 6611308Santhony.gutierrez@amd.com} 6711308Santhony.gutierrez@amd.com 6811308Santhony.gutierrez@amd.comBaseSlavePort& 6911308Santhony.gutierrez@amd.comTLBCoalescer::getSlavePort(const std::string &if_name, PortID idx) 7011308Santhony.gutierrez@amd.com{ 7111308Santhony.gutierrez@amd.com if (if_name == "slave") { 7211308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(cpuSidePort.size())) { 7311308Santhony.gutierrez@amd.com panic("TLBCoalescer::getSlavePort: unknown index %d\n", idx); 7411308Santhony.gutierrez@amd.com } 7511308Santhony.gutierrez@amd.com 7611308Santhony.gutierrez@amd.com return *cpuSidePort[idx]; 7711308Santhony.gutierrez@amd.com } else { 7811308Santhony.gutierrez@amd.com panic("TLBCoalescer::getSlavePort: unknown port %s\n", if_name); 7911308Santhony.gutierrez@amd.com } 8011308Santhony.gutierrez@amd.com} 8111308Santhony.gutierrez@amd.com 8211308Santhony.gutierrez@amd.comBaseMasterPort& 8311308Santhony.gutierrez@amd.comTLBCoalescer::getMasterPort(const std::string &if_name, PortID idx) 8411308Santhony.gutierrez@amd.com{ 8511308Santhony.gutierrez@amd.com if (if_name == "master") { 8611308Santhony.gutierrez@amd.com if (idx >= static_cast<PortID>(memSidePort.size())) { 8711308Santhony.gutierrez@amd.com panic("TLBCoalescer::getMasterPort: unknown index %d\n", idx); 8811308Santhony.gutierrez@amd.com } 8911308Santhony.gutierrez@amd.com 9011308Santhony.gutierrez@amd.com return *memSidePort[idx]; 9111308Santhony.gutierrez@amd.com } else { 9211308Santhony.gutierrez@amd.com panic("TLBCoalescer::getMasterPort: unknown port %s\n", if_name); 9311308Santhony.gutierrez@amd.com } 9411308Santhony.gutierrez@amd.com} 9511308Santhony.gutierrez@amd.com 9611308Santhony.gutierrez@amd.com/* 9711308Santhony.gutierrez@amd.com * This method returns true if the <incoming_pkt> 9811308Santhony.gutierrez@amd.com * can be coalesced with <coalesced_pkt> and false otherwise. 9911308Santhony.gutierrez@amd.com * A given set of rules is checked. 10011308Santhony.gutierrez@amd.com * The rules can potentially be modified based on the TLB level. 10111308Santhony.gutierrez@amd.com */ 10211308Santhony.gutierrez@amd.combool 10311308Santhony.gutierrez@amd.comTLBCoalescer::canCoalesce(PacketPtr incoming_pkt, PacketPtr coalesced_pkt) 10411308Santhony.gutierrez@amd.com{ 10511308Santhony.gutierrez@amd.com if (disableCoalescing) 10611308Santhony.gutierrez@amd.com return false; 10711308Santhony.gutierrez@amd.com 10811308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *incoming_state = 10911308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(incoming_pkt->senderState); 11011308Santhony.gutierrez@amd.com 11111308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *coalesced_state = 11211308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(coalesced_pkt->senderState); 11311308Santhony.gutierrez@amd.com 11411308Santhony.gutierrez@amd.com // Rule 1: Coalesce requests only if they 11511308Santhony.gutierrez@amd.com // fall within the same virtual page 11611308Santhony.gutierrez@amd.com Addr incoming_virt_page_addr = roundDown(incoming_pkt->req->getVaddr(), 11711308Santhony.gutierrez@amd.com TheISA::PageBytes); 11811308Santhony.gutierrez@amd.com 11911308Santhony.gutierrez@amd.com Addr coalesced_virt_page_addr = roundDown(coalesced_pkt->req->getVaddr(), 12011308Santhony.gutierrez@amd.com TheISA::PageBytes); 12111308Santhony.gutierrez@amd.com 12211308Santhony.gutierrez@amd.com if (incoming_virt_page_addr != coalesced_virt_page_addr) 12311308Santhony.gutierrez@amd.com return false; 12411308Santhony.gutierrez@amd.com 12511308Santhony.gutierrez@amd.com //* Rule 2: Coalesce requests only if they 12611308Santhony.gutierrez@amd.com // share a TLB Mode, i.e. they are both read 12711308Santhony.gutierrez@amd.com // or write requests. 12811308Santhony.gutierrez@amd.com BaseTLB::Mode incoming_mode = incoming_state->tlbMode; 12911308Santhony.gutierrez@amd.com BaseTLB::Mode coalesced_mode = coalesced_state->tlbMode; 13011308Santhony.gutierrez@amd.com 13111308Santhony.gutierrez@amd.com if (incoming_mode != coalesced_mode) 13211308Santhony.gutierrez@amd.com return false; 13311308Santhony.gutierrez@amd.com 13411308Santhony.gutierrez@amd.com // when we can coalesce a packet update the reqCnt 13511308Santhony.gutierrez@amd.com // that is the number of packets represented by 13611308Santhony.gutierrez@amd.com // this coalesced packet 13711308Santhony.gutierrez@amd.com if (!incoming_state->prefetch) 13811308Santhony.gutierrez@amd.com coalesced_state->reqCnt.back() += incoming_state->reqCnt.back(); 13911308Santhony.gutierrez@amd.com 14011308Santhony.gutierrez@amd.com return true; 14111308Santhony.gutierrez@amd.com} 14211308Santhony.gutierrez@amd.com 14311308Santhony.gutierrez@amd.com/* 14411308Santhony.gutierrez@amd.com * We need to update the physical addresses of all the translation requests 14511308Santhony.gutierrez@amd.com * that were coalesced into the one that just returned. 14611308Santhony.gutierrez@amd.com */ 14711308Santhony.gutierrez@amd.comvoid 14811308Santhony.gutierrez@amd.comTLBCoalescer::updatePhysAddresses(PacketPtr pkt) 14911308Santhony.gutierrez@amd.com{ 15011308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 15111308Santhony.gutierrez@amd.com 15211308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Update phys. addr. for %d coalesced reqs for page %#x\n", 15311308Santhony.gutierrez@amd.com issuedTranslationsTable[virt_page_addr].size(), virt_page_addr); 15411308Santhony.gutierrez@amd.com 15511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 15611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 15711308Santhony.gutierrez@amd.com 15811308Santhony.gutierrez@amd.com TheISA::GpuTlbEntry *tlb_entry = sender_state->tlbEntry; 15911308Santhony.gutierrez@amd.com assert(tlb_entry); 16011308Santhony.gutierrez@amd.com Addr first_entry_vaddr = tlb_entry->vaddr; 16111308Santhony.gutierrez@amd.com Addr first_entry_paddr = tlb_entry->paddr; 16211308Santhony.gutierrez@amd.com int page_size = tlb_entry->size(); 16311308Santhony.gutierrez@amd.com bool uncacheable = tlb_entry->uncacheable; 16411308Santhony.gutierrez@amd.com int first_hit_level = sender_state->hitLevel; 16511308Santhony.gutierrez@amd.com bool valid = tlb_entry->valid; 16611308Santhony.gutierrez@amd.com 16711308Santhony.gutierrez@amd.com // Get the physical page address of the translated request 16811308Santhony.gutierrez@amd.com // Using the page_size specified in the TLBEntry allows us 16911308Santhony.gutierrez@amd.com // to support different page sizes. 17011308Santhony.gutierrez@amd.com Addr phys_page_paddr = pkt->req->getPaddr(); 17111308Santhony.gutierrez@amd.com phys_page_paddr &= ~(page_size - 1); 17211308Santhony.gutierrez@amd.com 17311308Santhony.gutierrez@amd.com for (int i = 0; i < issuedTranslationsTable[virt_page_addr].size(); ++i) { 17411308Santhony.gutierrez@amd.com PacketPtr local_pkt = issuedTranslationsTable[virt_page_addr][i]; 17511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 17611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>( 17711308Santhony.gutierrez@amd.com local_pkt->senderState); 17811308Santhony.gutierrez@amd.com 17911308Santhony.gutierrez@amd.com // we are sending the packet back, so pop the reqCnt associated 18011308Santhony.gutierrez@amd.com // with this level in the TLB hiearchy 18111308Santhony.gutierrez@amd.com if (!sender_state->prefetch) 18211308Santhony.gutierrez@amd.com sender_state->reqCnt.pop_back(); 18311308Santhony.gutierrez@amd.com 18411308Santhony.gutierrez@amd.com /* 18511308Santhony.gutierrez@amd.com * Only the first packet from this coalesced request has been 18611308Santhony.gutierrez@amd.com * translated. Grab the translated phys. page addr and update the 18711308Santhony.gutierrez@amd.com * physical addresses of the remaining packets with the appropriate 18811308Santhony.gutierrez@amd.com * page offsets. 18911308Santhony.gutierrez@amd.com */ 19011308Santhony.gutierrez@amd.com if (i) { 19111308Santhony.gutierrez@amd.com Addr paddr = phys_page_paddr; 19211308Santhony.gutierrez@amd.com paddr |= (local_pkt->req->getVaddr() & (page_size - 1)); 19311308Santhony.gutierrez@amd.com local_pkt->req->setPaddr(paddr); 19411308Santhony.gutierrez@amd.com 19511308Santhony.gutierrez@amd.com if (uncacheable) 19611308Santhony.gutierrez@amd.com local_pkt->req->setFlags(Request::UNCACHEABLE); 19711308Santhony.gutierrez@amd.com 19811308Santhony.gutierrez@amd.com // update senderState->tlbEntry, so we can insert 19911308Santhony.gutierrez@amd.com // the correct TLBEentry in the TLBs above. 20011308Santhony.gutierrez@amd.com sender_state->tlbEntry = 20111308Santhony.gutierrez@amd.com new TheISA::GpuTlbEntry(0, first_entry_vaddr, first_entry_paddr, 20211308Santhony.gutierrez@amd.com valid); 20311308Santhony.gutierrez@amd.com 20411308Santhony.gutierrez@amd.com // update the hitLevel for all uncoalesced reqs 20511308Santhony.gutierrez@amd.com // so that each packet knows where it hit 20611308Santhony.gutierrez@amd.com // (used for statistics in the CUs) 20711308Santhony.gutierrez@amd.com sender_state->hitLevel = first_hit_level; 20811308Santhony.gutierrez@amd.com } 20911308Santhony.gutierrez@amd.com 21011308Santhony.gutierrez@amd.com SlavePort *return_port = sender_state->ports.back(); 21111308Santhony.gutierrez@amd.com sender_state->ports.pop_back(); 21211308Santhony.gutierrez@amd.com 21311308Santhony.gutierrez@amd.com // Translation is done - Convert to a response pkt if necessary and 21411308Santhony.gutierrez@amd.com // send the translation back 21511308Santhony.gutierrez@amd.com if (local_pkt->isRequest()) { 21611308Santhony.gutierrez@amd.com local_pkt->makeTimingResponse(); 21711308Santhony.gutierrez@amd.com } 21811308Santhony.gutierrez@amd.com 21911308Santhony.gutierrez@amd.com return_port->sendTimingResp(local_pkt); 22011308Santhony.gutierrez@amd.com } 22111308Santhony.gutierrez@amd.com 22211308Santhony.gutierrez@amd.com // schedule clean up for end of this cycle 22311308Santhony.gutierrez@amd.com // This is a maximum priority event and must be on 22411308Santhony.gutierrez@amd.com // the same cycle as GPUTLB cleanup event to prevent 22511308Santhony.gutierrez@amd.com // race conditions with an IssueProbeEvent caused by 22611308Santhony.gutierrez@amd.com // MemSidePort::recvReqRetry 22711308Santhony.gutierrez@amd.com cleanupQueue.push(virt_page_addr); 22811308Santhony.gutierrez@amd.com 22911308Santhony.gutierrez@amd.com if (!cleanupEvent.scheduled()) 23011308Santhony.gutierrez@amd.com schedule(cleanupEvent, curTick()); 23111308Santhony.gutierrez@amd.com} 23211308Santhony.gutierrez@amd.com 23311308Santhony.gutierrez@amd.com// Receive translation requests, create a coalesced request, 23411308Santhony.gutierrez@amd.com// and send them to the TLB (TLBProbesPerCycle) 23511308Santhony.gutierrez@amd.combool 23611308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvTimingReq(PacketPtr pkt) 23711308Santhony.gutierrez@amd.com{ 23811308Santhony.gutierrez@amd.com // first packet of a coalesced request 23911308Santhony.gutierrez@amd.com PacketPtr first_packet = nullptr; 24011308Santhony.gutierrez@amd.com // true if we are able to do coalescing 24111308Santhony.gutierrez@amd.com bool didCoalesce = false; 24211308Santhony.gutierrez@amd.com // number of coalesced reqs for a given window 24311308Santhony.gutierrez@amd.com int coalescedReq_cnt = 0; 24411308Santhony.gutierrez@amd.com 24511308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 24611308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 24711308Santhony.gutierrez@amd.com 24811308Santhony.gutierrez@amd.com // push back the port to remember the path back 24911308Santhony.gutierrez@amd.com sender_state->ports.push_back(this); 25011308Santhony.gutierrez@amd.com 25111308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 25211308Santhony.gutierrez@amd.com 25311308Santhony.gutierrez@amd.com if (update_stats) { 25411308Santhony.gutierrez@amd.com // if reqCnt is empty then this packet does not represent 25511308Santhony.gutierrez@amd.com // multiple uncoalesced reqs(pkts) but just a single pkt. 25611308Santhony.gutierrez@amd.com // If it does though then the reqCnt for each level in the 25711308Santhony.gutierrez@amd.com // hierarchy accumulates the total number of reqs this packet 25811308Santhony.gutierrez@amd.com // represents 25911308Santhony.gutierrez@amd.com int req_cnt = 1; 26011308Santhony.gutierrez@amd.com 26111308Santhony.gutierrez@amd.com if (!sender_state->reqCnt.empty()) 26211308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 26311308Santhony.gutierrez@amd.com 26411308Santhony.gutierrez@amd.com sender_state->reqCnt.push_back(req_cnt); 26511308Santhony.gutierrez@amd.com 26611308Santhony.gutierrez@amd.com // update statistics 26711308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 26811308Santhony.gutierrez@amd.com req_cnt = sender_state->reqCnt.back(); 26911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "receiving pkt w/ req_cnt %d\n", req_cnt); 27011308Santhony.gutierrez@amd.com coalescer->queuingCycles -= (curTick() * req_cnt); 27111308Santhony.gutierrez@amd.com coalescer->localqueuingCycles -= curTick(); 27211308Santhony.gutierrez@amd.com } 27311308Santhony.gutierrez@amd.com 27411308Santhony.gutierrez@amd.com // FIXME if you want to coalesce not based on the issueTime 27511308Santhony.gutierrez@amd.com // of the packets (i.e., from the compute unit's perspective) 27611308Santhony.gutierrez@amd.com // but based on when they reached this coalescer then 27711308Santhony.gutierrez@amd.com // remove the following if statement and use curTick() or 27811308Santhony.gutierrez@amd.com // coalescingWindow for the tick_index. 27911308Santhony.gutierrez@amd.com if (!sender_state->issueTime) 28011308Santhony.gutierrez@amd.com sender_state->issueTime = curTick(); 28111308Santhony.gutierrez@amd.com 28211308Santhony.gutierrez@amd.com // The tick index is used as a key to the coalescerFIFO hashmap. 28311308Santhony.gutierrez@amd.com // It is shared by all candidates that fall within the 28411308Santhony.gutierrez@amd.com // given coalescingWindow. 28511308Santhony.gutierrez@amd.com int64_t tick_index = sender_state->issueTime / coalescer->coalescingWindow; 28611308Santhony.gutierrez@amd.com 28711308Santhony.gutierrez@amd.com if (coalescer->coalescerFIFO.count(tick_index)) { 28811308Santhony.gutierrez@amd.com coalescedReq_cnt = coalescer->coalescerFIFO[tick_index].size(); 28911308Santhony.gutierrez@amd.com } 29011308Santhony.gutierrez@amd.com 29111308Santhony.gutierrez@amd.com // see if we can coalesce the incoming pkt with another 29211308Santhony.gutierrez@amd.com // coalesced request with the same tick_index 29311308Santhony.gutierrez@amd.com for (int i = 0; i < coalescedReq_cnt; ++i) { 29411308Santhony.gutierrez@amd.com first_packet = coalescer->coalescerFIFO[tick_index][i][0]; 29511308Santhony.gutierrez@amd.com 29611308Santhony.gutierrez@amd.com if (coalescer->canCoalesce(pkt, first_packet)) { 29711308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].push_back(pkt); 29811308Santhony.gutierrez@amd.com 29911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Coalesced req %i w/ tick_index %d has %d reqs\n", 30011308Santhony.gutierrez@amd.com i, tick_index, 30111308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index][i].size()); 30211308Santhony.gutierrez@amd.com 30311308Santhony.gutierrez@amd.com didCoalesce = true; 30411308Santhony.gutierrez@amd.com break; 30511308Santhony.gutierrez@amd.com } 30611308Santhony.gutierrez@amd.com } 30711308Santhony.gutierrez@amd.com 30811308Santhony.gutierrez@amd.com // if this is the first request for this tick_index 30911308Santhony.gutierrez@amd.com // or we did not manage to coalesce, update stats 31011308Santhony.gutierrez@amd.com // and make necessary allocations. 31111308Santhony.gutierrez@amd.com if (!coalescedReq_cnt || !didCoalesce) { 31211308Santhony.gutierrez@amd.com if (update_stats) 31311308Santhony.gutierrez@amd.com coalescer->coalescedAccesses++; 31411308Santhony.gutierrez@amd.com 31511308Santhony.gutierrez@amd.com std::vector<PacketPtr> new_array; 31611308Santhony.gutierrez@amd.com new_array.push_back(pkt); 31711308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].push_back(new_array); 31811308Santhony.gutierrez@amd.com 31911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescerFIFO[%d] now has %d coalesced reqs after " 32011308Santhony.gutierrez@amd.com "push\n", tick_index, 32111308Santhony.gutierrez@amd.com coalescer->coalescerFIFO[tick_index].size()); 32211308Santhony.gutierrez@amd.com } 32311308Santhony.gutierrez@amd.com 32411308Santhony.gutierrez@amd.com //schedule probeTLBEvent next cycle to send the 32511308Santhony.gutierrez@amd.com //coalesced requests to the TLB 32611308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) { 32711308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 32811308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 32911308Santhony.gutierrez@amd.com } 33011308Santhony.gutierrez@amd.com 33111308Santhony.gutierrez@amd.com return true; 33211308Santhony.gutierrez@amd.com} 33311308Santhony.gutierrez@amd.com 33411308Santhony.gutierrez@amd.comvoid 33511308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvReqRetry() 33611308Santhony.gutierrez@amd.com{ 33711308Santhony.gutierrez@amd.com assert(false); 33811308Santhony.gutierrez@amd.com} 33911308Santhony.gutierrez@amd.com 34011308Santhony.gutierrez@amd.comvoid 34111308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvFunctional(PacketPtr pkt) 34211308Santhony.gutierrez@amd.com{ 34311308Santhony.gutierrez@amd.com 34411308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *sender_state = 34511308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState); 34611308Santhony.gutierrez@amd.com 34711308Santhony.gutierrez@amd.com bool update_stats = !sender_state->prefetch; 34811308Santhony.gutierrez@amd.com 34911308Santhony.gutierrez@amd.com if (update_stats) 35011308Santhony.gutierrez@amd.com coalescer->uncoalescedAccesses++; 35111308Santhony.gutierrez@amd.com 35211308Santhony.gutierrez@amd.com // If there is a pending timing request for this virtual address 35311308Santhony.gutierrez@amd.com // print a warning message. This is a temporary caveat of 35411308Santhony.gutierrez@amd.com // the current simulator where atomic and timing requests can 35511308Santhony.gutierrez@amd.com // coexist. FIXME remove this check/warning in the future. 35611308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes); 35711308Santhony.gutierrez@amd.com int map_count = coalescer->issuedTranslationsTable.count(virt_page_addr); 35811308Santhony.gutierrez@amd.com 35911308Santhony.gutierrez@amd.com if (map_count) { 36011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Warning! Functional access to addr %#x sees timing " 36111308Santhony.gutierrez@amd.com "req. pending\n", virt_page_addr); 36211308Santhony.gutierrez@amd.com } 36311308Santhony.gutierrez@amd.com 36411308Santhony.gutierrez@amd.com coalescer->memSidePort[0]->sendFunctional(pkt); 36511308Santhony.gutierrez@amd.com} 36611308Santhony.gutierrez@amd.com 36711308Santhony.gutierrez@amd.comAddrRangeList 36811308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::getAddrRanges() const 36911308Santhony.gutierrez@amd.com{ 37011308Santhony.gutierrez@amd.com // currently not checked by the master 37111308Santhony.gutierrez@amd.com AddrRangeList ranges; 37211308Santhony.gutierrez@amd.com 37311308Santhony.gutierrez@amd.com return ranges; 37411308Santhony.gutierrez@amd.com} 37511308Santhony.gutierrez@amd.com 37611308Santhony.gutierrez@amd.combool 37711308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvTimingResp(PacketPtr pkt) 37811308Santhony.gutierrez@amd.com{ 37911308Santhony.gutierrez@amd.com // a translation completed and returned 38011308Santhony.gutierrez@amd.com coalescer->updatePhysAddresses(pkt); 38111308Santhony.gutierrez@amd.com 38211308Santhony.gutierrez@amd.com return true; 38311308Santhony.gutierrez@amd.com} 38411308Santhony.gutierrez@amd.com 38511308Santhony.gutierrez@amd.comvoid 38611308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvReqRetry() 38711308Santhony.gutierrez@amd.com{ 38811308Santhony.gutierrez@amd.com //we've receeived a retry. Schedule a probeTLBEvent 38911308Santhony.gutierrez@amd.com if (!coalescer->probeTLBEvent.scheduled()) 39011308Santhony.gutierrez@amd.com coalescer->schedule(coalescer->probeTLBEvent, 39111308Santhony.gutierrez@amd.com curTick() + coalescer->ticks(1)); 39211308Santhony.gutierrez@amd.com} 39311308Santhony.gutierrez@amd.com 39411308Santhony.gutierrez@amd.comvoid 39511308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvFunctional(PacketPtr pkt) 39611308Santhony.gutierrez@amd.com{ 39711308Santhony.gutierrez@amd.com fatal("Memory side recvFunctional() not implemented in TLB coalescer.\n"); 39811308Santhony.gutierrez@amd.com} 39911308Santhony.gutierrez@amd.com 40011308Santhony.gutierrez@amd.com/* 40111308Santhony.gutierrez@amd.com * Here we scan the coalescer FIFO and issue the max 40211308Santhony.gutierrez@amd.com * number of permitted probes to the TLB below. We 40311308Santhony.gutierrez@amd.com * permit bypassing of coalesced requests for the same 40411308Santhony.gutierrez@amd.com * tick_index. 40511308Santhony.gutierrez@amd.com * 40611308Santhony.gutierrez@amd.com * We do not access the next tick_index unless we've 40711308Santhony.gutierrez@amd.com * drained the previous one. The coalesced requests 40811308Santhony.gutierrez@amd.com * that are successfully sent are moved to the 40911308Santhony.gutierrez@amd.com * issuedTranslationsTable table (the table which keeps 41011308Santhony.gutierrez@amd.com * track of the outstanding reqs) 41111308Santhony.gutierrez@amd.com */ 41211308Santhony.gutierrez@amd.comvoid 41312126Sspwilson2@wisc.eduTLBCoalescer::processProbeTLBEvent() 41411308Santhony.gutierrez@amd.com{ 41511308Santhony.gutierrez@amd.com // number of TLB probes sent so far 41611308Santhony.gutierrez@amd.com int sent_probes = 0; 41711308Santhony.gutierrez@amd.com // rejected denotes a blocking event 41811308Santhony.gutierrez@amd.com bool rejected = false; 41911308Santhony.gutierrez@amd.com 42011308Santhony.gutierrez@amd.com // It is set to true either when the recvTiming of the TLB below 42111308Santhony.gutierrez@amd.com // returns false or when there is another outstanding request for the 42211308Santhony.gutierrez@amd.com // same virt. page. 42311308Santhony.gutierrez@amd.com 42412126Sspwilson2@wisc.edu DPRINTF(GPUTLB, "triggered TLBCoalescer %s\n", __func__); 42511308Santhony.gutierrez@amd.com 42612126Sspwilson2@wisc.edu for (auto iter = coalescerFIFO.begin(); 42712126Sspwilson2@wisc.edu iter != coalescerFIFO.end() && !rejected; ) { 42811308Santhony.gutierrez@amd.com int coalescedReq_cnt = iter->second.size(); 42911308Santhony.gutierrez@amd.com int i = 0; 43011308Santhony.gutierrez@amd.com int vector_index = 0; 43111308Santhony.gutierrez@amd.com 43211308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "coalescedReq_cnt is %d for tick_index %d\n", 43311308Santhony.gutierrez@amd.com coalescedReq_cnt, iter->first); 43411308Santhony.gutierrez@amd.com 43511308Santhony.gutierrez@amd.com while (i < coalescedReq_cnt) { 43611308Santhony.gutierrez@amd.com ++i; 43711308Santhony.gutierrez@amd.com PacketPtr first_packet = iter->second[vector_index][0]; 43811308Santhony.gutierrez@amd.com 43911308Santhony.gutierrez@amd.com // compute virtual page address for this request 44011308Santhony.gutierrez@amd.com Addr virt_page_addr = roundDown(first_packet->req->getVaddr(), 44111308Santhony.gutierrez@amd.com TheISA::PageBytes); 44211308Santhony.gutierrez@amd.com 44311308Santhony.gutierrez@amd.com // is there another outstanding request for the same page addr? 44411308Santhony.gutierrez@amd.com int pending_reqs = 44512126Sspwilson2@wisc.edu issuedTranslationsTable.count(virt_page_addr); 44611308Santhony.gutierrez@amd.com 44711308Santhony.gutierrez@amd.com if (pending_reqs) { 44811308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cannot issue - There are pending reqs for " 44911308Santhony.gutierrez@amd.com "page %#x\n", virt_page_addr); 45011308Santhony.gutierrez@amd.com 45111308Santhony.gutierrez@amd.com ++vector_index; 45211308Santhony.gutierrez@amd.com rejected = true; 45311308Santhony.gutierrez@amd.com 45411308Santhony.gutierrez@amd.com continue; 45511308Santhony.gutierrez@amd.com } 45611308Santhony.gutierrez@amd.com 45711308Santhony.gutierrez@amd.com // send the coalesced request for virt_page_addr 45812126Sspwilson2@wisc.edu if (!memSidePort[0]->sendTimingReq(first_packet)) { 45911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Failed to send TLB request for page %#x", 46011308Santhony.gutierrez@amd.com virt_page_addr); 46111308Santhony.gutierrez@amd.com 46211308Santhony.gutierrez@amd.com // No need for a retries queue since we are already buffering 46311308Santhony.gutierrez@amd.com // the coalesced request in coalescerFIFO. 46411308Santhony.gutierrez@amd.com rejected = true; 46511308Santhony.gutierrez@amd.com ++vector_index; 46611308Santhony.gutierrez@amd.com } else { 46711308Santhony.gutierrez@amd.com TheISA::GpuTLB::TranslationState *tmp_sender_state = 46811308Santhony.gutierrez@amd.com safe_cast<TheISA::GpuTLB::TranslationState*> 46911308Santhony.gutierrez@amd.com (first_packet->senderState); 47011308Santhony.gutierrez@amd.com 47111308Santhony.gutierrez@amd.com bool update_stats = !tmp_sender_state->prefetch; 47211308Santhony.gutierrez@amd.com 47311308Santhony.gutierrez@amd.com if (update_stats) { 47411308Santhony.gutierrez@amd.com // req_cnt is total number of packets represented 47511308Santhony.gutierrez@amd.com // by the one we just sent counting all the way from 47611308Santhony.gutierrez@amd.com // the top of TLB hiearchy (i.e., from the CU) 47711308Santhony.gutierrez@amd.com int req_cnt = tmp_sender_state->reqCnt.back(); 47812126Sspwilson2@wisc.edu queuingCycles += (curTick() * req_cnt); 47911308Santhony.gutierrez@amd.com 48011308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "%s sending pkt w/ req_cnt %d\n", 48112126Sspwilson2@wisc.edu name(), req_cnt); 48211308Santhony.gutierrez@amd.com 48311308Santhony.gutierrez@amd.com // pkt_cnt is number of packets we coalesced into the one 48411308Santhony.gutierrez@amd.com // we just sent but only at this coalescer level 48511308Santhony.gutierrez@amd.com int pkt_cnt = iter->second[vector_index].size(); 48612126Sspwilson2@wisc.edu localqueuingCycles += (curTick() * pkt_cnt); 48711308Santhony.gutierrez@amd.com } 48811308Santhony.gutierrez@amd.com 48911308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Successfully sent TLB request for page %#x", 49011308Santhony.gutierrez@amd.com virt_page_addr); 49111308Santhony.gutierrez@amd.com 49211308Santhony.gutierrez@amd.com //copy coalescedReq to issuedTranslationsTable 49312126Sspwilson2@wisc.edu issuedTranslationsTable[virt_page_addr] 49411308Santhony.gutierrez@amd.com = iter->second[vector_index]; 49511308Santhony.gutierrez@amd.com 49611308Santhony.gutierrez@amd.com //erase the entry of this coalesced req 49711308Santhony.gutierrez@amd.com iter->second.erase(iter->second.begin() + vector_index); 49811308Santhony.gutierrez@amd.com 49911308Santhony.gutierrez@amd.com if (iter->second.empty()) 50011308Santhony.gutierrez@amd.com assert(i == coalescedReq_cnt); 50111308Santhony.gutierrez@amd.com 50211308Santhony.gutierrez@amd.com sent_probes++; 50312126Sspwilson2@wisc.edu if (sent_probes == TLBProbesPerCycle) 50411308Santhony.gutierrez@amd.com return; 50511308Santhony.gutierrez@amd.com } 50611308Santhony.gutierrez@amd.com } 50711308Santhony.gutierrez@amd.com 50811308Santhony.gutierrez@amd.com //if there are no more coalesced reqs for this tick_index 50911308Santhony.gutierrez@amd.com //erase the hash_map with the first iterator 51011308Santhony.gutierrez@amd.com if (iter->second.empty()) { 51112126Sspwilson2@wisc.edu coalescerFIFO.erase(iter++); 51211308Santhony.gutierrez@amd.com } else { 51311308Santhony.gutierrez@amd.com ++iter; 51411308Santhony.gutierrez@amd.com } 51511308Santhony.gutierrez@amd.com } 51611308Santhony.gutierrez@amd.com} 51711308Santhony.gutierrez@amd.com 51812126Sspwilson2@wisc.eduvoid 51912126Sspwilson2@wisc.eduTLBCoalescer::processCleanupEvent() 52011308Santhony.gutierrez@amd.com{ 52112126Sspwilson2@wisc.edu while (!cleanupQueue.empty()) { 52212126Sspwilson2@wisc.edu Addr cleanup_addr = cleanupQueue.front(); 52312126Sspwilson2@wisc.edu cleanupQueue.pop(); 52412126Sspwilson2@wisc.edu issuedTranslationsTable.erase(cleanup_addr); 52511308Santhony.gutierrez@amd.com 52611308Santhony.gutierrez@amd.com DPRINTF(GPUTLB, "Cleanup - Delete coalescer entry with key %#x\n", 52711308Santhony.gutierrez@amd.com cleanup_addr); 52811308Santhony.gutierrez@amd.com } 52911308Santhony.gutierrez@amd.com} 53011308Santhony.gutierrez@amd.com 53111308Santhony.gutierrez@amd.comvoid 53211308Santhony.gutierrez@amd.comTLBCoalescer::regStats() 53311308Santhony.gutierrez@amd.com{ 53411523Sdavid.guillen@arm.com MemObject::regStats(); 53511523Sdavid.guillen@arm.com 53611308Santhony.gutierrez@amd.com uncoalescedAccesses 53711308Santhony.gutierrez@amd.com .name(name() + ".uncoalesced_accesses") 53811308Santhony.gutierrez@amd.com .desc("Number of uncoalesced TLB accesses") 53911308Santhony.gutierrez@amd.com ; 54011308Santhony.gutierrez@amd.com 54111308Santhony.gutierrez@amd.com coalescedAccesses 54211308Santhony.gutierrez@amd.com .name(name() + ".coalesced_accesses") 54311308Santhony.gutierrez@amd.com .desc("Number of coalesced TLB accesses") 54411308Santhony.gutierrez@amd.com ; 54511308Santhony.gutierrez@amd.com 54611308Santhony.gutierrez@amd.com queuingCycles 54711308Santhony.gutierrez@amd.com .name(name() + ".queuing_cycles") 54811308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue") 54911308Santhony.gutierrez@amd.com ; 55011308Santhony.gutierrez@amd.com 55111308Santhony.gutierrez@amd.com localqueuingCycles 55211308Santhony.gutierrez@amd.com .name(name() + ".local_queuing_cycles") 55311308Santhony.gutierrez@amd.com .desc("Number of cycles spent in queue for all incoming reqs") 55411308Santhony.gutierrez@amd.com ; 55511308Santhony.gutierrez@amd.com 55611308Santhony.gutierrez@amd.com localLatency 55711308Santhony.gutierrez@amd.com .name(name() + ".local_latency") 55811308Santhony.gutierrez@amd.com .desc("Avg. latency over all incoming pkts") 55911308Santhony.gutierrez@amd.com ; 56011308Santhony.gutierrez@amd.com 56111308Santhony.gutierrez@amd.com localLatency = localqueuingCycles / uncoalescedAccesses; 56211308Santhony.gutierrez@amd.com} 56311308Santhony.gutierrez@amd.com 56411308Santhony.gutierrez@amd.com 56511308Santhony.gutierrez@amd.comTLBCoalescer* 56611308Santhony.gutierrez@amd.comTLBCoalescerParams::create() 56711308Santhony.gutierrez@amd.com{ 56811308Santhony.gutierrez@amd.com return new TLBCoalescer(this); 56911308Santhony.gutierrez@amd.com} 57011308Santhony.gutierrez@amd.com 571