tlb_coalescer.cc revision 13449
111308Santhony.gutierrez@amd.com/*
211308Santhony.gutierrez@amd.com * Copyright (c) 2011-2015 Advanced Micro Devices, Inc.
311308Santhony.gutierrez@amd.com * All rights reserved.
411308Santhony.gutierrez@amd.com *
511308Santhony.gutierrez@amd.com * For use for simulation and test purposes only
611308Santhony.gutierrez@amd.com *
711308Santhony.gutierrez@amd.com * Redistribution and use in source and binary forms, with or without
811308Santhony.gutierrez@amd.com * modification, are permitted provided that the following conditions are met:
911308Santhony.gutierrez@amd.com *
1011308Santhony.gutierrez@amd.com * 1. Redistributions of source code must retain the above copyright notice,
1111308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer.
1211308Santhony.gutierrez@amd.com *
1311308Santhony.gutierrez@amd.com * 2. Redistributions in binary form must reproduce the above copyright notice,
1411308Santhony.gutierrez@amd.com * this list of conditions and the following disclaimer in the documentation
1511308Santhony.gutierrez@amd.com * and/or other materials provided with the distribution.
1611308Santhony.gutierrez@amd.com *
1712697Santhony.gutierrez@amd.com * 3. Neither the name of the copyright holder nor the names of its
1812697Santhony.gutierrez@amd.com * contributors may be used to endorse or promote products derived from this
1912697Santhony.gutierrez@amd.com * software without specific prior written permission.
2011308Santhony.gutierrez@amd.com *
2111308Santhony.gutierrez@amd.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2211308Santhony.gutierrez@amd.com * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2311308Santhony.gutierrez@amd.com * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2411308Santhony.gutierrez@amd.com * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
2511308Santhony.gutierrez@amd.com * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
2611308Santhony.gutierrez@amd.com * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
2711308Santhony.gutierrez@amd.com * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
2811308Santhony.gutierrez@amd.com * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
2911308Santhony.gutierrez@amd.com * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
3011308Santhony.gutierrez@amd.com * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
3111308Santhony.gutierrez@amd.com * POSSIBILITY OF SUCH DAMAGE.
3211308Santhony.gutierrez@amd.com *
3312697Santhony.gutierrez@amd.com * Authors: Lisa Hsu
3411308Santhony.gutierrez@amd.com */
3511308Santhony.gutierrez@amd.com
3611308Santhony.gutierrez@amd.com#include "gpu-compute/tlb_coalescer.hh"
3711308Santhony.gutierrez@amd.com
3811308Santhony.gutierrez@amd.com#include <cstring>
3911308Santhony.gutierrez@amd.com
4013449Sgabeblack@google.com#include "base/logging.hh"
4111308Santhony.gutierrez@amd.com#include "debug/GPUTLB.hh"
4212717Sbrandon.potter@amd.com#include "sim/process.hh"
4311308Santhony.gutierrez@amd.com
4412126Sspwilson2@wisc.eduTLBCoalescer::TLBCoalescer(const Params *p)
4512126Sspwilson2@wisc.edu    : MemObject(p),
4612126Sspwilson2@wisc.edu      clock(p->clk_domain->clockPeriod()),
4712126Sspwilson2@wisc.edu      TLBProbesPerCycle(p->probesPerCycle),
4812126Sspwilson2@wisc.edu      coalescingWindow(p->coalescingWindow),
4912126Sspwilson2@wisc.edu      disableCoalescing(p->disableCoalescing),
5012126Sspwilson2@wisc.edu      probeTLBEvent([this]{ processProbeTLBEvent(); },
5112126Sspwilson2@wisc.edu                    "Probe the TLB below",
5212126Sspwilson2@wisc.edu                    false, Event::CPU_Tick_Pri),
5312126Sspwilson2@wisc.edu      cleanupEvent([this]{ processCleanupEvent(); },
5412126Sspwilson2@wisc.edu                   "Cleanup issuedTranslationsTable hashmap",
5512126Sspwilson2@wisc.edu                   false, Event::Maximum_Pri)
5611308Santhony.gutierrez@amd.com{
5711308Santhony.gutierrez@amd.com    // create the slave ports based on the number of connected ports
5811308Santhony.gutierrez@amd.com    for (size_t i = 0; i < p->port_slave_connection_count; ++i) {
5911308Santhony.gutierrez@amd.com        cpuSidePort.push_back(new CpuSidePort(csprintf("%s-port%d", name(), i),
6011308Santhony.gutierrez@amd.com                                              this, i));
6111308Santhony.gutierrez@amd.com    }
6211308Santhony.gutierrez@amd.com
6311308Santhony.gutierrez@amd.com    // create the master ports based on the number of connected ports
6411308Santhony.gutierrez@amd.com    for (size_t i = 0; i < p->port_master_connection_count; ++i) {
6511308Santhony.gutierrez@amd.com        memSidePort.push_back(new MemSidePort(csprintf("%s-port%d", name(), i),
6611308Santhony.gutierrez@amd.com                                              this, i));
6711308Santhony.gutierrez@amd.com    }
6811308Santhony.gutierrez@amd.com}
6911308Santhony.gutierrez@amd.com
7011308Santhony.gutierrez@amd.comBaseSlavePort&
7111308Santhony.gutierrez@amd.comTLBCoalescer::getSlavePort(const std::string &if_name, PortID idx)
7211308Santhony.gutierrez@amd.com{
7311308Santhony.gutierrez@amd.com    if (if_name == "slave") {
7411308Santhony.gutierrez@amd.com        if (idx >= static_cast<PortID>(cpuSidePort.size())) {
7511308Santhony.gutierrez@amd.com            panic("TLBCoalescer::getSlavePort: unknown index %d\n", idx);
7611308Santhony.gutierrez@amd.com        }
7711308Santhony.gutierrez@amd.com
7811308Santhony.gutierrez@amd.com        return *cpuSidePort[idx];
7911308Santhony.gutierrez@amd.com    } else {
8011308Santhony.gutierrez@amd.com        panic("TLBCoalescer::getSlavePort: unknown port %s\n", if_name);
8111308Santhony.gutierrez@amd.com    }
8211308Santhony.gutierrez@amd.com}
8311308Santhony.gutierrez@amd.com
8411308Santhony.gutierrez@amd.comBaseMasterPort&
8511308Santhony.gutierrez@amd.comTLBCoalescer::getMasterPort(const std::string &if_name, PortID idx)
8611308Santhony.gutierrez@amd.com{
8711308Santhony.gutierrez@amd.com    if (if_name == "master") {
8811308Santhony.gutierrez@amd.com        if (idx >= static_cast<PortID>(memSidePort.size())) {
8911308Santhony.gutierrez@amd.com            panic("TLBCoalescer::getMasterPort: unknown index %d\n", idx);
9011308Santhony.gutierrez@amd.com        }
9111308Santhony.gutierrez@amd.com
9211308Santhony.gutierrez@amd.com        return *memSidePort[idx];
9311308Santhony.gutierrez@amd.com    } else {
9411308Santhony.gutierrez@amd.com        panic("TLBCoalescer::getMasterPort: unknown port %s\n", if_name);
9511308Santhony.gutierrez@amd.com    }
9611308Santhony.gutierrez@amd.com}
9711308Santhony.gutierrez@amd.com
9811308Santhony.gutierrez@amd.com/*
9911308Santhony.gutierrez@amd.com * This method returns true if the <incoming_pkt>
10011308Santhony.gutierrez@amd.com * can be coalesced with <coalesced_pkt> and false otherwise.
10111308Santhony.gutierrez@amd.com * A given set of rules is checked.
10211308Santhony.gutierrez@amd.com * The rules can potentially be modified based on the TLB level.
10311308Santhony.gutierrez@amd.com */
10411308Santhony.gutierrez@amd.combool
10511308Santhony.gutierrez@amd.comTLBCoalescer::canCoalesce(PacketPtr incoming_pkt, PacketPtr coalesced_pkt)
10611308Santhony.gutierrez@amd.com{
10711308Santhony.gutierrez@amd.com    if (disableCoalescing)
10811308Santhony.gutierrez@amd.com        return false;
10911308Santhony.gutierrez@amd.com
11011308Santhony.gutierrez@amd.com    TheISA::GpuTLB::TranslationState *incoming_state =
11111308Santhony.gutierrez@amd.com      safe_cast<TheISA::GpuTLB::TranslationState*>(incoming_pkt->senderState);
11211308Santhony.gutierrez@amd.com
11311308Santhony.gutierrez@amd.com    TheISA::GpuTLB::TranslationState *coalesced_state =
11411308Santhony.gutierrez@amd.com     safe_cast<TheISA::GpuTLB::TranslationState*>(coalesced_pkt->senderState);
11511308Santhony.gutierrez@amd.com
11611308Santhony.gutierrez@amd.com    // Rule 1: Coalesce requests only if they
11711308Santhony.gutierrez@amd.com    // fall within the same virtual page
11811308Santhony.gutierrez@amd.com    Addr incoming_virt_page_addr = roundDown(incoming_pkt->req->getVaddr(),
11911308Santhony.gutierrez@amd.com                                             TheISA::PageBytes);
12011308Santhony.gutierrez@amd.com
12111308Santhony.gutierrez@amd.com    Addr coalesced_virt_page_addr = roundDown(coalesced_pkt->req->getVaddr(),
12211308Santhony.gutierrez@amd.com                                              TheISA::PageBytes);
12311308Santhony.gutierrez@amd.com
12411308Santhony.gutierrez@amd.com    if (incoming_virt_page_addr != coalesced_virt_page_addr)
12511308Santhony.gutierrez@amd.com        return false;
12611308Santhony.gutierrez@amd.com
12711308Santhony.gutierrez@amd.com    //* Rule 2: Coalesce requests only if they
12811308Santhony.gutierrez@amd.com    // share a TLB Mode, i.e. they are both read
12911308Santhony.gutierrez@amd.com    // or write requests.
13011308Santhony.gutierrez@amd.com    BaseTLB::Mode incoming_mode = incoming_state->tlbMode;
13111308Santhony.gutierrez@amd.com    BaseTLB::Mode coalesced_mode = coalesced_state->tlbMode;
13211308Santhony.gutierrez@amd.com
13311308Santhony.gutierrez@amd.com    if (incoming_mode != coalesced_mode)
13411308Santhony.gutierrez@amd.com        return false;
13511308Santhony.gutierrez@amd.com
13611308Santhony.gutierrez@amd.com    // when we can coalesce a packet update the reqCnt
13711308Santhony.gutierrez@amd.com    // that is the number of packets represented by
13811308Santhony.gutierrez@amd.com    // this coalesced packet
13911308Santhony.gutierrez@amd.com    if (!incoming_state->prefetch)
14011308Santhony.gutierrez@amd.com        coalesced_state->reqCnt.back() += incoming_state->reqCnt.back();
14111308Santhony.gutierrez@amd.com
14211308Santhony.gutierrez@amd.com    return true;
14311308Santhony.gutierrez@amd.com}
14411308Santhony.gutierrez@amd.com
14511308Santhony.gutierrez@amd.com/*
14611308Santhony.gutierrez@amd.com * We need to update the physical addresses of all the translation requests
14711308Santhony.gutierrez@amd.com * that were coalesced into the one that just returned.
14811308Santhony.gutierrez@amd.com */
14911308Santhony.gutierrez@amd.comvoid
15011308Santhony.gutierrez@amd.comTLBCoalescer::updatePhysAddresses(PacketPtr pkt)
15111308Santhony.gutierrez@amd.com{
15211308Santhony.gutierrez@amd.com    Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes);
15311308Santhony.gutierrez@amd.com
15411308Santhony.gutierrez@amd.com    DPRINTF(GPUTLB, "Update phys. addr. for %d coalesced reqs for page %#x\n",
15511308Santhony.gutierrez@amd.com            issuedTranslationsTable[virt_page_addr].size(), virt_page_addr);
15611308Santhony.gutierrez@amd.com
15711308Santhony.gutierrez@amd.com    TheISA::GpuTLB::TranslationState *sender_state =
15811308Santhony.gutierrez@amd.com        safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
15911308Santhony.gutierrez@amd.com
16012717Sbrandon.potter@amd.com    TheISA::TlbEntry *tlb_entry = sender_state->tlbEntry;
16111308Santhony.gutierrez@amd.com    assert(tlb_entry);
16211308Santhony.gutierrez@amd.com    Addr first_entry_vaddr = tlb_entry->vaddr;
16311308Santhony.gutierrez@amd.com    Addr first_entry_paddr = tlb_entry->paddr;
16411308Santhony.gutierrez@amd.com    int page_size = tlb_entry->size();
16511308Santhony.gutierrez@amd.com    bool uncacheable = tlb_entry->uncacheable;
16611308Santhony.gutierrez@amd.com    int first_hit_level = sender_state->hitLevel;
16711308Santhony.gutierrez@amd.com
16811308Santhony.gutierrez@amd.com    // Get the physical page address of the translated request
16911308Santhony.gutierrez@amd.com    // Using the page_size specified in the TLBEntry allows us
17011308Santhony.gutierrez@amd.com    // to support different page sizes.
17111308Santhony.gutierrez@amd.com    Addr phys_page_paddr = pkt->req->getPaddr();
17211308Santhony.gutierrez@amd.com    phys_page_paddr &= ~(page_size - 1);
17311308Santhony.gutierrez@amd.com
17411308Santhony.gutierrez@amd.com    for (int i = 0; i < issuedTranslationsTable[virt_page_addr].size(); ++i) {
17511308Santhony.gutierrez@amd.com        PacketPtr local_pkt = issuedTranslationsTable[virt_page_addr][i];
17611308Santhony.gutierrez@amd.com        TheISA::GpuTLB::TranslationState *sender_state =
17711308Santhony.gutierrez@amd.com            safe_cast<TheISA::GpuTLB::TranslationState*>(
17811308Santhony.gutierrez@amd.com                    local_pkt->senderState);
17911308Santhony.gutierrez@amd.com
18011308Santhony.gutierrez@amd.com        // we are sending the packet back, so pop the reqCnt associated
18111308Santhony.gutierrez@amd.com        // with this level in the TLB hiearchy
18211308Santhony.gutierrez@amd.com        if (!sender_state->prefetch)
18311308Santhony.gutierrez@amd.com            sender_state->reqCnt.pop_back();
18411308Santhony.gutierrez@amd.com
18511308Santhony.gutierrez@amd.com        /*
18611308Santhony.gutierrez@amd.com         * Only the first packet from this coalesced request has been
18711308Santhony.gutierrez@amd.com         * translated. Grab the translated phys. page addr and update the
18811308Santhony.gutierrez@amd.com         * physical addresses of the remaining packets with the appropriate
18911308Santhony.gutierrez@amd.com         * page offsets.
19011308Santhony.gutierrez@amd.com         */
19111308Santhony.gutierrez@amd.com        if (i) {
19211308Santhony.gutierrez@amd.com            Addr paddr = phys_page_paddr;
19311308Santhony.gutierrez@amd.com            paddr |= (local_pkt->req->getVaddr() & (page_size - 1));
19411308Santhony.gutierrez@amd.com            local_pkt->req->setPaddr(paddr);
19511308Santhony.gutierrez@amd.com
19611308Santhony.gutierrez@amd.com            if (uncacheable)
19711308Santhony.gutierrez@amd.com                local_pkt->req->setFlags(Request::UNCACHEABLE);
19811308Santhony.gutierrez@amd.com
19911308Santhony.gutierrez@amd.com            // update senderState->tlbEntry, so we can insert
20011308Santhony.gutierrez@amd.com            // the correct TLBEentry in the TLBs above.
20112717Sbrandon.potter@amd.com            auto p = sender_state->tc->getProcessPtr();
20211308Santhony.gutierrez@amd.com            sender_state->tlbEntry =
20312717Sbrandon.potter@amd.com                new TheISA::TlbEntry(p->pid(), first_entry_vaddr,
20412717Sbrandon.potter@amd.com                    first_entry_paddr, false, false);
20511308Santhony.gutierrez@amd.com
20611308Santhony.gutierrez@amd.com            // update the hitLevel for all uncoalesced reqs
20711308Santhony.gutierrez@amd.com            // so that each packet knows where it hit
20811308Santhony.gutierrez@amd.com            // (used for statistics in the CUs)
20911308Santhony.gutierrez@amd.com            sender_state->hitLevel = first_hit_level;
21011308Santhony.gutierrez@amd.com        }
21111308Santhony.gutierrez@amd.com
21211308Santhony.gutierrez@amd.com        SlavePort *return_port = sender_state->ports.back();
21311308Santhony.gutierrez@amd.com        sender_state->ports.pop_back();
21411308Santhony.gutierrez@amd.com
21511308Santhony.gutierrez@amd.com        // Translation is done - Convert to a response pkt if necessary and
21611308Santhony.gutierrez@amd.com        // send the translation back
21711308Santhony.gutierrez@amd.com        if (local_pkt->isRequest()) {
21811308Santhony.gutierrez@amd.com            local_pkt->makeTimingResponse();
21911308Santhony.gutierrez@amd.com        }
22011308Santhony.gutierrez@amd.com
22111308Santhony.gutierrez@amd.com        return_port->sendTimingResp(local_pkt);
22211308Santhony.gutierrez@amd.com    }
22311308Santhony.gutierrez@amd.com
22411308Santhony.gutierrez@amd.com    // schedule clean up for end of this cycle
22511308Santhony.gutierrez@amd.com    // This is a maximum priority event and must be on
22611308Santhony.gutierrez@amd.com    // the same cycle as GPUTLB cleanup event to prevent
22711308Santhony.gutierrez@amd.com    // race conditions with an IssueProbeEvent caused by
22811308Santhony.gutierrez@amd.com    // MemSidePort::recvReqRetry
22911308Santhony.gutierrez@amd.com    cleanupQueue.push(virt_page_addr);
23011308Santhony.gutierrez@amd.com
23111308Santhony.gutierrez@amd.com    if (!cleanupEvent.scheduled())
23211308Santhony.gutierrez@amd.com        schedule(cleanupEvent, curTick());
23311308Santhony.gutierrez@amd.com}
23411308Santhony.gutierrez@amd.com
23511308Santhony.gutierrez@amd.com// Receive translation requests, create a coalesced request,
23611308Santhony.gutierrez@amd.com// and send them to the TLB (TLBProbesPerCycle)
23711308Santhony.gutierrez@amd.combool
23811308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvTimingReq(PacketPtr pkt)
23911308Santhony.gutierrez@amd.com{
24011308Santhony.gutierrez@amd.com    // first packet of a coalesced request
24111308Santhony.gutierrez@amd.com    PacketPtr first_packet = nullptr;
24211308Santhony.gutierrez@amd.com    // true if we are able to do coalescing
24311308Santhony.gutierrez@amd.com    bool didCoalesce = false;
24411308Santhony.gutierrez@amd.com    // number of coalesced reqs for a given window
24511308Santhony.gutierrez@amd.com    int coalescedReq_cnt = 0;
24611308Santhony.gutierrez@amd.com
24711308Santhony.gutierrez@amd.com    TheISA::GpuTLB::TranslationState *sender_state =
24811308Santhony.gutierrez@amd.com        safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
24911308Santhony.gutierrez@amd.com
25011308Santhony.gutierrez@amd.com    // push back the port to remember the path back
25111308Santhony.gutierrez@amd.com    sender_state->ports.push_back(this);
25211308Santhony.gutierrez@amd.com
25311308Santhony.gutierrez@amd.com    bool update_stats = !sender_state->prefetch;
25411308Santhony.gutierrez@amd.com
25511308Santhony.gutierrez@amd.com    if (update_stats) {
25611308Santhony.gutierrez@amd.com        // if reqCnt is empty then this packet does not represent
25711308Santhony.gutierrez@amd.com        // multiple uncoalesced reqs(pkts) but just a single pkt.
25811308Santhony.gutierrez@amd.com        // If it does though then the reqCnt for each level in the
25911308Santhony.gutierrez@amd.com        // hierarchy accumulates the total number of reqs this packet
26011308Santhony.gutierrez@amd.com        // represents
26111308Santhony.gutierrez@amd.com        int req_cnt = 1;
26211308Santhony.gutierrez@amd.com
26311308Santhony.gutierrez@amd.com        if (!sender_state->reqCnt.empty())
26411308Santhony.gutierrez@amd.com            req_cnt = sender_state->reqCnt.back();
26511308Santhony.gutierrez@amd.com
26611308Santhony.gutierrez@amd.com        sender_state->reqCnt.push_back(req_cnt);
26711308Santhony.gutierrez@amd.com
26811308Santhony.gutierrez@amd.com        // update statistics
26911308Santhony.gutierrez@amd.com        coalescer->uncoalescedAccesses++;
27011308Santhony.gutierrez@amd.com        req_cnt = sender_state->reqCnt.back();
27111308Santhony.gutierrez@amd.com        DPRINTF(GPUTLB, "receiving pkt w/ req_cnt %d\n", req_cnt);
27211308Santhony.gutierrez@amd.com        coalescer->queuingCycles -= (curTick() * req_cnt);
27311308Santhony.gutierrez@amd.com        coalescer->localqueuingCycles -= curTick();
27411308Santhony.gutierrez@amd.com    }
27511308Santhony.gutierrez@amd.com
27611308Santhony.gutierrez@amd.com    // FIXME if you want to coalesce not based on the issueTime
27711308Santhony.gutierrez@amd.com    // of the packets (i.e., from the compute unit's perspective)
27811308Santhony.gutierrez@amd.com    // but based on when they reached this coalescer then
27911308Santhony.gutierrez@amd.com    // remove the following if statement and use curTick() or
28011308Santhony.gutierrez@amd.com    // coalescingWindow for the tick_index.
28111308Santhony.gutierrez@amd.com    if (!sender_state->issueTime)
28211308Santhony.gutierrez@amd.com       sender_state->issueTime = curTick();
28311308Santhony.gutierrez@amd.com
28411308Santhony.gutierrez@amd.com    // The tick index is used as a key to the coalescerFIFO hashmap.
28511308Santhony.gutierrez@amd.com    // It is shared by all candidates that fall within the
28611308Santhony.gutierrez@amd.com    // given coalescingWindow.
28711308Santhony.gutierrez@amd.com    int64_t tick_index = sender_state->issueTime / coalescer->coalescingWindow;
28811308Santhony.gutierrez@amd.com
28911308Santhony.gutierrez@amd.com    if (coalescer->coalescerFIFO.count(tick_index)) {
29011308Santhony.gutierrez@amd.com        coalescedReq_cnt = coalescer->coalescerFIFO[tick_index].size();
29111308Santhony.gutierrez@amd.com    }
29211308Santhony.gutierrez@amd.com
29311308Santhony.gutierrez@amd.com    // see if we can coalesce the incoming pkt with another
29411308Santhony.gutierrez@amd.com    // coalesced request with the same tick_index
29511308Santhony.gutierrez@amd.com    for (int i = 0; i < coalescedReq_cnt; ++i) {
29611308Santhony.gutierrez@amd.com        first_packet = coalescer->coalescerFIFO[tick_index][i][0];
29711308Santhony.gutierrez@amd.com
29811308Santhony.gutierrez@amd.com        if (coalescer->canCoalesce(pkt, first_packet)) {
29911308Santhony.gutierrez@amd.com            coalescer->coalescerFIFO[tick_index][i].push_back(pkt);
30011308Santhony.gutierrez@amd.com
30111308Santhony.gutierrez@amd.com            DPRINTF(GPUTLB, "Coalesced req %i w/ tick_index %d has %d reqs\n",
30211308Santhony.gutierrez@amd.com                    i, tick_index,
30311308Santhony.gutierrez@amd.com                    coalescer->coalescerFIFO[tick_index][i].size());
30411308Santhony.gutierrez@amd.com
30511308Santhony.gutierrez@amd.com            didCoalesce = true;
30611308Santhony.gutierrez@amd.com            break;
30711308Santhony.gutierrez@amd.com        }
30811308Santhony.gutierrez@amd.com    }
30911308Santhony.gutierrez@amd.com
31011308Santhony.gutierrez@amd.com    // if this is the first request for this tick_index
31111308Santhony.gutierrez@amd.com    // or we did not manage to coalesce, update stats
31211308Santhony.gutierrez@amd.com    // and make necessary allocations.
31311308Santhony.gutierrez@amd.com    if (!coalescedReq_cnt || !didCoalesce) {
31411308Santhony.gutierrez@amd.com        if (update_stats)
31511308Santhony.gutierrez@amd.com            coalescer->coalescedAccesses++;
31611308Santhony.gutierrez@amd.com
31711308Santhony.gutierrez@amd.com        std::vector<PacketPtr> new_array;
31811308Santhony.gutierrez@amd.com        new_array.push_back(pkt);
31911308Santhony.gutierrez@amd.com        coalescer->coalescerFIFO[tick_index].push_back(new_array);
32011308Santhony.gutierrez@amd.com
32111308Santhony.gutierrez@amd.com        DPRINTF(GPUTLB, "coalescerFIFO[%d] now has %d coalesced reqs after "
32211308Santhony.gutierrez@amd.com                "push\n", tick_index,
32311308Santhony.gutierrez@amd.com                coalescer->coalescerFIFO[tick_index].size());
32411308Santhony.gutierrez@amd.com    }
32511308Santhony.gutierrez@amd.com
32611308Santhony.gutierrez@amd.com    //schedule probeTLBEvent next cycle to send the
32711308Santhony.gutierrez@amd.com    //coalesced requests to the TLB
32811308Santhony.gutierrez@amd.com    if (!coalescer->probeTLBEvent.scheduled()) {
32911308Santhony.gutierrez@amd.com        coalescer->schedule(coalescer->probeTLBEvent,
33011308Santhony.gutierrez@amd.com                curTick() + coalescer->ticks(1));
33111308Santhony.gutierrez@amd.com    }
33211308Santhony.gutierrez@amd.com
33311308Santhony.gutierrez@amd.com    return true;
33411308Santhony.gutierrez@amd.com}
33511308Santhony.gutierrez@amd.com
33611308Santhony.gutierrez@amd.comvoid
33711308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvReqRetry()
33811308Santhony.gutierrez@amd.com{
33913449Sgabeblack@google.com    panic("recvReqRetry called");
34011308Santhony.gutierrez@amd.com}
34111308Santhony.gutierrez@amd.com
34211308Santhony.gutierrez@amd.comvoid
34311308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::recvFunctional(PacketPtr pkt)
34411308Santhony.gutierrez@amd.com{
34511308Santhony.gutierrez@amd.com
34611308Santhony.gutierrez@amd.com    TheISA::GpuTLB::TranslationState *sender_state =
34711308Santhony.gutierrez@amd.com        safe_cast<TheISA::GpuTLB::TranslationState*>(pkt->senderState);
34811308Santhony.gutierrez@amd.com
34911308Santhony.gutierrez@amd.com    bool update_stats = !sender_state->prefetch;
35011308Santhony.gutierrez@amd.com
35111308Santhony.gutierrez@amd.com    if (update_stats)
35211308Santhony.gutierrez@amd.com        coalescer->uncoalescedAccesses++;
35311308Santhony.gutierrez@amd.com
35411308Santhony.gutierrez@amd.com    // If there is a pending timing request for this virtual address
35511308Santhony.gutierrez@amd.com    // print a warning message. This is a temporary caveat of
35611308Santhony.gutierrez@amd.com    // the current simulator where atomic and timing requests can
35711308Santhony.gutierrez@amd.com    // coexist. FIXME remove this check/warning in the future.
35811308Santhony.gutierrez@amd.com    Addr virt_page_addr = roundDown(pkt->req->getVaddr(), TheISA::PageBytes);
35911308Santhony.gutierrez@amd.com    int map_count = coalescer->issuedTranslationsTable.count(virt_page_addr);
36011308Santhony.gutierrez@amd.com
36111308Santhony.gutierrez@amd.com    if (map_count) {
36211308Santhony.gutierrez@amd.com        DPRINTF(GPUTLB, "Warning! Functional access to addr %#x sees timing "
36311308Santhony.gutierrez@amd.com                "req. pending\n", virt_page_addr);
36411308Santhony.gutierrez@amd.com    }
36511308Santhony.gutierrez@amd.com
36611308Santhony.gutierrez@amd.com    coalescer->memSidePort[0]->sendFunctional(pkt);
36711308Santhony.gutierrez@amd.com}
36811308Santhony.gutierrez@amd.com
36911308Santhony.gutierrez@amd.comAddrRangeList
37011308Santhony.gutierrez@amd.comTLBCoalescer::CpuSidePort::getAddrRanges() const
37111308Santhony.gutierrez@amd.com{
37211308Santhony.gutierrez@amd.com    // currently not checked by the master
37311308Santhony.gutierrez@amd.com    AddrRangeList ranges;
37411308Santhony.gutierrez@amd.com
37511308Santhony.gutierrez@amd.com    return ranges;
37611308Santhony.gutierrez@amd.com}
37711308Santhony.gutierrez@amd.com
37811308Santhony.gutierrez@amd.combool
37911308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvTimingResp(PacketPtr pkt)
38011308Santhony.gutierrez@amd.com{
38111308Santhony.gutierrez@amd.com    // a translation completed and returned
38211308Santhony.gutierrez@amd.com    coalescer->updatePhysAddresses(pkt);
38311308Santhony.gutierrez@amd.com
38411308Santhony.gutierrez@amd.com    return true;
38511308Santhony.gutierrez@amd.com}
38611308Santhony.gutierrez@amd.com
38711308Santhony.gutierrez@amd.comvoid
38811308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvReqRetry()
38911308Santhony.gutierrez@amd.com{
39011308Santhony.gutierrez@amd.com    //we've receeived a retry. Schedule a probeTLBEvent
39111308Santhony.gutierrez@amd.com    if (!coalescer->probeTLBEvent.scheduled())
39211308Santhony.gutierrez@amd.com        coalescer->schedule(coalescer->probeTLBEvent,
39311308Santhony.gutierrez@amd.com                curTick() + coalescer->ticks(1));
39411308Santhony.gutierrez@amd.com}
39511308Santhony.gutierrez@amd.com
39611308Santhony.gutierrez@amd.comvoid
39711308Santhony.gutierrez@amd.comTLBCoalescer::MemSidePort::recvFunctional(PacketPtr pkt)
39811308Santhony.gutierrez@amd.com{
39911308Santhony.gutierrez@amd.com    fatal("Memory side recvFunctional() not implemented in TLB coalescer.\n");
40011308Santhony.gutierrez@amd.com}
40111308Santhony.gutierrez@amd.com
40211308Santhony.gutierrez@amd.com/*
40311308Santhony.gutierrez@amd.com * Here we scan the coalescer FIFO and issue the max
40411308Santhony.gutierrez@amd.com * number of permitted probes to the TLB below. We
40511308Santhony.gutierrez@amd.com * permit bypassing of coalesced requests for the same
40611308Santhony.gutierrez@amd.com * tick_index.
40711308Santhony.gutierrez@amd.com *
40811308Santhony.gutierrez@amd.com * We do not access the next tick_index unless we've
40911308Santhony.gutierrez@amd.com * drained the previous one. The coalesced requests
41011308Santhony.gutierrez@amd.com * that are successfully sent are moved to the
41111308Santhony.gutierrez@amd.com * issuedTranslationsTable table (the table which keeps
41211308Santhony.gutierrez@amd.com * track of the outstanding reqs)
41311308Santhony.gutierrez@amd.com */
41411308Santhony.gutierrez@amd.comvoid
41512126Sspwilson2@wisc.eduTLBCoalescer::processProbeTLBEvent()
41611308Santhony.gutierrez@amd.com{
41711308Santhony.gutierrez@amd.com    // number of TLB probes sent so far
41811308Santhony.gutierrez@amd.com    int sent_probes = 0;
41911308Santhony.gutierrez@amd.com    // rejected denotes a blocking event
42011308Santhony.gutierrez@amd.com    bool rejected = false;
42111308Santhony.gutierrez@amd.com
42211308Santhony.gutierrez@amd.com    // It is set to true either when the recvTiming of the TLB below
42311308Santhony.gutierrez@amd.com    // returns false or when there is another outstanding request for the
42411308Santhony.gutierrez@amd.com    // same virt. page.
42511308Santhony.gutierrez@amd.com
42612126Sspwilson2@wisc.edu    DPRINTF(GPUTLB, "triggered TLBCoalescer %s\n", __func__);
42711308Santhony.gutierrez@amd.com
42812126Sspwilson2@wisc.edu    for (auto iter = coalescerFIFO.begin();
42912126Sspwilson2@wisc.edu         iter != coalescerFIFO.end() && !rejected; ) {
43011308Santhony.gutierrez@amd.com        int coalescedReq_cnt = iter->second.size();
43111308Santhony.gutierrez@amd.com        int i = 0;
43211308Santhony.gutierrez@amd.com        int vector_index = 0;
43311308Santhony.gutierrez@amd.com
43411308Santhony.gutierrez@amd.com        DPRINTF(GPUTLB, "coalescedReq_cnt is %d for tick_index %d\n",
43511308Santhony.gutierrez@amd.com               coalescedReq_cnt, iter->first);
43611308Santhony.gutierrez@amd.com
43711308Santhony.gutierrez@amd.com        while (i < coalescedReq_cnt) {
43811308Santhony.gutierrez@amd.com            ++i;
43911308Santhony.gutierrez@amd.com            PacketPtr first_packet = iter->second[vector_index][0];
44011308Santhony.gutierrez@amd.com
44111308Santhony.gutierrez@amd.com            // compute virtual page address for this request
44211308Santhony.gutierrez@amd.com            Addr virt_page_addr = roundDown(first_packet->req->getVaddr(),
44311308Santhony.gutierrez@amd.com                    TheISA::PageBytes);
44411308Santhony.gutierrez@amd.com
44511308Santhony.gutierrez@amd.com            // is there another outstanding request for the same page addr?
44611308Santhony.gutierrez@amd.com            int pending_reqs =
44712126Sspwilson2@wisc.edu                issuedTranslationsTable.count(virt_page_addr);
44811308Santhony.gutierrez@amd.com
44911308Santhony.gutierrez@amd.com            if (pending_reqs) {
45011308Santhony.gutierrez@amd.com                DPRINTF(GPUTLB, "Cannot issue - There are pending reqs for "
45111308Santhony.gutierrez@amd.com                        "page %#x\n", virt_page_addr);
45211308Santhony.gutierrez@amd.com
45311308Santhony.gutierrez@amd.com                ++vector_index;
45411308Santhony.gutierrez@amd.com                rejected = true;
45511308Santhony.gutierrez@amd.com
45611308Santhony.gutierrez@amd.com                continue;
45711308Santhony.gutierrez@amd.com            }
45811308Santhony.gutierrez@amd.com
45911308Santhony.gutierrez@amd.com            // send the coalesced request for virt_page_addr
46012126Sspwilson2@wisc.edu            if (!memSidePort[0]->sendTimingReq(first_packet)) {
46111308Santhony.gutierrez@amd.com                DPRINTF(GPUTLB, "Failed to send TLB request for page %#x",
46211308Santhony.gutierrez@amd.com                       virt_page_addr);
46311308Santhony.gutierrez@amd.com
46411308Santhony.gutierrez@amd.com                // No need for a retries queue since we are already buffering
46511308Santhony.gutierrez@amd.com                // the coalesced request in coalescerFIFO.
46611308Santhony.gutierrez@amd.com                rejected = true;
46711308Santhony.gutierrez@amd.com                ++vector_index;
46811308Santhony.gutierrez@amd.com            } else {
46911308Santhony.gutierrez@amd.com                TheISA::GpuTLB::TranslationState *tmp_sender_state =
47011308Santhony.gutierrez@amd.com                    safe_cast<TheISA::GpuTLB::TranslationState*>
47111308Santhony.gutierrez@amd.com                    (first_packet->senderState);
47211308Santhony.gutierrez@amd.com
47311308Santhony.gutierrez@amd.com                bool update_stats = !tmp_sender_state->prefetch;
47411308Santhony.gutierrez@amd.com
47511308Santhony.gutierrez@amd.com                if (update_stats) {
47611308Santhony.gutierrez@amd.com                    // req_cnt is total number of packets represented
47711308Santhony.gutierrez@amd.com                    // by the one we just sent counting all the way from
47811308Santhony.gutierrez@amd.com                    // the top of TLB hiearchy (i.e., from the CU)
47911308Santhony.gutierrez@amd.com                    int req_cnt = tmp_sender_state->reqCnt.back();
48012126Sspwilson2@wisc.edu                    queuingCycles += (curTick() * req_cnt);
48111308Santhony.gutierrez@amd.com
48211308Santhony.gutierrez@amd.com                    DPRINTF(GPUTLB, "%s sending pkt w/ req_cnt %d\n",
48312126Sspwilson2@wisc.edu                            name(), req_cnt);
48411308Santhony.gutierrez@amd.com
48511308Santhony.gutierrez@amd.com                    // pkt_cnt is number of packets we coalesced into the one
48611308Santhony.gutierrez@amd.com                    // we just sent but only at this coalescer level
48711308Santhony.gutierrez@amd.com                    int pkt_cnt = iter->second[vector_index].size();
48812126Sspwilson2@wisc.edu                    localqueuingCycles += (curTick() * pkt_cnt);
48911308Santhony.gutierrez@amd.com                }
49011308Santhony.gutierrez@amd.com
49111308Santhony.gutierrez@amd.com                DPRINTF(GPUTLB, "Successfully sent TLB request for page %#x",
49211308Santhony.gutierrez@amd.com                       virt_page_addr);
49311308Santhony.gutierrez@amd.com
49411308Santhony.gutierrez@amd.com                //copy coalescedReq to issuedTranslationsTable
49512126Sspwilson2@wisc.edu                issuedTranslationsTable[virt_page_addr]
49611308Santhony.gutierrez@amd.com                    = iter->second[vector_index];
49711308Santhony.gutierrez@amd.com
49811308Santhony.gutierrez@amd.com                //erase the entry of this coalesced req
49911308Santhony.gutierrez@amd.com                iter->second.erase(iter->second.begin() + vector_index);
50011308Santhony.gutierrez@amd.com
50111308Santhony.gutierrez@amd.com                if (iter->second.empty())
50211308Santhony.gutierrez@amd.com                    assert(i == coalescedReq_cnt);
50311308Santhony.gutierrez@amd.com
50411308Santhony.gutierrez@amd.com                sent_probes++;
50512126Sspwilson2@wisc.edu                if (sent_probes == TLBProbesPerCycle)
50611308Santhony.gutierrez@amd.com                   return;
50711308Santhony.gutierrez@amd.com            }
50811308Santhony.gutierrez@amd.com        }
50911308Santhony.gutierrez@amd.com
51011308Santhony.gutierrez@amd.com        //if there are no more coalesced reqs for this tick_index
51111308Santhony.gutierrez@amd.com        //erase the hash_map with the first iterator
51211308Santhony.gutierrez@amd.com        if (iter->second.empty()) {
51312126Sspwilson2@wisc.edu            coalescerFIFO.erase(iter++);
51411308Santhony.gutierrez@amd.com        } else {
51511308Santhony.gutierrez@amd.com            ++iter;
51611308Santhony.gutierrez@amd.com        }
51711308Santhony.gutierrez@amd.com    }
51811308Santhony.gutierrez@amd.com}
51911308Santhony.gutierrez@amd.com
52012126Sspwilson2@wisc.eduvoid
52112126Sspwilson2@wisc.eduTLBCoalescer::processCleanupEvent()
52211308Santhony.gutierrez@amd.com{
52312126Sspwilson2@wisc.edu    while (!cleanupQueue.empty()) {
52412126Sspwilson2@wisc.edu        Addr cleanup_addr = cleanupQueue.front();
52512126Sspwilson2@wisc.edu        cleanupQueue.pop();
52612126Sspwilson2@wisc.edu        issuedTranslationsTable.erase(cleanup_addr);
52711308Santhony.gutierrez@amd.com
52811308Santhony.gutierrez@amd.com        DPRINTF(GPUTLB, "Cleanup - Delete coalescer entry with key %#x\n",
52911308Santhony.gutierrez@amd.com                cleanup_addr);
53011308Santhony.gutierrez@amd.com    }
53111308Santhony.gutierrez@amd.com}
53211308Santhony.gutierrez@amd.com
53311308Santhony.gutierrez@amd.comvoid
53411308Santhony.gutierrez@amd.comTLBCoalescer::regStats()
53511308Santhony.gutierrez@amd.com{
53611523Sdavid.guillen@arm.com    MemObject::regStats();
53711523Sdavid.guillen@arm.com
53811308Santhony.gutierrez@amd.com    uncoalescedAccesses
53911308Santhony.gutierrez@amd.com        .name(name() + ".uncoalesced_accesses")
54011308Santhony.gutierrez@amd.com        .desc("Number of uncoalesced TLB accesses")
54111308Santhony.gutierrez@amd.com        ;
54211308Santhony.gutierrez@amd.com
54311308Santhony.gutierrez@amd.com    coalescedAccesses
54411308Santhony.gutierrez@amd.com        .name(name() + ".coalesced_accesses")
54511308Santhony.gutierrez@amd.com        .desc("Number of coalesced TLB accesses")
54611308Santhony.gutierrez@amd.com        ;
54711308Santhony.gutierrez@amd.com
54811308Santhony.gutierrez@amd.com    queuingCycles
54911308Santhony.gutierrez@amd.com        .name(name() + ".queuing_cycles")
55011308Santhony.gutierrez@amd.com        .desc("Number of cycles spent in queue")
55111308Santhony.gutierrez@amd.com        ;
55211308Santhony.gutierrez@amd.com
55311308Santhony.gutierrez@amd.com    localqueuingCycles
55411308Santhony.gutierrez@amd.com        .name(name() + ".local_queuing_cycles")
55511308Santhony.gutierrez@amd.com        .desc("Number of cycles spent in queue for all incoming reqs")
55611308Santhony.gutierrez@amd.com        ;
55711308Santhony.gutierrez@amd.com
55811308Santhony.gutierrez@amd.com    localLatency
55911308Santhony.gutierrez@amd.com        .name(name() + ".local_latency")
56011308Santhony.gutierrez@amd.com        .desc("Avg. latency over all incoming pkts")
56111308Santhony.gutierrez@amd.com        ;
56211308Santhony.gutierrez@amd.com
56311308Santhony.gutierrez@amd.com    localLatency = localqueuingCycles / uncoalescedAccesses;
56411308Santhony.gutierrez@amd.com}
56511308Santhony.gutierrez@amd.com
56611308Santhony.gutierrez@amd.com
56711308Santhony.gutierrez@amd.comTLBCoalescer*
56811308Santhony.gutierrez@amd.comTLBCoalescerParams::create()
56911308Santhony.gutierrez@amd.com{
57011308Santhony.gutierrez@amd.com    return new TLBCoalescer(this);
57111308Santhony.gutierrez@amd.com}
57211308Santhony.gutierrez@amd.com
573