dev/arm/smmu_v3_transl.cc

/*
 * Copyright (c) 2013, 2018-2019 ARM Limited
 * All rights reserved
 *
 * The license below extends only to copyright in the software and shall
 * not be construed as granting a license to any other intellectual
 * property including but not limited to intellectual property relating
 * to a hardware implementation of the functionality of the software
 * licensed hereunder.  You may use the software subject to the license
 * terms below provided that you ensure that this notice is replicated
 * unmodified and in its entirety in all distributions of the software,
 * modified or unmodified, in source code or in binary form.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met: redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer;
 * redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution;
 * neither the name of the copyright holders nor the names of its
 * contributors may be used to endorse or promote products derived from
 * this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Authors: Stan Czerniawski
 */

#include "dev/arm/smmu_v3_transl.hh"

#include "debug/SMMUv3.hh"
#include "debug/SMMUv3Hazard.hh"
#include "dev/arm/amba.hh"
#include "dev/arm/smmu_v3.hh"
#include "sim/system.hh"

SMMUTranslRequest
SMMUTranslRequest::fromPacket(PacketPtr pkt, bool ats)
{
    SMMUTranslRequest req;
    req.addr         = pkt->getAddr();
    req.size         = pkt->getSize();
    req.sid          = pkt->req->streamId();
    req.ssid         = pkt->req->hasSubstreamId() ?
        pkt->req->substreamId() : 0;
    req.isWrite      = pkt->isWrite();
    req.isPrefetch   = false;
    req.isAtsRequest = ats;
    req.pkt          = pkt;

    return req;
}

SMMUTranslRequest
SMMUTranslRequest::prefetch(Addr addr, uint32_t sid, uint32_t ssid)
{
    SMMUTranslRequest req;
    req.addr         = addr;
    req.size         = 0;
    req.sid          = sid;
    req.ssid         = ssid;
    req.isWrite      = false;
    req.isPrefetch   = true;
    req.isAtsRequest = false;
    req.pkt          = NULL;

    return req;
}

void
SMMUTranslationProcess::beginTransaction(const SMMUTranslRequest &req)
{
    request = req;

    reinit();
}

void
SMMUTranslationProcess::resumeTransaction()
{
    assert(smmu.system.isTimingMode());

    assert(!"Stalls are broken");

    Tick resumeTick = curTick();

    (void) resumeTick;
    DPRINTF(SMMUv3, "Resume at tick = %d. Fault duration = %d (%.3fus)\n",
        resumeTick, resumeTick-faultTick, (resumeTick-faultTick) / 1e6);

    beginTransaction(request);

    smmu.runProcessTiming(this, request.pkt);
}

void
SMMUTranslationProcess::main(Yield &yield)
{
    // Hack:
    // The coroutine starts running as soon as it's created.
    // But we need to wait for request data esp. in atomic mode.
    SMMUAction a;
    a.type = ACTION_INITIAL_NOP;
    a.pkt = NULL;
    yield(a);

    const Addr next4k = (request.addr + 0x1000ULL) & ~0xfffULL;

    if ((request.addr + request.size) > next4k)
        panic("Transaction crosses 4k boundary (addr=%#x size=%#x)!\n",
                request.addr, request.size);


    unsigned numSlaveBeats = request.isWrite ?
        (request.size + (ifc.portWidth - 1)) / ifc.portWidth : 1;

    doSemaphoreDown(yield, ifc.slavePortSem);
    doDelay(yield, Cycles(numSlaveBeats));
    doSemaphoreUp(ifc.slavePortSem);


    recvTick = curTick();


    if (!(smmu.regs.cr0 & 0x1)) {
        // SMMU disabled
        doDelay(yield, Cycles(1));
        completeTransaction(yield, bypass(request.addr));
        return;
    }

    TranslResult tr;
    bool wasPrefetched = false;

    if (request.isPrefetch) {
        // Abort prefetch if:
        //   - there's already a transaction looking up the same 4k page, OR
        //   - requested address is already in the TLB.
        if (hazard4kCheck() || ifcTLBLookup(yield, tr, wasPrefetched))
            completePrefetch(yield); // this never returns

        hazard4kRegister();

        tr = smmuTranslation(yield);

        if (tr.fault == FAULT_NONE)
            ifcTLBUpdate(yield, tr);

        hazard4kRelease();

        completePrefetch(yield);
    } else {
        hazardIdRegister();

        if (!microTLBLookup(yield, tr)) {
            bool hit = ifcTLBLookup(yield, tr, wasPrefetched);
            if (!hit) {
                while (!hit && hazard4kCheck()) {
                    hazard4kHold(yield);
                    hit = ifcTLBLookup(yield, tr, wasPrefetched);
                }
            }

            // Issue prefetch if:
            //   - there was a TLB hit and the entry was prefetched, OR
            //   - TLB miss was successfully serviced
            if (hit) {
                if (wasPrefetched)
                    issuePrefetch(next4k);
            } else {
                hazard4kRegister();

                tr = smmuTranslation(yield);

                if (tr.fault == FAULT_NONE) {
                    ifcTLBUpdate(yield, tr);

                    issuePrefetch(next4k);
                }

                hazard4kRelease();
            }

            if (tr.fault == FAULT_NONE)
                microTLBUpdate(yield, tr);
        }

        hazardIdHold(yield);
        hazardIdRelease();

        if (tr.fault != FAULT_NONE)
            panic("fault\n");

        completeTransaction(yield, tr);
    }
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::bypass(Addr addr) const
{
    TranslResult tr;
    tr.fault = FAULT_NONE;
    tr.addr = addr;
    tr.addrMask = 0;
    tr.writable = 1;

    return tr;
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::smmuTranslation(Yield &yield)
{
    TranslResult tr;

    // Need SMMU credit to proceed
    doSemaphoreDown(yield, smmu.transSem);

    // Simulate pipelined IFC->SMMU link
    doSemaphoreDown(yield, smmu.ifcSmmuSem);
    doDelay(yield, Cycles(1)); // serialize transactions
    doSemaphoreUp(smmu.ifcSmmuSem);
    doDelay(yield, smmu.ifcSmmuLat - Cycles(1)); // remaining pipeline delay

    bool haveConfig = true;
    if (!configCacheLookup(yield, context)) {
        if(findConfig(yield, context, tr)) {
            configCacheUpdate(yield, context);
        } else {
            haveConfig = false;
        }
    }

    if (haveConfig && !smmuTLBLookup(yield, tr)) {
        // SMMU main TLB miss

        // Need PTW slot to proceed
        doSemaphoreDown(yield, smmu.ptwSem);

        // Page table walk
        Tick ptwStartTick = curTick();

        if (context.stage1Enable) {
            tr = translateStage1And2(yield, request.addr);
        } else if (context.stage2Enable) {
            tr = translateStage2(yield, request.addr, true);
        } else {
            tr = bypass(request.addr);
        }

        if (context.stage1Enable || context.stage2Enable)
            smmu.ptwTimeDist.sample(curTick() - ptwStartTick);

        // Free PTW slot
        doSemaphoreUp(smmu.ptwSem);

        if (tr.fault == FAULT_NONE)
            smmuTLBUpdate(yield, tr);
    }

    // Simulate pipelined SMMU->SLAVE INTERFACE link
    doSemaphoreDown(yield, smmu.smmuIfcSem);
    doDelay(yield, Cycles(1)); // serialize transactions
    doSemaphoreUp(smmu.smmuIfcSem);
    doDelay(yield, smmu.smmuIfcLat - Cycles(1)); // remaining pipeline delay

    // return SMMU credit
    doSemaphoreUp(smmu.transSem);

    return tr;
}

bool
SMMUTranslationProcess::microTLBLookup(Yield &yield, TranslResult &tr)
{
    if (!ifc.microTLBEnable)
        return false;

    doSemaphoreDown(yield, ifc.microTLBSem);
    doDelay(yield, ifc.microTLBLat);
    const SMMUTLB::Entry *e =
        ifc.microTLB->lookup(request.sid, request.ssid, request.addr);
    doSemaphoreUp(ifc.microTLBSem);

    if (!e) {
        DPRINTF(SMMUv3, "micro TLB miss vaddr=%#x sid=%#x ssid=%#x\n",
            request.addr, request.sid, request.ssid);

        return false;
    }

    DPRINTF(SMMUv3,
        "micro TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x paddr=%#x\n",
        request.addr, e->vaMask, request.sid, request.ssid, e->pa);

    tr.fault = FAULT_NONE;
    tr.addr = e->pa + (request.addr & ~e->vaMask);;
    tr.addrMask = e->vaMask;
    tr.writable = e->permissions;

    return true;
}

bool
SMMUTranslationProcess::ifcTLBLookup(Yield &yield, TranslResult &tr,
                                     bool &wasPrefetched)
{
    if (!ifc.mainTLBEnable)
        return false;

    doSemaphoreDown(yield, ifc.mainTLBSem);
    doDelay(yield, ifc.mainTLBLat);
    const SMMUTLB::Entry *e =
        ifc.mainTLB->lookup(request.sid, request.ssid, request.addr);
    doSemaphoreUp(ifc.mainTLBSem);

    if (!e) {
        DPRINTF(SMMUv3,
                "SLAVE Interface TLB miss vaddr=%#x sid=%#x ssid=%#x\n",
                request.addr, request.sid, request.ssid);

        return false;
    }

    DPRINTF(SMMUv3,
            "SLAVE Interface TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x "
            "paddr=%#x\n", request.addr, e->vaMask, request.sid,
            request.ssid, e->pa);

    tr.fault = FAULT_NONE;
    tr.addr = e->pa + (request.addr & ~e->vaMask);;
    tr.addrMask = e->vaMask;
    tr.writable = e->permissions;
    wasPrefetched = e->prefetched;

    return true;
}

bool
SMMUTranslationProcess::smmuTLBLookup(Yield &yield, TranslResult &tr)
{
    if (!smmu.tlbEnable)
        return false;

    doSemaphoreDown(yield, smmu.tlbSem);
    doDelay(yield, smmu.tlbLat);
    const ARMArchTLB::Entry *e =
        smmu.tlb.lookup(request.addr, context.asid, context.vmid);
    doSemaphoreUp(smmu.tlbSem);

    if (!e) {
        DPRINTF(SMMUv3, "SMMU TLB miss vaddr=%#x asid=%#x vmid=%#x\n",
            request.addr, context.asid, context.vmid);

        return false;
    }

    DPRINTF(SMMUv3,
            "SMMU TLB hit vaddr=%#x amask=%#x asid=%#x vmid=%#x paddr=%#x\n",
            request.addr, e->vaMask, context.asid, context.vmid, e->pa);

    tr.fault = FAULT_NONE;
    tr.addr = e->pa + (request.addr & ~e->vaMask);;
    tr.addrMask = e->vaMask;
    tr.writable = e->permissions;

    return true;
}

void
SMMUTranslationProcess::microTLBUpdate(Yield &yield,
                                       const TranslResult &tr)
{
    assert(tr.fault == FAULT_NONE);

    if (!ifc.microTLBEnable)
        return;

    SMMUTLB::Entry e;
    e.valid = true;
    e.prefetched = false;
    e.sid = request.sid;
    e.ssid = request.ssid;
    e.vaMask = tr.addrMask;
    e.va = request.addr & e.vaMask;
    e.pa = tr.addr & e.vaMask;
    e.permissions = tr.writable;
    e.asid = context.asid;
    e.vmid = context.vmid;

    doSemaphoreDown(yield, ifc.microTLBSem);

    DPRINTF(SMMUv3,
        "micro TLB upd vaddr=%#x amask=%#x paddr=%#x sid=%#x ssid=%#x\n",
        e.va, e.vaMask, e.pa, e.sid, e.ssid);

    ifc.microTLB->store(e, SMMUTLB::ALLOC_ANY_WAY);

    doSemaphoreUp(ifc.microTLBSem);
}

void
SMMUTranslationProcess::ifcTLBUpdate(Yield &yield,
                                     const TranslResult &tr)
{
    assert(tr.fault == FAULT_NONE);

    if (!ifc.mainTLBEnable)
        return;

    SMMUTLB::Entry e;
    e.valid = true;
    e.prefetched = request.isPrefetch;
    e.sid = request.sid;
    e.ssid = request.ssid;
    e.vaMask = tr.addrMask;
    e.va = request.addr & e.vaMask;
    e.pa = tr.addr & e.vaMask;
    e.permissions = tr.writable;
    e.asid = context.asid;
    e.vmid = context.vmid;

    SMMUTLB::AllocPolicy alloc = SMMUTLB::ALLOC_ANY_WAY;
    if (ifc.prefetchEnable && ifc.prefetchReserveLastWay)
        alloc = request.isPrefetch ?
            SMMUTLB::ALLOC_LAST_WAY : SMMUTLB::ALLOC_ANY_BUT_LAST_WAY;

    doSemaphoreDown(yield, ifc.mainTLBSem);

    DPRINTF(SMMUv3,
            "SLAVE Interface upd vaddr=%#x amask=%#x paddr=%#x sid=%#x "
            "ssid=%#x\n", e.va, e.vaMask, e.pa, e.sid, e.ssid);

    ifc.mainTLB->store(e, alloc);

    doSemaphoreUp(ifc.mainTLBSem);
}

void
SMMUTranslationProcess::smmuTLBUpdate(Yield &yield,
                                      const TranslResult &tr)
{
    assert(tr.fault == FAULT_NONE);

    if (!smmu.tlbEnable)
        return;

    ARMArchTLB::Entry e;
    e.valid = true;
    e.vaMask = tr.addrMask;
    e.va = request.addr & e.vaMask;
    e.asid = context.asid;
    e.vmid = context.vmid;
    e.pa = tr.addr & e.vaMask;
    e.permissions = tr.writable;

    doSemaphoreDown(yield, smmu.tlbSem);

    DPRINTF(SMMUv3,
            "SMMU TLB upd vaddr=%#x amask=%#x paddr=%#x asid=%#x vmid=%#x\n",
            e.va, e.vaMask, e.pa, e.asid, e.vmid);

    smmu.tlb.store(e);

    doSemaphoreUp(smmu.tlbSem);
}

bool
SMMUTranslationProcess::configCacheLookup(Yield &yield, TranslContext &tc)
{
    if (!smmu.configCacheEnable)
        return false;

    doSemaphoreDown(yield, smmu.configSem);
    doDelay(yield, smmu.configLat);
    const ConfigCache::Entry *e =
        smmu.configCache.lookup(request.sid, request.ssid);
    doSemaphoreUp(smmu.configSem);

    if (!e) {
        DPRINTF(SMMUv3, "Config miss sid=%#x ssid=%#x\n",
                request.sid, request.ssid);

        return false;
    }

    DPRINTF(SMMUv3, "Config hit sid=%#x ssid=%#x ttb=%#08x asid=%#x\n",
            request.sid, request.ssid, e->ttb0, e->asid);

    tc.stage1Enable = e->stage1_en;
    tc.stage2Enable = e->stage2_en;

    tc.ttb0 = e->ttb0;
    tc.ttb1 = e->ttb1;
    tc.asid = e->asid;
    tc.httb = e->httb;
    tc.vmid = e->vmid;

    tc.stage1TranslGranule = e->stage1_tg;
    tc.stage2TranslGranule = e->stage2_tg;

    return true;
}

void
SMMUTranslationProcess::configCacheUpdate(Yield &yield,
                                          const TranslContext &tc)
{
    if (!smmu.configCacheEnable)
        return;

    ConfigCache::Entry e;
    e.valid = true;
    e.sid = request.sid;
    e.ssid = request.ssid;
    e.stage1_en = tc.stage1Enable;
    e.stage2_en = tc.stage2Enable;
    e.ttb0 = tc.ttb0;
    e.ttb1 = tc.ttb1;
    e.asid = tc.asid;
    e.httb = tc.httb;
    e.vmid = tc.vmid;
    e.stage1_tg = tc.stage1TranslGranule;
    e.stage2_tg = tc.stage2TranslGranule;

    doSemaphoreDown(yield, smmu.configSem);

    DPRINTF(SMMUv3, "Config upd  sid=%#x ssid=%#x\n", e.sid, e.ssid);

    smmu.configCache.store(e);

    doSemaphoreUp(smmu.configSem);
}

bool
SMMUTranslationProcess::findConfig(Yield &yield,
                                   TranslContext &tc,
                                   TranslResult &tr)
{
    tc.stage1Enable = false;
    tc.stage2Enable = false;

    StreamTableEntry ste;
    doReadSTE(yield, ste, request.sid);

    switch (ste.dw0.config) {
        case STE_CONFIG_BYPASS:
            break;

        case STE_CONFIG_STAGE1_ONLY:
            tc.stage1Enable = true;
            break;

        case STE_CONFIG_STAGE2_ONLY:
            tc.stage2Enable = true;
            break;

        case STE_CONFIG_STAGE1_AND_2:
            tc.stage1Enable = true;
            tc.stage2Enable = true;
            break;

        default:
            panic("Bad or unimplemented STE config %d\n",
                ste.dw0.config);
    }


    // Establish stage 2 context first since
    // Context Descriptors can be in IPA space.
    if (tc.stage2Enable) {
        tc.httb = ste.dw3.s2ttb << STE_S2TTB_SHIFT;
        tc.vmid = ste.dw2.s2vmid;
        tc.stage2TranslGranule = ste.dw2.s2tg;
    } else {
        tc.httb = 0xdeadbeef;
        tc.vmid = 0;
        tc.stage2TranslGranule = TRANS_GRANULE_INVALID;
    }


    // Now fetch stage 1 config.
    if (context.stage1Enable) {
        ContextDescriptor cd;
        doReadCD(yield, cd, ste, request.sid, request.ssid);

        tc.ttb0 = cd.dw1.ttb0 << CD_TTB_SHIFT;
        tc.ttb1 = cd.dw2.ttb1 << CD_TTB_SHIFT;
        tc.asid = cd.dw0.asid;
        tc.stage1TranslGranule = cd.dw0.tg0;
    } else {
        tc.ttb0 = 0xcafebabe;
        tc.ttb1 = 0xcafed00d;
        tc.asid = 0;
        tc.stage1TranslGranule = TRANS_GRANULE_INVALID;
    }

    return true;
}

void
SMMUTranslationProcess::walkCacheLookup(
        Yield &yield,
        const WalkCache::Entry *&walkEntry,
        Addr addr, uint16_t asid, uint16_t vmid,
        unsigned stage, unsigned level)
{
    const char *indent = stage==2 ? "  " : "";
    (void) indent; // this is only used in DPRINTFs

    const PageTableOps *pt_ops =
        stage == 1 ?
            smmu.getPageTableOps(context.stage1TranslGranule) :
            smmu.getPageTableOps(context.stage2TranslGranule);

    unsigned walkCacheLevels =
        smmu.walkCacheEnable ?
            (stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels) :
            0;

    if ((1 << level) & walkCacheLevels) {
        doSemaphoreDown(yield, smmu.walkSem);
        doDelay(yield, smmu.walkLat);

        walkEntry = smmu.walkCache.lookup(addr, pt_ops->walkMask(level),
                                          asid, vmid, stage, level);

        if (walkEntry) {
            DPRINTF(SMMUv3, "%sWalkCache hit  va=%#x asid=%#x vmid=%#x "
                            "base=%#x (S%d, L%d)\n",
                    indent, addr, asid, vmid, walkEntry->pa, stage, level);
        } else {
            DPRINTF(SMMUv3, "%sWalkCache miss va=%#x asid=%#x vmid=%#x "
                            "(S%d, L%d)\n",
                    indent, addr, asid, vmid, stage, level);
        }

        doSemaphoreUp(smmu.walkSem);
    }
}

void
SMMUTranslationProcess::walkCacheUpdate(Yield &yield, Addr va,
                                        Addr vaMask, Addr pa,
                                        unsigned stage, unsigned level,
                                        bool leaf, uint8_t permissions)
{
    unsigned walkCacheLevels =
        stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels;

    if (smmu.walkCacheEnable && ((1<<level) & walkCacheLevels)) {
        WalkCache::Entry e;
        e.valid = true;
        e.va = va;
        e.vaMask = vaMask;
        e.asid = stage==1 ? context.asid : 0;
        e.vmid = context.vmid;
        e.stage = stage;
        e.level = level;
        e.leaf = leaf;
        e.pa = pa;
        e.permissions = permissions;

        doSemaphoreDown(yield, smmu.walkSem);

        DPRINTF(SMMUv3, "%sWalkCache upd  va=%#x mask=%#x asid=%#x vmid=%#x "
                        "tpa=%#x leaf=%s (S%d, L%d)\n",
                e.stage==2 ? "  " : "",
                e.va, e.vaMask, e.asid, e.vmid,
                e.pa, e.leaf, e.stage, e.level);

        smmu.walkCache.store(e);

        doSemaphoreUp(smmu.walkSem);
    }
}

/*
 * Please note:
 * This does not deal with the case where stage 1 page size
 * is larger than stage 2 page size.
 */
SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr,
                                       const PageTableOps *pt_ops,
                                       unsigned level, Addr walkPtr)
{
    PageTableOps::pte_t pte = 0;

    doSemaphoreDown(yield, smmu.cycleSem);
    doDelay(yield, Cycles(1));
    doSemaphoreUp(smmu.cycleSem);

    for (; level <= pt_ops->lastLevel(); level++) {
        Addr pte_addr = walkPtr + pt_ops->index(addr, level);

        DPRINTF(SMMUv3, "Fetching S1 L%d PTE from pa=%#08x\n",
                level, pte_addr);

        doReadPTE(yield, addr, pte_addr, &pte, 1, level);

        DPRINTF(SMMUv3, "Got S1 L%d PTE=%#x from pa=%#08x\n",
                level, pte, pte_addr);

        doSemaphoreDown(yield, smmu.cycleSem);
        doDelay(yield, Cycles(1));
        doSemaphoreUp(smmu.cycleSem);

        bool valid = pt_ops->isValid(pte, level);
        bool leaf  = pt_ops->isLeaf(pte, level);

        if (!valid) {
            DPRINTF(SMMUv3, "S1 PTE not valid - fault\n");

            TranslResult tr;
            tr.fault = FAULT_TRANSLATION;
            return tr;
        }

        if (valid && leaf && request.isWrite &&
            !pt_ops->isWritable(pte, level, false))
        {
            DPRINTF(SMMUv3, "S1 page not writable - fault\n");

            TranslResult tr;
            tr.fault = FAULT_PERMISSION;
            return tr;
        }

        walkPtr = pt_ops->nextLevelPointer(pte, level);

        if (leaf)
            break;

        if (context.stage2Enable) {
            TranslResult s2tr = translateStage2(yield, walkPtr, false);
            if (s2tr.fault != FAULT_NONE)
                return s2tr;

            walkPtr = s2tr.addr;
        }

        walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr,
                        1, level, leaf, 0);
    }

    TranslResult tr;
    tr.fault    = FAULT_NONE;
    tr.addrMask = pt_ops->pageMask(pte, level);
    tr.addr     = walkPtr + (addr & ~tr.addrMask);
    tr.writable = pt_ops->isWritable(pte, level, false);

    if (context.stage2Enable) {
        TranslResult s2tr = translateStage2(yield, tr.addr, true);
        if (s2tr.fault != FAULT_NONE)
            return s2tr;

        tr = combineTranslations(tr, s2tr);
    }

    walkCacheUpdate(yield, addr, tr.addrMask, tr.addr,
                    1, level, true, tr.writable);

    return tr;
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr,
                                   const PageTableOps *pt_ops,
                                   unsigned level, Addr walkPtr)
{
    PageTableOps::pte_t pte;

    doSemaphoreDown(yield, smmu.cycleSem);
    doDelay(yield, Cycles(1));
    doSemaphoreUp(smmu.cycleSem);

    for (; level <= pt_ops->lastLevel(); level++) {
        Addr pte_addr = walkPtr + pt_ops->index(addr, level);

        DPRINTF(SMMUv3, "  Fetching S2 L%d PTE from pa=%#08x\n",
                level, pte_addr);

        doReadPTE(yield, addr, pte_addr, &pte, 2, level);

        DPRINTF(SMMUv3, "  Got S2 L%d PTE=%#x from pa=%#08x\n",
                level, pte, pte_addr);

        doSemaphoreDown(yield, smmu.cycleSem);
        doDelay(yield, Cycles(1));
        doSemaphoreUp(smmu.cycleSem);

        bool valid = pt_ops->isValid(pte, level);
        bool leaf  = pt_ops->isLeaf(pte, level);

        if (!valid) {
            DPRINTF(SMMUv3, "  S2 PTE not valid - fault\n");

            TranslResult tr;
            tr.fault = FAULT_TRANSLATION;
            return tr;
        }

        if (valid && leaf && request.isWrite &&
            !pt_ops->isWritable(pte, level, true))
        {
            DPRINTF(SMMUv3, "  S2 PTE not writable = fault\n");

            TranslResult tr;
            tr.fault = FAULT_PERMISSION;
            return tr;
        }

        walkPtr = pt_ops->nextLevelPointer(pte, level);

        if (final_tr || smmu.walkCacheNonfinalEnable)
            walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr,
                            2, level, leaf,
                            leaf ? pt_ops->isWritable(pte, level, true) : 0);
        if (leaf)
            break;
    }

    TranslResult tr;
    tr.fault    = FAULT_NONE;
    tr.addrMask = pt_ops->pageMask(pte, level);
    tr.addr     = walkPtr + (addr & ~tr.addrMask);
    tr.writable = pt_ops->isWritable(pte, level, true);

    return tr;
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr)
{
    const PageTableOps *pt_ops =
        smmu.getPageTableOps(context.stage1TranslGranule);

    const WalkCache::Entry *walk_ep = NULL;
    unsigned level;

    // Level here is actually (level+1) so we can count down
    // to 0 using unsigned int.
    for (level = pt_ops->lastLevel() + 1;
        level > pt_ops->firstLevel();
        level--)
    {
        walkCacheLookup(yield, walk_ep, addr,
                        context.asid, context.vmid, 1, level-1);

        if (walk_ep)
            break;
    }

    // Correct level (see above).
    level -= 1;

    TranslResult tr;
    if (walk_ep) {
        if (walk_ep->leaf) {
            tr.fault    = FAULT_NONE;
            tr.addr     = walk_ep->pa + (addr & ~walk_ep->vaMask);
            tr.addrMask = walk_ep->vaMask;
            tr.writable = walk_ep->permissions;
        } else {
            tr = walkStage1And2(yield, addr, pt_ops, level+1, walk_ep->pa);
        }
    } else {
        Addr table_addr = context.ttb0;
        if (context.stage2Enable) {
            TranslResult s2tr = translateStage2(yield, table_addr, false);
            if (s2tr.fault != FAULT_NONE)
                return s2tr;

            table_addr = s2tr.addr;
        }

        tr = walkStage1And2(yield, addr, pt_ops, pt_ops->firstLevel(),
                            table_addr);
    }

    if (tr.fault == FAULT_NONE)
        DPRINTF(SMMUv3, "Translated vaddr %#x to paddr %#x\n", addr, tr.addr);

    return tr;
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr)
{
    const PageTableOps *pt_ops =
            smmu.getPageTableOps(context.stage2TranslGranule);

    const IPACache::Entry *ipa_ep = NULL;
    if (smmu.ipaCacheEnable) {
        doSemaphoreDown(yield, smmu.ipaSem);
        doDelay(yield, smmu.ipaLat);
        ipa_ep = smmu.ipaCache.lookup(addr, context.vmid);
        doSemaphoreUp(smmu.ipaSem);
    }

    if (ipa_ep) {
        TranslResult tr;
        tr.fault    = FAULT_NONE;
        tr.addr     = ipa_ep->pa + (addr & ~ipa_ep->ipaMask);
        tr.addrMask = ipa_ep->ipaMask;
        tr.writable = ipa_ep->permissions;

        DPRINTF(SMMUv3, "  IPACache hit  ipa=%#x vmid=%#x pa=%#x\n",
            addr, context.vmid, tr.addr);

        return tr;
    } else if (smmu.ipaCacheEnable) {
        DPRINTF(SMMUv3, "  IPACache miss ipa=%#x vmid=%#x\n",
                addr, context.vmid);
    }

    const WalkCache::Entry *walk_ep = NULL;
    unsigned level = pt_ops->firstLevel();

    if (final_tr || smmu.walkCacheNonfinalEnable) {
        // Level here is actually (level+1) so we can count down
        // to 0 using unsigned int.
        for (level = pt_ops->lastLevel() + 1;
            level > pt_ops->firstLevel();
            level--)
        {
            walkCacheLookup(yield, walk_ep, addr,
                            0, context.vmid, 2, level-1);

            if (walk_ep)
                break;
        }

        // Correct level (see above).
        level -= 1;
    }

    TranslResult tr;
    if (walk_ep) {
        if (walk_ep->leaf) {
            tr.fault    = FAULT_NONE;
            tr.addr     = walk_ep->pa + (addr & ~walk_ep->vaMask);
            tr.addrMask = walk_ep->vaMask;
            tr.writable = walk_ep->permissions;
        } else {
            tr = walkStage2(yield, addr, final_tr, pt_ops,
                            level + 1, walk_ep->pa);
        }
    } else {
        tr = walkStage2(yield, addr, final_tr, pt_ops, pt_ops->firstLevel(),
                        context.httb);
    }

    if (tr.fault == FAULT_NONE)
        DPRINTF(SMMUv3, "  Translated %saddr %#x to paddr %#x\n",
            context.stage1Enable ? "ip" : "v", addr, tr.addr);

    if (smmu.ipaCacheEnable) {
        IPACache::Entry e;
        e.valid = true;
        e.ipaMask = tr.addrMask;
        e.ipa = addr & e.ipaMask;
        e.pa = tr.addr & tr.addrMask;
        e.permissions = tr.writable;
        e.vmid = context.vmid;

        doSemaphoreDown(yield, smmu.ipaSem);
        smmu.ipaCache.store(e);
        doSemaphoreUp(smmu.ipaSem);
    }

    return tr;
}

SMMUTranslationProcess::TranslResult
SMMUTranslationProcess::combineTranslations(const TranslResult &s1tr,
                                            const TranslResult &s2tr) const
{
    if (s2tr.fault != FAULT_NONE)
        return s2tr;

    assert(s1tr.fault == FAULT_NONE);

    TranslResult tr;
    tr.fault    = FAULT_NONE;
    tr.addr     = s2tr.addr;
    tr.addrMask = s1tr.addrMask | s2tr.addrMask;
    tr.writable = s1tr.writable & s2tr.writable;

    return tr;
}

bool
SMMUTranslationProcess::hazard4kCheck()
{
    Addr addr4k = request.addr & ~0xfffULL;

    for (auto it = ifc.duplicateReqs.begin();
         it != ifc.duplicateReqs.end();
         ++it)
    {
        Addr other4k = (*it)->request.addr & ~0xfffULL;
        if (addr4k == other4k)
            return true;
    }

    return false;
}

void
SMMUTranslationProcess::hazard4kRegister()
{
    DPRINTF(SMMUv3Hazard, "4kReg:  p=%p a4k=%#x\n",
            this, request.addr & ~0xfffULL);

    ifc.duplicateReqs.push_back(this);
}

void
SMMUTranslationProcess::hazard4kHold(Yield &yield)
{
    Addr addr4k = request.addr & ~0xfffULL;

    bool found_hazard;

    do {
        found_hazard = false;

        for (auto it = ifc.duplicateReqs.begin();
             it!=ifc.duplicateReqs.end() && *it!=this;
             ++it)
        {
            Addr other4k = (*it)->request.addr & ~0xfffULL;

            DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x Q: p=%p a4k=%#x\n",
                    this, addr4k, *it, other4k);

            if (addr4k == other4k) {
                DPRINTF(SMMUv3Hazard,
                        "4kHold: p=%p a4k=%#x WAIT on p=%p a4k=%#x\n",
                        this, addr4k, *it, other4k);

                doWaitForSignal(yield, ifc.duplicateReqRemoved);

                DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x RESUME\n",
                        this, addr4k);

                // This is to avoid checking *it!=this after doWaitForSignal()
                // since it could have been deleted.
                found_hazard = true;
                break;
            }
        }
    } while (found_hazard);
}

void
SMMUTranslationProcess::hazard4kRelease()
{
    DPRINTF(SMMUv3Hazard, "4kRel:  p=%p a4k=%#x\n",
            this, request.addr & ~0xfffULL);

    std::list<SMMUTranslationProcess *>::iterator it;

    for (it = ifc.duplicateReqs.begin(); it != ifc.duplicateReqs.end(); ++it)
        if (*it == this)
            break;

    if (it == ifc.duplicateReqs.end())
        panic("hazard4kRelease: request not found");

    ifc.duplicateReqs.erase(it);

    doBroadcastSignal(ifc.duplicateReqRemoved);
}

void
SMMUTranslationProcess::hazardIdRegister()
{
    auto orderId = AMBA::orderId(request.pkt);

    DPRINTF(SMMUv3Hazard, "IdReg:  p=%p oid=%d\n", this, orderId);

    assert(orderId < SMMU_MAX_TRANS_ID);

    std::list<SMMUTranslationProcess *> &depReqs =
        request.isWrite ?
            ifc.dependentWrites[orderId] : ifc.dependentReads[orderId];
    depReqs.push_back(this);
}

void
SMMUTranslationProcess::hazardIdHold(Yield &yield)
{
    auto orderId = AMBA::orderId(request.pkt);

    DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d\n", this, orderId);

    std::list<SMMUTranslationProcess *> &depReqs =
        request.isWrite ?
            ifc.dependentWrites[orderId] : ifc.dependentReads[orderId];
    std::list<SMMUTranslationProcess *>::iterator it;

    bool found_hazard;

    do {
        found_hazard = false;

        for (auto it = depReqs.begin(); it!=depReqs.end() && *it!=this; ++it) {
            DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d Q: %p\n",
                    this, orderId, *it);

            if (AMBA::orderId((*it)->request.pkt) == orderId) {
                DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d WAIT on=%p\n",
                        this, orderId, *it);

                doWaitForSignal(yield, ifc.dependentReqRemoved);

                DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d RESUME\n",
                        this, orderId);

                // This is to avoid checking *it!=this after doWaitForSignal()
                // since it could have been deleted.
                found_hazard = true;
                break;
            }
        }
    } while (found_hazard);
}

void
SMMUTranslationProcess::hazardIdRelease()
{
    auto orderId = AMBA::orderId(request.pkt);

    DPRINTF(SMMUv3Hazard, "IdRel:  p=%p oid=%d\n", this, orderId);

    std::list<SMMUTranslationProcess *> &depReqs =
        request.isWrite ?
            ifc.dependentWrites[orderId] : ifc.dependentReads[orderId];
    std::list<SMMUTranslationProcess *>::iterator it;

    for (it = depReqs.begin(); it != depReqs.end(); ++it) {
        if (*it == this)
            break;
    }

    if (it == depReqs.end())
        panic("hazardIdRelease: request not found");

    depReqs.erase(it);

    doBroadcastSignal(ifc.dependentReqRemoved);
}

void
SMMUTranslationProcess::issuePrefetch(Addr addr)
{
    if (!smmu.system.isTimingMode())
        return;

    if (!ifc.prefetchEnable || ifc.xlateSlotsRemaining == 0)
        return;

    ifc.xlateSlotsRemaining--;

    std::string proc_name = csprintf("%sprf", name());
    SMMUTranslationProcess *proc =
        new SMMUTranslationProcess(proc_name, smmu, ifc);

    proc->beginTransaction(
            SMMUTranslRequest::prefetch(addr, request.sid, request.ssid));
    proc->scheduleWakeup(smmu.clockEdge(Cycles(1)));
}

void
SMMUTranslationProcess::completeTransaction(Yield &yield,
                                            const TranslResult &tr)
{
    assert(tr.fault == FAULT_NONE);

    unsigned numMasterBeats = request.isWrite ?
        (request.size + (smmu.masterPortWidth-1))
            / smmu.masterPortWidth :
        1;

    doSemaphoreDown(yield, smmu.masterPortSem);
    doDelay(yield, Cycles(numMasterBeats));
    doSemaphoreUp(smmu.masterPortSem);


    smmu.translationTimeDist.sample(curTick() - recvTick);
    ifc.xlateSlotsRemaining++;
    if (!request.isAtsRequest && request.isWrite)
        ifc.wrBufSlotsRemaining +=
            (request.size + (ifc.portWidth-1)) / ifc.portWidth;

    smmu.scheduleSlaveRetries();


    SMMUAction a;

    if (request.isAtsRequest) {
        a.type = ACTION_SEND_RESP_ATS;

        if (smmu.system.isAtomicMode()) {
            request.pkt->makeAtomicResponse();
        } else if (smmu.system.isTimingMode()) {
            request.pkt->makeTimingResponse();
        } else {
            panic("Not in atomic or timing mode");
        }
    } else {
        a.type = ACTION_SEND_REQ_FINAL;
        a.ifc = &ifc;
    }

    a.pkt = request.pkt;
    a.delay = 0;

    a.pkt->setAddr(tr.addr);
    a.pkt->req->setPaddr(tr.addr);

    yield(a);

    if (!request.isAtsRequest) {
        PacketPtr pkt = yield.get();
        pkt->setAddr(request.addr);

        a.type = ACTION_SEND_RESP;
        a.pkt = pkt;
        a.ifc = &ifc;
        a.delay = 0;
        yield(a);
    }
}

void
SMMUTranslationProcess::completePrefetch(Yield &yield)
{
    ifc.xlateSlotsRemaining++;

    SMMUAction a;
    a.type = ACTION_TERMINATE;
    a.pkt = NULL;
    a.ifc = &ifc;
    a.delay = 0;
    yield(a);
}

void
SMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev)
{
    int sizeMask = mask(smmu.regs.eventq_base & Q_BASE_SIZE_MASK) &
            Q_CONS_PROD_MASK;

    if (((smmu.regs.eventq_prod+1) & sizeMask) ==
            (smmu.regs.eventq_cons & sizeMask))
        panic("Event queue full - aborting\n");

    Addr event_addr =
        (smmu.regs.eventq_base & Q_BASE_ADDR_MASK) +
        (smmu.regs.eventq_prod & sizeMask) * sizeof(ev);

    DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): type=%#x stag=%#x "
        "flags=%#x sid=%#x ssid=%#x va=%#08x ipa=%#x\n",
        event_addr, smmu.regs.eventq_prod, ev.type, ev.stag,
        ev.flags, ev.streamId, ev.substreamId, ev.va, ev.ipa);

    // This deliberately resets the overflow field in eventq_prod!
    smmu.regs.eventq_prod = (smmu.regs.eventq_prod + 1) & sizeMask;

    doWrite(yield, event_addr, &ev, sizeof(ev));

    if (!(smmu.regs.eventq_irq_cfg0 & E_BASE_ENABLE_MASK))
        panic("eventq msi not enabled\n");

    doWrite(yield, smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK,
            &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1));
}

void
SMMUTranslationProcess::doReadSTE(Yield &yield,
                                  StreamTableEntry &ste,
                                  uint32_t sid)
{
    unsigned max_sid = 1 << (smmu.regs.strtab_base_cfg & ST_CFG_SIZE_MASK);
    if (sid >= max_sid)
        panic("SID %#x out of range, max=%#x", sid, max_sid);

    Addr ste_addr;

    if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_2LEVEL) {
        unsigned split =
            (smmu.regs.strtab_base_cfg & ST_CFG_SPLIT_MASK) >> ST_CFG_SPLIT_SHIFT;

        if (split!= 7 && split!=8 && split!=16)
            panic("Invalid stream table split %d", split);

        uint64_t l2_ptr;
        uint64_t l2_addr =
            (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) +
            bits(sid, 32, split) * sizeof(l2_ptr);

        DPRINTF(SMMUv3, "Read L1STE at %#x\n", l2_addr);

        doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, 0);

        DPRINTF(SMMUv3, "Got L1STE L1 at %#x: 0x%016x\n", l2_addr, l2_ptr);

        unsigned span = l2_ptr & ST_L2_SPAN_MASK;
        if (span == 0)
            panic("Invalid level 1 stream table descriptor");

        unsigned index = bits(sid, split-1, 0);
        if (index >= (1 << span))
            panic("StreamID %d out of level 1 descriptor range %d",
                  sid, 1<<span);

        ste_addr = (l2_ptr & ST_L2_ADDR_MASK) + index * sizeof(ste);

        smmu.steL1Fetches++;
    } else if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_LINEAR) {
        ste_addr =
            (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + sid * sizeof(ste);
    } else {
        panic("Invalid stream table format");
    }

    DPRINTF(SMMUv3, "Read STE at %#x\n", ste_addr);

    doReadConfig(yield, ste_addr, &ste, sizeof(ste), sid, 0);

    DPRINTF(SMMUv3, "Got STE at %#x [0]: 0x%016x\n", ste_addr, ste.dw0);
    DPRINTF(SMMUv3, "    STE at %#x [1]: 0x%016x\n", ste_addr, ste.dw1);
    DPRINTF(SMMUv3, "    STE at %#x [2]: 0x%016x\n", ste_addr, ste.dw2);
    DPRINTF(SMMUv3, "    STE at %#x [3]: 0x%016x\n", ste_addr, ste.dw3);
    DPRINTF(SMMUv3, "    STE at %#x [4]: 0x%016x\n", ste_addr, ste._pad[0]);
    DPRINTF(SMMUv3, "    STE at %#x [5]: 0x%016x\n", ste_addr, ste._pad[1]);
    DPRINTF(SMMUv3, "    STE at %#x [6]: 0x%016x\n", ste_addr, ste._pad[2]);
    DPRINTF(SMMUv3, "    STE at %#x [7]: 0x%016x\n", ste_addr, ste._pad[3]);

    if (!ste.dw0.valid)
        panic("STE @ %#x not valid\n", ste_addr);

    smmu.steFetches++;
}

void
SMMUTranslationProcess::doReadCD(Yield &yield,
                                 ContextDescriptor &cd,
                                 const StreamTableEntry &ste,
                                 uint32_t sid, uint32_t ssid)
{
    Addr cd_addr;

    if (ste.dw0.s1cdmax == 0) {
        cd_addr = ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT;
    } else {
        unsigned max_ssid = 1 << ste.dw0.s1cdmax;
        if (ssid >= max_ssid)
            panic("SSID %#x out of range, max=%#x", ssid, max_ssid);

        if (ste.dw0.s1fmt==STAGE1_CFG_2L_4K ||
            ste.dw0.s1fmt==STAGE1_CFG_2L_64K)
        {
            unsigned split = ste.dw0.s1fmt==STAGE1_CFG_2L_4K ? 7 : 11;

            uint64_t l2_ptr;
            uint64_t l2_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) +
                bits(ssid, 24, split) * sizeof(l2_ptr);

            if (context.stage2Enable)
                l2_addr = translateStage2(yield, l2_addr, false).addr;

            DPRINTF(SMMUv3, "Read L1CD at %#x\n", l2_addr);

            doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, ssid);

            DPRINTF(SMMUv3, "Got L1CD at %#x: 0x%016x\n", l2_addr, l2_ptr);

            cd_addr = l2_ptr + bits(ssid, split-1, 0) * sizeof(cd);

            smmu.cdL1Fetches++;
        } else if (ste.dw0.s1fmt == STAGE1_CFG_1L) {
            cd_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + ssid*sizeof(cd);
        }
    }

    if (context.stage2Enable)
        cd_addr = translateStage2(yield, cd_addr, false).addr;

    DPRINTF(SMMUv3, "Read CD at %#x\n", cd_addr);

    doReadConfig(yield, cd_addr, &cd, sizeof(cd), sid, ssid);

    DPRINTF(SMMUv3, "Got CD at %#x [0]: 0x%016x\n", cd_addr, cd.dw0);
    DPRINTF(SMMUv3, "    CD at %#x [1]: 0x%016x\n", cd_addr, cd.dw1);
    DPRINTF(SMMUv3, "    CD at %#x [2]: 0x%016x\n", cd_addr, cd.dw2);
    DPRINTF(SMMUv3, "    CD at %#x [3]: 0x%016x\n", cd_addr, cd.mair);
    DPRINTF(SMMUv3, "    CD at %#x [4]: 0x%016x\n", cd_addr, cd.amair);
    DPRINTF(SMMUv3, "    CD at %#x [5]: 0x%016x\n", cd_addr, cd._pad[0]);
    DPRINTF(SMMUv3, "    CD at %#x [6]: 0x%016x\n", cd_addr, cd._pad[1]);
    DPRINTF(SMMUv3, "    CD at %#x [7]: 0x%016x\n", cd_addr, cd._pad[2]);


    if (!cd.dw0.valid)
        panic("CD @ %#x not valid\n", cd_addr);

    smmu.cdFetches++;
}

void
SMMUTranslationProcess::doReadConfig(Yield &yield, Addr addr,
                                     void *ptr, size_t size,
                                     uint32_t sid, uint32_t ssid)
{
    doRead(yield, addr, ptr, size);
}

void
SMMUTranslationProcess::doReadPTE(Yield &yield, Addr va, Addr addr,
                                  void *ptr, unsigned stage,
                                  unsigned level)
{
    size_t pte_size = sizeof(PageTableOps::pte_t);

    Addr mask = pte_size - 1;
    Addr base = addr & ~mask;

    doRead(yield, base, ptr, pte_size);
}