smmu_v3_transl.cc revision 14101
12SN/A/* 29448SAndreas.Sandberg@ARM.com * Copyright (c) 2013, 2018-2019 ARM Limited 39920Syasuko.eckert@amd.com * All rights reserved 47338SAli.Saidi@ARM.com * 57338SAli.Saidi@ARM.com * The license below extends only to copyright in the software and shall 67338SAli.Saidi@ARM.com * not be construed as granting a license to any other intellectual 77338SAli.Saidi@ARM.com * property including but not limited to intellectual property relating 87338SAli.Saidi@ARM.com * to a hardware implementation of the functionality of the software 97338SAli.Saidi@ARM.com * licensed hereunder. You may use the software subject to the license 107338SAli.Saidi@ARM.com * terms below provided that you ensure that this notice is replicated 117338SAli.Saidi@ARM.com * unmodified and in its entirety in all distributions of the software, 127338SAli.Saidi@ARM.com * modified or unmodified, in source code or in binary form. 137338SAli.Saidi@ARM.com * 147338SAli.Saidi@ARM.com * Redistribution and use in source and binary forms, with or without 151762SN/A * modification, are permitted provided that the following conditions are 162SN/A * met: redistributions of source code must retain the above copyright 172SN/A * notice, this list of conditions and the following disclaimer; 182SN/A * redistributions in binary form must reproduce the above copyright 192SN/A * notice, this list of conditions and the following disclaimer in the 202SN/A * documentation and/or other materials provided with the distribution; 212SN/A * neither the name of the copyright holders nor the names of its 222SN/A * contributors may be used to endorse or promote products derived from 232SN/A * this software without specific prior written permission. 242SN/A * 252SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 262SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 272SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 282SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 292SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 302SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 312SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 322SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 332SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 342SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 352SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 362SN/A * 372SN/A * Authors: Stan Czerniawski 382SN/A */ 392SN/A 402665Ssaidi@eecs.umich.edu#include "dev/arm/smmu_v3_transl.hh" 412665Ssaidi@eecs.umich.edu 422SN/A#include "debug/SMMUv3.hh" 432SN/A#include "debug/SMMUv3Hazard.hh" 448779Sgblack@eecs.umich.edu#include "dev/arm/amba.hh" 458779Sgblack@eecs.umich.edu#include "dev/arm/smmu_v3.hh" 468779Sgblack@eecs.umich.edu#include "sim/system.hh" 472439SN/A 488779Sgblack@eecs.umich.eduSMMUTranslRequest 498229Snate@binkert.orgSMMUTranslRequest::fromPacket(PacketPtr pkt, bool ats) 506216Snate@binkert.org{ 51146SN/A SMMUTranslRequest req; 52146SN/A req.addr = pkt->getAddr(); 53146SN/A req.size = pkt->getSize(); 54146SN/A req.sid = pkt->req->streamId(); 55146SN/A req.ssid = pkt->req->hasSubstreamId() ? 566216Snate@binkert.org pkt->req->substreamId() : 0; 576658Snate@binkert.org req.isWrite = pkt->isWrite(); 588229Snate@binkert.org req.isPrefetch = false; 591717SN/A req.isAtsRequest = ats; 608887Sgeoffrey.blake@arm.com req.pkt = pkt; 618887Sgeoffrey.blake@arm.com 62146SN/A return req; 6310061Sandreas@sandberg.pp.se} 641977SN/A 652683Sktlim@umich.eduSMMUTranslRequest 661717SN/ASMMUTranslRequest::prefetch(Addr addr, uint32_t sid, uint32_t ssid) 67146SN/A{ 682683Sktlim@umich.edu SMMUTranslRequest req; 698232Snate@binkert.org req.addr = addr; 708232Snate@binkert.org req.size = 0; 718232Snate@binkert.org req.sid = sid; 728779Sgblack@eecs.umich.edu req.ssid = ssid; 733348Sbinkertn@umich.edu req.isWrite = false; 746105Ssteve.reinhardt@amd.com req.isPrefetch = true; 756216Snate@binkert.org req.isAtsRequest = false; 762036SN/A req.pkt = NULL; 77146SN/A 788817Sgblack@eecs.umich.edu return req; 798793Sgblack@eecs.umich.edu} 8056SN/A 8156SN/ASMMUTranslationProcess::SMMUTranslationProcess(const std::string &name, 82695SN/A SMMUv3 &_smmu, SMMUv3SlaveInterface &_ifc) 832901Ssaidi@eecs.umich.edu : 842SN/A SMMUProcess(name, _smmu), 852SN/A ifc(_ifc) 862449SN/A{ 871355SN/A // Decrease number of pending translation slots on the slave interface 885529Snate@binkert.org assert(ifc.xlateSlotsRemaining > 0); 8910061Sandreas@sandberg.pp.se ifc.xlateSlotsRemaining--; 9010061Sandreas@sandberg.pp.se reinit(); 9110061Sandreas@sandberg.pp.se} 92224SN/A 938793Sgblack@eecs.umich.eduSMMUTranslationProcess::~SMMUTranslationProcess() 949384SAndreas.Sandberg@arm.com{ 959384SAndreas.Sandberg@arm.com // Increase number of pending translation slots on the slave interface 968793Sgblack@eecs.umich.edu ifc.xlateSlotsRemaining++; 978820Sgblack@eecs.umich.edu // If no more SMMU translations are pending (all slots available), 989384SAndreas.Sandberg@arm.com // signal SMMU Slave Interface as drained 992SN/A if (ifc.xlateSlotsRemaining == ifc.params()->xlate_slots) { 1006029Ssteve.reinhardt@amd.com ifc.signalDrainDone(); 1012672Sktlim@umich.edu } 1022683Sktlim@umich.edu} 1032SN/A 1048733Sgeoffrey.blake@arm.comvoid 1058733Sgeoffrey.blake@arm.comSMMUTranslationProcess::beginTransaction(const SMMUTranslRequest &req) 1068733Sgeoffrey.blake@arm.com{ 1078733Sgeoffrey.blake@arm.com request = req; 1088733Sgeoffrey.blake@arm.com 1098733Sgeoffrey.blake@arm.com reinit(); 1108733Sgeoffrey.blake@arm.com} 1118733Sgeoffrey.blake@arm.com 1128733Sgeoffrey.blake@arm.comvoid 1138733Sgeoffrey.blake@arm.comSMMUTranslationProcess::resumeTransaction() 1148733Sgeoffrey.blake@arm.com{ 1152SN/A assert(smmu.system.isTimingMode()); 116334SN/A 1178834Satgutier@umich.edu assert(!"Stalls are broken"); 1188834Satgutier@umich.edu 119140SN/A Tick resumeTick = curTick(); 120334SN/A 1212SN/A (void) resumeTick; 1222SN/A DPRINTF(SMMUv3, "Resume at tick = %d. Fault duration = %d (%.3fus)\n", 1232SN/A resumeTick, resumeTick-faultTick, (resumeTick-faultTick) / 1e6); 1242680Sktlim@umich.edu 1254377Sgblack@eecs.umich.edu beginTransaction(request); 1265169Ssaidi@eecs.umich.edu 1274377Sgblack@eecs.umich.edu smmu.runProcessTiming(this, request.pkt); 1284377Sgblack@eecs.umich.edu} 1292SN/A 1302SN/Avoid 1312623SN/ASMMUTranslationProcess::main(Yield &yield) 1322SN/A{ 1332SN/A // Hack: 1342SN/A // The coroutine starts running as soon as it's created. 135180SN/A // But we need to wait for request data esp. in atomic mode. 1368737Skoansin.tan@gmail.com SMMUAction a; 137393SN/A a.type = ACTION_INITIAL_NOP; 138393SN/A a.pkt = NULL; 139393SN/A yield(a); 140393SN/A 141384SN/A const Addr next4k = (request.addr + 0x1000ULL) & ~0xfffULL; 142384SN/A 143393SN/A if ((request.addr + request.size) > next4k) 1448737Skoansin.tan@gmail.com panic("Transaction crosses 4k boundary (addr=%#x size=%#x)!\n", 145393SN/A request.addr, request.size); 146393SN/A 147393SN/A 148393SN/A unsigned numSlaveBeats = request.isWrite ? 149384SN/A (request.size + (ifc.portWidth - 1)) / ifc.portWidth : 1; 150189SN/A 151189SN/A doSemaphoreDown(yield, ifc.slavePortSem); 1522623SN/A doDelay(yield, Cycles(numSlaveBeats)); 1532SN/A doSemaphoreUp(ifc.slavePortSem); 154729SN/A 155334SN/A 1562SN/A recvTick = curTick(); 1572SN/A 1582SN/A 1598834Satgutier@umich.edu if (!(smmu.regs.cr0 & 0x1)) { 1608834Satgutier@umich.edu // SMMU disabled 1618834Satgutier@umich.edu doDelay(yield, Cycles(1)); 1628834Satgutier@umich.edu completeTransaction(yield, bypass(request.addr)); 1638834Satgutier@umich.edu return; 1648834Satgutier@umich.edu } 1658834Satgutier@umich.edu 1662SN/A TranslResult tr; 1672SN/A bool wasPrefetched = false; 1687897Shestness@cs.utexas.edu 1697897Shestness@cs.utexas.edu if (request.isPrefetch) { 1707897Shestness@cs.utexas.edu // Abort prefetch if: 1717897Shestness@cs.utexas.edu // - there's already a transaction looking up the same 4k page, OR 1727897Shestness@cs.utexas.edu // - requested address is already in the TLB. 1737897Shestness@cs.utexas.edu if (hazard4kCheck() || ifcTLBLookup(yield, tr, wasPrefetched)) 1747897Shestness@cs.utexas.edu completePrefetch(yield); // this never returns 1757897Shestness@cs.utexas.edu 1767897Shestness@cs.utexas.edu hazard4kRegister(); 1777897Shestness@cs.utexas.edu 1787897Shestness@cs.utexas.edu tr = smmuTranslation(yield); 1797897Shestness@cs.utexas.edu 1807897Shestness@cs.utexas.edu if (tr.fault == FAULT_NONE) 1817897Shestness@cs.utexas.edu ifcTLBUpdate(yield, tr); 1827897Shestness@cs.utexas.edu 1837897Shestness@cs.utexas.edu hazard4kRelease(); 1847897Shestness@cs.utexas.edu 1857897Shestness@cs.utexas.edu completePrefetch(yield); 1867897Shestness@cs.utexas.edu } else { 1877897Shestness@cs.utexas.edu hazardIdRegister(); 1887897Shestness@cs.utexas.edu 1897897Shestness@cs.utexas.edu if (!microTLBLookup(yield, tr)) { 1907897Shestness@cs.utexas.edu bool hit = ifcTLBLookup(yield, tr, wasPrefetched); 1917897Shestness@cs.utexas.edu if (!hit) { 1927897Shestness@cs.utexas.edu while (!hit && hazard4kCheck()) { 1937897Shestness@cs.utexas.edu hazard4kHold(yield); 1947897Shestness@cs.utexas.edu hit = ifcTLBLookup(yield, tr, wasPrefetched); 1957897Shestness@cs.utexas.edu } 1967897Shestness@cs.utexas.edu } 1977897Shestness@cs.utexas.edu 1987897Shestness@cs.utexas.edu // Issue prefetch if: 1997897Shestness@cs.utexas.edu // - there was a TLB hit and the entry was prefetched, OR 2007897Shestness@cs.utexas.edu // - TLB miss was successfully serviced 2017897Shestness@cs.utexas.edu if (hit) { 2027897Shestness@cs.utexas.edu if (wasPrefetched) 2037897Shestness@cs.utexas.edu issuePrefetch(next4k); 2047897Shestness@cs.utexas.edu } else { 2057897Shestness@cs.utexas.edu hazard4kRegister(); 2067897Shestness@cs.utexas.edu 2077897Shestness@cs.utexas.edu tr = smmuTranslation(yield); 2087897Shestness@cs.utexas.edu 2097897Shestness@cs.utexas.edu if (tr.fault == FAULT_NONE) { 2107897Shestness@cs.utexas.edu ifcTLBUpdate(yield, tr); 2117897Shestness@cs.utexas.edu 2127897Shestness@cs.utexas.edu issuePrefetch(next4k); 2137897Shestness@cs.utexas.edu } 2147897Shestness@cs.utexas.edu 2157897Shestness@cs.utexas.edu hazard4kRelease(); 2167897Shestness@cs.utexas.edu } 2177897Shestness@cs.utexas.edu 2189920Syasuko.eckert@amd.com if (tr.fault == FAULT_NONE) 2199920Syasuko.eckert@amd.com microTLBUpdate(yield, tr); 2209920Syasuko.eckert@amd.com } 2219920Syasuko.eckert@amd.com 2229920Syasuko.eckert@amd.com hazardIdHold(yield); 2239920Syasuko.eckert@amd.com hazardIdRelease(); 2249920Syasuko.eckert@amd.com 2259920Syasuko.eckert@amd.com if (tr.fault != FAULT_NONE) 2269920Syasuko.eckert@amd.com panic("fault\n"); 2279920Syasuko.eckert@amd.com 2289920Syasuko.eckert@amd.com completeTransaction(yield, tr); 2299920Syasuko.eckert@amd.com } 2302SN/A} 2317897Shestness@cs.utexas.edu 2327897Shestness@cs.utexas.eduSMMUTranslationProcess::TranslResult 2337897Shestness@cs.utexas.eduSMMUTranslationProcess::bypass(Addr addr) const 2347897Shestness@cs.utexas.edu{ 2357897Shestness@cs.utexas.edu TranslResult tr; 2367897Shestness@cs.utexas.edu tr.fault = FAULT_NONE; 2377897Shestness@cs.utexas.edu tr.addr = addr; 2387897Shestness@cs.utexas.edu tr.addrMask = 0; 2397897Shestness@cs.utexas.edu tr.writable = 1; 2407897Shestness@cs.utexas.edu 2417897Shestness@cs.utexas.edu return tr; 2427897Shestness@cs.utexas.edu} 2432SN/A 2442SN/ASMMUTranslationProcess::TranslResult 2451001SN/ASMMUTranslationProcess::smmuTranslation(Yield &yield) 2461001SN/A{ 2471001SN/A TranslResult tr; 2481001SN/A 2491001SN/A // Need SMMU credit to proceed 2502SN/A doSemaphoreDown(yield, smmu.transSem); 2512SN/A 2522SN/A // Simulate pipelined IFC->SMMU link 2532SN/A doSemaphoreDown(yield, smmu.ifcSmmuSem); 2542SN/A doDelay(yield, Cycles(1)); // serialize transactions 2557897Shestness@cs.utexas.edu doSemaphoreUp(smmu.ifcSmmuSem); 2567897Shestness@cs.utexas.edu doDelay(yield, smmu.ifcSmmuLat - Cycles(1)); // remaining pipeline delay 2577897Shestness@cs.utexas.edu 2587897Shestness@cs.utexas.edu bool haveConfig = true; 2597897Shestness@cs.utexas.edu if (!configCacheLookup(yield, context)) { 2607897Shestness@cs.utexas.edu if(findConfig(yield, context, tr)) { 2617897Shestness@cs.utexas.edu configCacheUpdate(yield, context); 2627897Shestness@cs.utexas.edu } else { 2637897Shestness@cs.utexas.edu haveConfig = false; 2647897Shestness@cs.utexas.edu } 2652SN/A } 2662SN/A 2672SN/A if (haveConfig && !smmuTLBLookup(yield, tr)) { 2682SN/A // SMMU main TLB miss 2692SN/A 2702SN/A // Need PTW slot to proceed 2712SN/A doSemaphoreDown(yield, smmu.ptwSem); 2722SN/A 2732SN/A // Page table walk 2742SN/A Tick ptwStartTick = curTick(); 2752SN/A 2762SN/A if (context.stage1Enable) { 2772390SN/A tr = translateStage1And2(yield, request.addr); 2782390SN/A } else if (context.stage2Enable) { 2792390SN/A tr = translateStage2(yield, request.addr, true); 2802390SN/A } else { 2812390SN/A tr = bypass(request.addr); 2822390SN/A } 2832390SN/A 2842390SN/A if (context.stage1Enable || context.stage2Enable) 2852390SN/A smmu.ptwTimeDist.sample(curTick() - ptwStartTick); 2862390SN/A 2872390SN/A // Free PTW slot 2882390SN/A doSemaphoreUp(smmu.ptwSem); 289385SN/A 2907897Shestness@cs.utexas.edu if (tr.fault == FAULT_NONE) 2917897Shestness@cs.utexas.edu smmuTLBUpdate(yield, tr); 29210061Sandreas@sandberg.pp.se } 29310061Sandreas@sandberg.pp.se 29410061Sandreas@sandberg.pp.se // Simulate pipelined SMMU->SLAVE INTERFACE link 29510061Sandreas@sandberg.pp.se doSemaphoreDown(yield, smmu.smmuIfcSem); 29610061Sandreas@sandberg.pp.se doDelay(yield, Cycles(1)); // serialize transactions 29710061Sandreas@sandberg.pp.se doSemaphoreUp(smmu.smmuIfcSem); 29810061Sandreas@sandberg.pp.se doDelay(yield, smmu.smmuIfcLat - Cycles(1)); // remaining pipeline delay 29910061Sandreas@sandberg.pp.se 30010061Sandreas@sandberg.pp.se // return SMMU credit 30110061Sandreas@sandberg.pp.se doSemaphoreUp(smmu.transSem); 30210061Sandreas@sandberg.pp.se 30310061Sandreas@sandberg.pp.se return tr; 30410061Sandreas@sandberg.pp.se} 30510061Sandreas@sandberg.pp.se 30610061Sandreas@sandberg.pp.sebool 3072SN/ASMMUTranslationProcess::microTLBLookup(Yield &yield, TranslResult &tr) 3082SN/A{ 3092SN/A if (!ifc.microTLBEnable) 3102623SN/A return false; 311334SN/A 3122361SN/A doSemaphoreDown(yield, ifc.microTLBSem); 3135496Ssaidi@eecs.umich.edu doDelay(yield, ifc.microTLBLat); 314334SN/A const SMMUTLB::Entry *e = 315334SN/A ifc.microTLB->lookup(request.sid, request.ssid, request.addr); 316334SN/A doSemaphoreUp(ifc.microTLBSem); 3179448SAndreas.Sandberg@ARM.com 3182SN/A if (!e) { 3199448SAndreas.Sandberg@ARM.com DPRINTF(SMMUv3, "micro TLB miss vaddr=%#x sid=%#x ssid=%#x\n", 3209448SAndreas.Sandberg@ARM.com request.addr, request.sid, request.ssid); 3219448SAndreas.Sandberg@ARM.com 3222683Sktlim@umich.edu return false; 3232SN/A } 3242SN/A 3252SN/A DPRINTF(SMMUv3, 3269448SAndreas.Sandberg@ARM.com "micro TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x paddr=%#x\n", 3279448SAndreas.Sandberg@ARM.com request.addr, e->vaMask, request.sid, request.ssid, e->pa); 3282SN/A 3299448SAndreas.Sandberg@ARM.com tr.fault = FAULT_NONE; 3309448SAndreas.Sandberg@ARM.com tr.addr = e->pa + (request.addr & ~e->vaMask);; 3319448SAndreas.Sandberg@ARM.com tr.addrMask = e->vaMask; 3322SN/A tr.writable = e->permissions; 3332SN/A 3342SN/A return true; 3356221Snate@binkert.org} 3362SN/A 3372SN/Abool 3382SN/ASMMUTranslationProcess::ifcTLBLookup(Yield &yield, TranslResult &tr, 3392SN/A bool &wasPrefetched) 3402623SN/A{ 3412SN/A if (!ifc.mainTLBEnable) 3422680Sktlim@umich.edu return false; 3432SN/A 3442SN/A doSemaphoreDown(yield, ifc.mainTLBSem); 3452SN/A doDelay(yield, ifc.mainTLBLat); 3465807Snate@binkert.org const SMMUTLB::Entry *e = 3472SN/A ifc.mainTLB->lookup(request.sid, request.ssid, request.addr); 3485807Snate@binkert.org doSemaphoreUp(ifc.mainTLBSem); 3495807Snate@binkert.org 3502SN/A if (!e) { 3515807Snate@binkert.org DPRINTF(SMMUv3, 3525807Snate@binkert.org "SLAVE Interface TLB miss vaddr=%#x sid=%#x ssid=%#x\n", 3532SN/A request.addr, request.sid, request.ssid); 3542SN/A 3552SN/A return false; 3562623SN/A } 3572SN/A 3585704Snate@binkert.org DPRINTF(SMMUv3, 3595647Sgblack@eecs.umich.edu "SLAVE Interface TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x " 3602SN/A "paddr=%#x\n", request.addr, e->vaMask, request.sid, 3613520Sgblack@eecs.umich.edu request.ssid, e->pa); 3627338SAli.Saidi@ARM.com 3635647Sgblack@eecs.umich.edu tr.fault = FAULT_NONE; 3643520Sgblack@eecs.umich.edu tr.addr = e->pa + (request.addr & ~e->vaMask);; 3659023Sgblack@eecs.umich.edu tr.addrMask = e->vaMask; 3662SN/A tr.writable = e->permissions; 3672SN/A wasPrefetched = e->prefetched; 3682623SN/A 3692SN/A return true; 3702623SN/A} 3715894Sgblack@eecs.umich.edu 3722662Sstever@eecs.umich.edubool 3732623SN/ASMMUTranslationProcess::smmuTLBLookup(Yield &yield, TranslResult &tr) 3747720Sgblack@eecs.umich.edu{ 3754495Sacolyte@umich.edu if (!smmu.tlbEnable) 3762623SN/A return false; 3777720Sgblack@eecs.umich.edu 3782623SN/A doSemaphoreDown(yield, smmu.tlbSem); 3797720Sgblack@eecs.umich.edu doDelay(yield, smmu.tlbLat); 3808832SAli.Saidi@ARM.com const ARMArchTLB::Entry *e = 3818832SAli.Saidi@ARM.com smmu.tlb.lookup(request.addr, context.asid, context.vmid); 3822623SN/A doSemaphoreUp(smmu.tlbSem); 3832623SN/A 3842623SN/A if (!e) { 3852623SN/A DPRINTF(SMMUv3, "SMMU TLB miss vaddr=%#x asid=%#x vmid=%#x\n", 3862623SN/A request.addr, context.asid, context.vmid); 3872623SN/A 3882SN/A return false; 3892683Sktlim@umich.edu } 3902427SN/A 3912683Sktlim@umich.edu DPRINTF(SMMUv3, 3922427SN/A "SMMU TLB hit vaddr=%#x amask=%#x asid=%#x vmid=%#x paddr=%#x\n", 3932SN/A request.addr, e->vaMask, context.asid, context.vmid, e->pa); 3942623SN/A 3952623SN/A tr.fault = FAULT_NONE; 3967897Shestness@cs.utexas.edu tr.addr = e->pa + (request.addr & ~e->vaMask);; 3972SN/A tr.addrMask = e->vaMask; 3982623SN/A tr.writable = e->permissions; 3992623SN/A 4004377Sgblack@eecs.umich.edu return true; 4017720Sgblack@eecs.umich.edu} 4024377Sgblack@eecs.umich.edu 4037720Sgblack@eecs.umich.eduvoid 4045665Sgblack@eecs.umich.eduSMMUTranslationProcess::microTLBUpdate(Yield &yield, 4057720Sgblack@eecs.umich.edu const TranslResult &tr) 4067720Sgblack@eecs.umich.edu{ 4075665Sgblack@eecs.umich.edu assert(tr.fault == FAULT_NONE); 4085665Sgblack@eecs.umich.edu 4094181Sgblack@eecs.umich.edu if (!ifc.microTLBEnable) 4104181Sgblack@eecs.umich.edu return; 4119023Sgblack@eecs.umich.edu 4129023Sgblack@eecs.umich.edu SMMUTLB::Entry e; 4134181Sgblack@eecs.umich.edu e.valid = true; 4144182Sgblack@eecs.umich.edu e.prefetched = false; 4157720Sgblack@eecs.umich.edu e.sid = request.sid; 4169023Sgblack@eecs.umich.edu e.ssid = request.ssid; 4179023Sgblack@eecs.umich.edu e.vaMask = tr.addrMask; 4184593Sgblack@eecs.umich.edu e.va = request.addr & e.vaMask; 4199023Sgblack@eecs.umich.edu e.pa = tr.addr & e.vaMask; 4204377Sgblack@eecs.umich.edu e.permissions = tr.writable; 4219023Sgblack@eecs.umich.edu e.asid = context.asid; 4224377Sgblack@eecs.umich.edu e.vmid = context.vmid; 4239023Sgblack@eecs.umich.edu 4249023Sgblack@eecs.umich.edu doSemaphoreDown(yield, ifc.microTLBSem); 4254377Sgblack@eecs.umich.edu 4267720Sgblack@eecs.umich.edu DPRINTF(SMMUv3, 4274377Sgblack@eecs.umich.edu "micro TLB upd vaddr=%#x amask=%#x paddr=%#x sid=%#x ssid=%#x\n", 4284377Sgblack@eecs.umich.edu e.va, e.vaMask, e.pa, e.sid, e.ssid); 4294377Sgblack@eecs.umich.edu 4304377Sgblack@eecs.umich.edu ifc.microTLB->store(e, SMMUTLB::ALLOC_ANY_WAY); 4314181Sgblack@eecs.umich.edu 4324181Sgblack@eecs.umich.edu doSemaphoreUp(ifc.microTLBSem); 4334181Sgblack@eecs.umich.edu} 4344539Sgblack@eecs.umich.edu 4353276Sgblack@eecs.umich.eduvoid 4367720Sgblack@eecs.umich.eduSMMUTranslationProcess::ifcTLBUpdate(Yield &yield, 4373280Sgblack@eecs.umich.edu const TranslResult &tr) 4383280Sgblack@eecs.umich.edu{ 4393276Sgblack@eecs.umich.edu assert(tr.fault == FAULT_NONE); 4403276Sgblack@eecs.umich.edu 4413276Sgblack@eecs.umich.edu if (!ifc.mainTLBEnable) 4427720Sgblack@eecs.umich.edu return; 4433276Sgblack@eecs.umich.edu 4443276Sgblack@eecs.umich.edu SMMUTLB::Entry e; 4454181Sgblack@eecs.umich.edu e.valid = true; 4468955Sgblack@eecs.umich.edu e.prefetched = request.isPrefetch; 4474522Ssaidi@eecs.umich.edu e.sid = request.sid; 4487823Ssteve.reinhardt@amd.com e.ssid = request.ssid; 4497720Sgblack@eecs.umich.edu e.vaMask = tr.addrMask; 4502470SN/A e.va = request.addr & e.vaMask; 4518955Sgblack@eecs.umich.edu e.pa = tr.addr & e.vaMask; 4524181Sgblack@eecs.umich.edu e.permissions = tr.writable; 4534522Ssaidi@eecs.umich.edu e.asid = context.asid; 4544181Sgblack@eecs.umich.edu e.vmid = context.vmid; 45510061Sandreas@sandberg.pp.se 45610061Sandreas@sandberg.pp.se SMMUTLB::AllocPolicy alloc = SMMUTLB::ALLOC_ANY_WAY; 45710061Sandreas@sandberg.pp.se if (ifc.prefetchEnable && ifc.prefetchReserveLastWay) 45810061Sandreas@sandberg.pp.se alloc = request.isPrefetch ? 45910061Sandreas@sandberg.pp.se SMMUTLB::ALLOC_LAST_WAY : SMMUTLB::ALLOC_ANY_BUT_LAST_WAY; 46010061Sandreas@sandberg.pp.se 46110061Sandreas@sandberg.pp.se doSemaphoreDown(yield, ifc.mainTLBSem); 46210061Sandreas@sandberg.pp.se 46310061Sandreas@sandberg.pp.se DPRINTF(SMMUv3, 46410061Sandreas@sandberg.pp.se "SLAVE Interface upd vaddr=%#x amask=%#x paddr=%#x sid=%#x " 46510061Sandreas@sandberg.pp.se "ssid=%#x\n", e.va, e.vaMask, e.pa, e.sid, e.ssid); 46610061Sandreas@sandberg.pp.se 46710061Sandreas@sandberg.pp.se ifc.mainTLB->store(e, alloc); 4682623SN/A 4692623SN/A doSemaphoreUp(ifc.mainTLBSem); 4702623SN/A} 4712623SN/A 4722623SN/Avoid 4737720Sgblack@eecs.umich.eduSMMUTranslationProcess::smmuTLBUpdate(Yield &yield, 4747720Sgblack@eecs.umich.edu const TranslResult &tr) 4757720Sgblack@eecs.umich.edu{ 4767720Sgblack@eecs.umich.edu assert(tr.fault == FAULT_NONE); 4778780Sgblack@eecs.umich.edu 4783577Sgblack@eecs.umich.edu if (!smmu.tlbEnable) 4797720Sgblack@eecs.umich.edu return; 4805086Sgblack@eecs.umich.edu 4812623SN/A ARMArchTLB::Entry e; 4822683Sktlim@umich.edu e.valid = true; 4832623SN/A e.vaMask = tr.addrMask; 4842SN/A e.va = request.addr & e.vaMask; 4852623SN/A e.asid = context.asid; 4862623SN/A e.vmid = context.vmid; 4872SN/A e.pa = tr.addr & e.vaMask; 4882SN/A e.permissions = tr.writable; 4892623SN/A 4902623SN/A doSemaphoreDown(yield, smmu.tlbSem); 4912623SN/A 4922623SN/A DPRINTF(SMMUv3, 4932SN/A "SMMU TLB upd vaddr=%#x amask=%#x paddr=%#x asid=%#x vmid=%#x\n", 4945953Ssaidi@eecs.umich.edu e.va, e.vaMask, e.pa, e.asid, e.vmid); 4957720Sgblack@eecs.umich.edu 4965953Ssaidi@eecs.umich.edu smmu.tlb.store(e); 4975953Ssaidi@eecs.umich.edu 49810061Sandreas@sandberg.pp.se doSemaphoreUp(smmu.tlbSem); 49910061Sandreas@sandberg.pp.se} 50010061Sandreas@sandberg.pp.se 50110061Sandreas@sandberg.pp.sebool 5027897Shestness@cs.utexas.eduSMMUTranslationProcess::configCacheLookup(Yield &yield, TranslContext &tc) 5037897Shestness@cs.utexas.edu{ 5047897Shestness@cs.utexas.edu if (!smmu.configCacheEnable) 5057897Shestness@cs.utexas.edu return false; 5067897Shestness@cs.utexas.edu 5077897Shestness@cs.utexas.edu doSemaphoreDown(yield, smmu.configSem); 5087897Shestness@cs.utexas.edu doDelay(yield, smmu.configLat); 5097897Shestness@cs.utexas.edu const ConfigCache::Entry *e = 5107897Shestness@cs.utexas.edu smmu.configCache.lookup(request.sid, request.ssid); 5117897Shestness@cs.utexas.edu doSemaphoreUp(smmu.configSem); 5127897Shestness@cs.utexas.edu 5137897Shestness@cs.utexas.edu if (!e) { 5147897Shestness@cs.utexas.edu DPRINTF(SMMUv3, "Config miss sid=%#x ssid=%#x\n", 5157897Shestness@cs.utexas.edu request.sid, request.ssid); 5167897Shestness@cs.utexas.edu 5177897Shestness@cs.utexas.edu return false; 5187897Shestness@cs.utexas.edu } 5197897Shestness@cs.utexas.edu 5207897Shestness@cs.utexas.edu DPRINTF(SMMUv3, "Config hit sid=%#x ssid=%#x ttb=%#08x asid=%#x\n", 5217897Shestness@cs.utexas.edu request.sid, request.ssid, e->ttb0, e->asid); 5227897Shestness@cs.utexas.edu 5237897Shestness@cs.utexas.edu tc.stage1Enable = e->stage1_en; 5247897Shestness@cs.utexas.edu tc.stage2Enable = e->stage2_en; 5257897Shestness@cs.utexas.edu 5267897Shestness@cs.utexas.edu tc.ttb0 = e->ttb0; 5277897Shestness@cs.utexas.edu tc.ttb1 = e->ttb1; 5287897Shestness@cs.utexas.edu tc.asid = e->asid; 5297897Shestness@cs.utexas.edu tc.httb = e->httb; 5307897Shestness@cs.utexas.edu tc.vmid = e->vmid; 5317897Shestness@cs.utexas.edu 5327897Shestness@cs.utexas.edu tc.stage1TranslGranule = e->stage1_tg; 5337897Shestness@cs.utexas.edu tc.stage2TranslGranule = e->stage2_tg; 5347897Shestness@cs.utexas.edu 5358780Sgblack@eecs.umich.edu tc.t0sz = e->t0sz; 5368780Sgblack@eecs.umich.edu tc.s2t0sz = e->s2t0sz; 5372644Sstever@eecs.umich.edu 5382644Sstever@eecs.umich.edu return true; 5394046Sbinkertn@umich.edu} 5404046Sbinkertn@umich.edu 5414046Sbinkertn@umich.eduvoid 5422644Sstever@eecs.umich.eduSMMUTranslationProcess::configCacheUpdate(Yield &yield, 5432623SN/A const TranslContext &tc) 5442SN/A{ 5452623SN/A if (!smmu.configCacheEnable) 5462623SN/A return; 5472623SN/A 54810061Sandreas@sandberg.pp.se ConfigCache::Entry e; 54910061Sandreas@sandberg.pp.se e.valid = true; 5504377Sgblack@eecs.umich.edu e.sid = request.sid; 5514377Sgblack@eecs.umich.edu e.ssid = request.ssid; 5522090SN/A e.stage1_en = tc.stage1Enable; 5533905Ssaidi@eecs.umich.edu e.stage2_en = tc.stage2Enable; 5547678Sgblack@eecs.umich.edu e.ttb0 = tc.ttb0; 5559023Sgblack@eecs.umich.edu e.ttb1 = tc.ttb1; 5564377Sgblack@eecs.umich.edu e.asid = tc.asid; 5577720Sgblack@eecs.umich.edu e.httb = tc.httb; 5587720Sgblack@eecs.umich.edu e.vmid = tc.vmid; 5597720Sgblack@eecs.umich.edu e.stage1_tg = tc.stage1TranslGranule; 5607720Sgblack@eecs.umich.edu e.stage2_tg = tc.stage2TranslGranule; 5617720Sgblack@eecs.umich.edu e.t0sz = tc.t0sz; 5627720Sgblack@eecs.umich.edu e.s2t0sz = tc.s2t0sz; 5633276Sgblack@eecs.umich.edu 5642SN/A doSemaphoreDown(yield, smmu.configSem); 56510061Sandreas@sandberg.pp.se 56610061Sandreas@sandberg.pp.se DPRINTF(SMMUv3, "Config upd sid=%#x ssid=%#x\n", e.sid, e.ssid); 56710061Sandreas@sandberg.pp.se 56810061Sandreas@sandberg.pp.se smmu.configCache.store(e); 56910061Sandreas@sandberg.pp.se 57010061Sandreas@sandberg.pp.se doSemaphoreUp(smmu.configSem); 57110061Sandreas@sandberg.pp.se} 57210061Sandreas@sandberg.pp.se 57310061Sandreas@sandberg.pp.sebool 57410061Sandreas@sandberg.pp.seSMMUTranslationProcess::findConfig(Yield &yield, 57510061Sandreas@sandberg.pp.se TranslContext &tc, 57610061Sandreas@sandberg.pp.se TranslResult &tr) 57710061Sandreas@sandberg.pp.se{ 57810061Sandreas@sandberg.pp.se tc.stage1Enable = false; 57910061Sandreas@sandberg.pp.se tc.stage2Enable = false; 58010061Sandreas@sandberg.pp.se 58110061Sandreas@sandberg.pp.se StreamTableEntry ste; 5822SN/A doReadSTE(yield, ste, request.sid); 5832SN/A 5849461Snilay@cs.wisc.edu switch (ste.dw0.config) { 5859461Snilay@cs.wisc.edu case STE_CONFIG_BYPASS: 5869461Snilay@cs.wisc.edu break; 5879461Snilay@cs.wisc.edu 5889461Snilay@cs.wisc.edu case STE_CONFIG_STAGE1_ONLY: 5899461Snilay@cs.wisc.edu tc.stage1Enable = true; 590 break; 591 592 case STE_CONFIG_STAGE2_ONLY: 593 tc.stage2Enable = true; 594 break; 595 596 case STE_CONFIG_STAGE1_AND_2: 597 tc.stage1Enable = true; 598 tc.stage2Enable = true; 599 break; 600 601 default: 602 panic("Bad or unimplemented STE config %d\n", 603 ste.dw0.config); 604 } 605 606 607 // Establish stage 2 context first since 608 // Context Descriptors can be in IPA space. 609 if (tc.stage2Enable) { 610 tc.httb = ste.dw3.s2ttb << STE_S2TTB_SHIFT; 611 tc.vmid = ste.dw2.s2vmid; 612 tc.stage2TranslGranule = ste.dw2.s2tg; 613 tc.s2t0sz = ste.dw2.s2t0sz; 614 } else { 615 tc.httb = 0xdeadbeef; 616 tc.vmid = 0; 617 tc.stage2TranslGranule = TRANS_GRANULE_INVALID; 618 tc.s2t0sz = 0; 619 } 620 621 622 // Now fetch stage 1 config. 623 if (context.stage1Enable) { 624 ContextDescriptor cd; 625 doReadCD(yield, cd, ste, request.sid, request.ssid); 626 627 tc.ttb0 = cd.dw1.ttb0 << CD_TTB_SHIFT; 628 tc.ttb1 = cd.dw2.ttb1 << CD_TTB_SHIFT; 629 tc.asid = cd.dw0.asid; 630 tc.stage1TranslGranule = cd.dw0.tg0; 631 tc.t0sz = cd.dw0.t0sz; 632 } else { 633 tc.ttb0 = 0xcafebabe; 634 tc.ttb1 = 0xcafed00d; 635 tc.asid = 0; 636 tc.stage1TranslGranule = TRANS_GRANULE_INVALID; 637 tc.t0sz = 0; 638 } 639 640 return true; 641} 642 643void 644SMMUTranslationProcess::walkCacheLookup( 645 Yield &yield, 646 const WalkCache::Entry *&walkEntry, 647 Addr addr, uint16_t asid, uint16_t vmid, 648 unsigned stage, unsigned level) 649{ 650 const char *indent = stage==2 ? " " : ""; 651 (void) indent; // this is only used in DPRINTFs 652 653 const PageTableOps *pt_ops = 654 stage == 1 ? 655 smmu.getPageTableOps(context.stage1TranslGranule) : 656 smmu.getPageTableOps(context.stage2TranslGranule); 657 658 unsigned walkCacheLevels = 659 smmu.walkCacheEnable ? 660 (stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels) : 661 0; 662 663 if ((1 << level) & walkCacheLevels) { 664 doSemaphoreDown(yield, smmu.walkSem); 665 doDelay(yield, smmu.walkLat); 666 667 walkEntry = smmu.walkCache.lookup(addr, pt_ops->walkMask(level), 668 asid, vmid, stage, level); 669 670 if (walkEntry) { 671 DPRINTF(SMMUv3, "%sWalkCache hit va=%#x asid=%#x vmid=%#x " 672 "base=%#x (S%d, L%d)\n", 673 indent, addr, asid, vmid, walkEntry->pa, stage, level); 674 } else { 675 DPRINTF(SMMUv3, "%sWalkCache miss va=%#x asid=%#x vmid=%#x " 676 "(S%d, L%d)\n", 677 indent, addr, asid, vmid, stage, level); 678 } 679 680 doSemaphoreUp(smmu.walkSem); 681 } 682} 683 684void 685SMMUTranslationProcess::walkCacheUpdate(Yield &yield, Addr va, 686 Addr vaMask, Addr pa, 687 unsigned stage, unsigned level, 688 bool leaf, uint8_t permissions) 689{ 690 unsigned walkCacheLevels = 691 stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels; 692 693 if (smmu.walkCacheEnable && ((1<<level) & walkCacheLevels)) { 694 WalkCache::Entry e; 695 e.valid = true; 696 e.va = va; 697 e.vaMask = vaMask; 698 e.asid = stage==1 ? context.asid : 0; 699 e.vmid = context.vmid; 700 e.stage = stage; 701 e.level = level; 702 e.leaf = leaf; 703 e.pa = pa; 704 e.permissions = permissions; 705 706 doSemaphoreDown(yield, smmu.walkSem); 707 708 DPRINTF(SMMUv3, "%sWalkCache upd va=%#x mask=%#x asid=%#x vmid=%#x " 709 "tpa=%#x leaf=%s (S%d, L%d)\n", 710 e.stage==2 ? " " : "", 711 e.va, e.vaMask, e.asid, e.vmid, 712 e.pa, e.leaf, e.stage, e.level); 713 714 smmu.walkCache.store(e); 715 716 doSemaphoreUp(smmu.walkSem); 717 } 718} 719 720/* 721 * Please note: 722 * This does not deal with the case where stage 1 page size 723 * is larger than stage 2 page size. 724 */ 725SMMUTranslationProcess::TranslResult 726SMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, 727 const PageTableOps *pt_ops, 728 unsigned level, Addr walkPtr) 729{ 730 PageTableOps::pte_t pte = 0; 731 732 doSemaphoreDown(yield, smmu.cycleSem); 733 doDelay(yield, Cycles(1)); 734 doSemaphoreUp(smmu.cycleSem); 735 736 for (; level <= pt_ops->lastLevel(); level++) { 737 Addr pte_addr = walkPtr + pt_ops->index(addr, level); 738 739 DPRINTF(SMMUv3, "Fetching S1 L%d PTE from pa=%#08x\n", 740 level, pte_addr); 741 742 doReadPTE(yield, addr, pte_addr, &pte, 1, level); 743 744 DPRINTF(SMMUv3, "Got S1 L%d PTE=%#x from pa=%#08x\n", 745 level, pte, pte_addr); 746 747 doSemaphoreDown(yield, smmu.cycleSem); 748 doDelay(yield, Cycles(1)); 749 doSemaphoreUp(smmu.cycleSem); 750 751 bool valid = pt_ops->isValid(pte, level); 752 bool leaf = pt_ops->isLeaf(pte, level); 753 754 if (!valid) { 755 DPRINTF(SMMUv3, "S1 PTE not valid - fault\n"); 756 757 TranslResult tr; 758 tr.fault = FAULT_TRANSLATION; 759 return tr; 760 } 761 762 if (valid && leaf && request.isWrite && 763 !pt_ops->isWritable(pte, level, false)) 764 { 765 DPRINTF(SMMUv3, "S1 page not writable - fault\n"); 766 767 TranslResult tr; 768 tr.fault = FAULT_PERMISSION; 769 return tr; 770 } 771 772 walkPtr = pt_ops->nextLevelPointer(pte, level); 773 774 if (leaf) 775 break; 776 777 if (context.stage2Enable) { 778 TranslResult s2tr = translateStage2(yield, walkPtr, false); 779 if (s2tr.fault != FAULT_NONE) 780 return s2tr; 781 782 walkPtr = s2tr.addr; 783 } 784 785 walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, 786 1, level, leaf, 0); 787 } 788 789 TranslResult tr; 790 tr.fault = FAULT_NONE; 791 tr.addrMask = pt_ops->pageMask(pte, level); 792 tr.addr = walkPtr + (addr & ~tr.addrMask); 793 tr.writable = pt_ops->isWritable(pte, level, false); 794 795 if (context.stage2Enable) { 796 TranslResult s2tr = translateStage2(yield, tr.addr, true); 797 if (s2tr.fault != FAULT_NONE) 798 return s2tr; 799 800 tr = combineTranslations(tr, s2tr); 801 } 802 803 walkCacheUpdate(yield, addr, tr.addrMask, tr.addr, 804 1, level, true, tr.writable); 805 806 return tr; 807} 808 809SMMUTranslationProcess::TranslResult 810SMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, 811 const PageTableOps *pt_ops, 812 unsigned level, Addr walkPtr) 813{ 814 PageTableOps::pte_t pte; 815 816 doSemaphoreDown(yield, smmu.cycleSem); 817 doDelay(yield, Cycles(1)); 818 doSemaphoreUp(smmu.cycleSem); 819 820 for (; level <= pt_ops->lastLevel(); level++) { 821 Addr pte_addr = walkPtr + pt_ops->index(addr, level); 822 823 DPRINTF(SMMUv3, " Fetching S2 L%d PTE from pa=%#08x\n", 824 level, pte_addr); 825 826 doReadPTE(yield, addr, pte_addr, &pte, 2, level); 827 828 DPRINTF(SMMUv3, " Got S2 L%d PTE=%#x from pa=%#08x\n", 829 level, pte, pte_addr); 830 831 doSemaphoreDown(yield, smmu.cycleSem); 832 doDelay(yield, Cycles(1)); 833 doSemaphoreUp(smmu.cycleSem); 834 835 bool valid = pt_ops->isValid(pte, level); 836 bool leaf = pt_ops->isLeaf(pte, level); 837 838 if (!valid) { 839 DPRINTF(SMMUv3, " S2 PTE not valid - fault\n"); 840 841 TranslResult tr; 842 tr.fault = FAULT_TRANSLATION; 843 return tr; 844 } 845 846 if (valid && leaf && request.isWrite && 847 !pt_ops->isWritable(pte, level, true)) 848 { 849 DPRINTF(SMMUv3, " S2 PTE not writable = fault\n"); 850 851 TranslResult tr; 852 tr.fault = FAULT_PERMISSION; 853 return tr; 854 } 855 856 walkPtr = pt_ops->nextLevelPointer(pte, level); 857 858 if (final_tr || smmu.walkCacheNonfinalEnable) 859 walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, 860 2, level, leaf, 861 leaf ? pt_ops->isWritable(pte, level, true) : 0); 862 if (leaf) 863 break; 864 } 865 866 TranslResult tr; 867 tr.fault = FAULT_NONE; 868 tr.addrMask = pt_ops->pageMask(pte, level); 869 tr.addr = walkPtr + (addr & ~tr.addrMask); 870 tr.writable = pt_ops->isWritable(pte, level, true); 871 872 return tr; 873} 874 875SMMUTranslationProcess::TranslResult 876SMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) 877{ 878 const PageTableOps *pt_ops = 879 smmu.getPageTableOps(context.stage1TranslGranule); 880 881 const WalkCache::Entry *walk_ep = NULL; 882 unsigned level; 883 884 // Level here is actually (level+1) so we can count down 885 // to 0 using unsigned int. 886 for (level = pt_ops->lastLevel() + 1; 887 level > pt_ops->firstLevel(context.t0sz); 888 level--) 889 { 890 walkCacheLookup(yield, walk_ep, addr, 891 context.asid, context.vmid, 1, level-1); 892 893 if (walk_ep) 894 break; 895 } 896 897 // Correct level (see above). 898 level -= 1; 899 900 TranslResult tr; 901 if (walk_ep) { 902 if (walk_ep->leaf) { 903 tr.fault = FAULT_NONE; 904 tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); 905 tr.addrMask = walk_ep->vaMask; 906 tr.writable = walk_ep->permissions; 907 } else { 908 tr = walkStage1And2(yield, addr, pt_ops, level+1, walk_ep->pa); 909 } 910 } else { 911 Addr table_addr = context.ttb0; 912 if (context.stage2Enable) { 913 TranslResult s2tr = translateStage2(yield, table_addr, false); 914 if (s2tr.fault != FAULT_NONE) 915 return s2tr; 916 917 table_addr = s2tr.addr; 918 } 919 920 tr = walkStage1And2(yield, addr, pt_ops, 921 pt_ops->firstLevel(context.t0sz), 922 table_addr); 923 } 924 925 if (tr.fault == FAULT_NONE) 926 DPRINTF(SMMUv3, "Translated vaddr %#x to paddr %#x\n", addr, tr.addr); 927 928 return tr; 929} 930 931SMMUTranslationProcess::TranslResult 932SMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) 933{ 934 const PageTableOps *pt_ops = 935 smmu.getPageTableOps(context.stage2TranslGranule); 936 937 const IPACache::Entry *ipa_ep = NULL; 938 if (smmu.ipaCacheEnable) { 939 doSemaphoreDown(yield, smmu.ipaSem); 940 doDelay(yield, smmu.ipaLat); 941 ipa_ep = smmu.ipaCache.lookup(addr, context.vmid); 942 doSemaphoreUp(smmu.ipaSem); 943 } 944 945 if (ipa_ep) { 946 TranslResult tr; 947 tr.fault = FAULT_NONE; 948 tr.addr = ipa_ep->pa + (addr & ~ipa_ep->ipaMask); 949 tr.addrMask = ipa_ep->ipaMask; 950 tr.writable = ipa_ep->permissions; 951 952 DPRINTF(SMMUv3, " IPACache hit ipa=%#x vmid=%#x pa=%#x\n", 953 addr, context.vmid, tr.addr); 954 955 return tr; 956 } else if (smmu.ipaCacheEnable) { 957 DPRINTF(SMMUv3, " IPACache miss ipa=%#x vmid=%#x\n", 958 addr, context.vmid); 959 } 960 961 const WalkCache::Entry *walk_ep = NULL; 962 unsigned level = pt_ops->firstLevel(context.s2t0sz); 963 964 if (final_tr || smmu.walkCacheNonfinalEnable) { 965 // Level here is actually (level+1) so we can count down 966 // to 0 using unsigned int. 967 for (level = pt_ops->lastLevel() + 1; 968 level > pt_ops->firstLevel(context.s2t0sz); 969 level--) 970 { 971 walkCacheLookup(yield, walk_ep, addr, 972 0, context.vmid, 2, level-1); 973 974 if (walk_ep) 975 break; 976 } 977 978 // Correct level (see above). 979 level -= 1; 980 } 981 982 TranslResult tr; 983 if (walk_ep) { 984 if (walk_ep->leaf) { 985 tr.fault = FAULT_NONE; 986 tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); 987 tr.addrMask = walk_ep->vaMask; 988 tr.writable = walk_ep->permissions; 989 } else { 990 tr = walkStage2(yield, addr, final_tr, pt_ops, 991 level + 1, walk_ep->pa); 992 } 993 } else { 994 tr = walkStage2(yield, addr, final_tr, pt_ops, 995 pt_ops->firstLevel(context.s2t0sz), 996 context.httb); 997 } 998 999 if (tr.fault == FAULT_NONE) 1000 DPRINTF(SMMUv3, " Translated %saddr %#x to paddr %#x\n", 1001 context.stage1Enable ? "ip" : "v", addr, tr.addr); 1002 1003 if (smmu.ipaCacheEnable) { 1004 IPACache::Entry e; 1005 e.valid = true; 1006 e.ipaMask = tr.addrMask; 1007 e.ipa = addr & e.ipaMask; 1008 e.pa = tr.addr & tr.addrMask; 1009 e.permissions = tr.writable; 1010 e.vmid = context.vmid; 1011 1012 doSemaphoreDown(yield, smmu.ipaSem); 1013 smmu.ipaCache.store(e); 1014 doSemaphoreUp(smmu.ipaSem); 1015 } 1016 1017 return tr; 1018} 1019 1020SMMUTranslationProcess::TranslResult 1021SMMUTranslationProcess::combineTranslations(const TranslResult &s1tr, 1022 const TranslResult &s2tr) const 1023{ 1024 if (s2tr.fault != FAULT_NONE) 1025 return s2tr; 1026 1027 assert(s1tr.fault == FAULT_NONE); 1028 1029 TranslResult tr; 1030 tr.fault = FAULT_NONE; 1031 tr.addr = s2tr.addr; 1032 tr.addrMask = s1tr.addrMask | s2tr.addrMask; 1033 tr.writable = s1tr.writable & s2tr.writable; 1034 1035 return tr; 1036} 1037 1038bool 1039SMMUTranslationProcess::hazard4kCheck() 1040{ 1041 Addr addr4k = request.addr & ~0xfffULL; 1042 1043 for (auto it = ifc.duplicateReqs.begin(); 1044 it != ifc.duplicateReqs.end(); 1045 ++it) 1046 { 1047 Addr other4k = (*it)->request.addr & ~0xfffULL; 1048 if (addr4k == other4k) 1049 return true; 1050 } 1051 1052 return false; 1053} 1054 1055void 1056SMMUTranslationProcess::hazard4kRegister() 1057{ 1058 DPRINTF(SMMUv3Hazard, "4kReg: p=%p a4k=%#x\n", 1059 this, request.addr & ~0xfffULL); 1060 1061 ifc.duplicateReqs.push_back(this); 1062} 1063 1064void 1065SMMUTranslationProcess::hazard4kHold(Yield &yield) 1066{ 1067 Addr addr4k = request.addr & ~0xfffULL; 1068 1069 bool found_hazard; 1070 1071 do { 1072 found_hazard = false; 1073 1074 for (auto it = ifc.duplicateReqs.begin(); 1075 it!=ifc.duplicateReqs.end() && *it!=this; 1076 ++it) 1077 { 1078 Addr other4k = (*it)->request.addr & ~0xfffULL; 1079 1080 DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x Q: p=%p a4k=%#x\n", 1081 this, addr4k, *it, other4k); 1082 1083 if (addr4k == other4k) { 1084 DPRINTF(SMMUv3Hazard, 1085 "4kHold: p=%p a4k=%#x WAIT on p=%p a4k=%#x\n", 1086 this, addr4k, *it, other4k); 1087 1088 doWaitForSignal(yield, ifc.duplicateReqRemoved); 1089 1090 DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x RESUME\n", 1091 this, addr4k); 1092 1093 // This is to avoid checking *it!=this after doWaitForSignal() 1094 // since it could have been deleted. 1095 found_hazard = true; 1096 break; 1097 } 1098 } 1099 } while (found_hazard); 1100} 1101 1102void 1103SMMUTranslationProcess::hazard4kRelease() 1104{ 1105 DPRINTF(SMMUv3Hazard, "4kRel: p=%p a4k=%#x\n", 1106 this, request.addr & ~0xfffULL); 1107 1108 std::list<SMMUTranslationProcess *>::iterator it; 1109 1110 for (it = ifc.duplicateReqs.begin(); it != ifc.duplicateReqs.end(); ++it) 1111 if (*it == this) 1112 break; 1113 1114 if (it == ifc.duplicateReqs.end()) 1115 panic("hazard4kRelease: request not found"); 1116 1117 ifc.duplicateReqs.erase(it); 1118 1119 doBroadcastSignal(ifc.duplicateReqRemoved); 1120} 1121 1122void 1123SMMUTranslationProcess::hazardIdRegister() 1124{ 1125 auto orderId = AMBA::orderId(request.pkt); 1126 1127 DPRINTF(SMMUv3Hazard, "IdReg: p=%p oid=%d\n", this, orderId); 1128 1129 assert(orderId < SMMU_MAX_TRANS_ID); 1130 1131 std::list<SMMUTranslationProcess *> &depReqs = 1132 request.isWrite ? 1133 ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 1134 depReqs.push_back(this); 1135} 1136 1137void 1138SMMUTranslationProcess::hazardIdHold(Yield &yield) 1139{ 1140 auto orderId = AMBA::orderId(request.pkt); 1141 1142 DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d\n", this, orderId); 1143 1144 std::list<SMMUTranslationProcess *> &depReqs = 1145 request.isWrite ? 1146 ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 1147 std::list<SMMUTranslationProcess *>::iterator it; 1148 1149 bool found_hazard; 1150 1151 do { 1152 found_hazard = false; 1153 1154 for (auto it = depReqs.begin(); it!=depReqs.end() && *it!=this; ++it) { 1155 DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d Q: %p\n", 1156 this, orderId, *it); 1157 1158 if (AMBA::orderId((*it)->request.pkt) == orderId) { 1159 DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d WAIT on=%p\n", 1160 this, orderId, *it); 1161 1162 doWaitForSignal(yield, ifc.dependentReqRemoved); 1163 1164 DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d RESUME\n", 1165 this, orderId); 1166 1167 // This is to avoid checking *it!=this after doWaitForSignal() 1168 // since it could have been deleted. 1169 found_hazard = true; 1170 break; 1171 } 1172 } 1173 } while (found_hazard); 1174} 1175 1176void 1177SMMUTranslationProcess::hazardIdRelease() 1178{ 1179 auto orderId = AMBA::orderId(request.pkt); 1180 1181 DPRINTF(SMMUv3Hazard, "IdRel: p=%p oid=%d\n", this, orderId); 1182 1183 std::list<SMMUTranslationProcess *> &depReqs = 1184 request.isWrite ? 1185 ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 1186 std::list<SMMUTranslationProcess *>::iterator it; 1187 1188 for (it = depReqs.begin(); it != depReqs.end(); ++it) { 1189 if (*it == this) 1190 break; 1191 } 1192 1193 if (it == depReqs.end()) 1194 panic("hazardIdRelease: request not found"); 1195 1196 depReqs.erase(it); 1197 1198 doBroadcastSignal(ifc.dependentReqRemoved); 1199} 1200 1201void 1202SMMUTranslationProcess::issuePrefetch(Addr addr) 1203{ 1204 if (!smmu.system.isTimingMode()) 1205 return; 1206 1207 if (!ifc.prefetchEnable || ifc.xlateSlotsRemaining == 0) 1208 return; 1209 1210 std::string proc_name = csprintf("%sprf", name()); 1211 SMMUTranslationProcess *proc = 1212 new SMMUTranslationProcess(proc_name, smmu, ifc); 1213 1214 proc->beginTransaction( 1215 SMMUTranslRequest::prefetch(addr, request.sid, request.ssid)); 1216 proc->scheduleWakeup(smmu.clockEdge(Cycles(1))); 1217} 1218 1219void 1220SMMUTranslationProcess::completeTransaction(Yield &yield, 1221 const TranslResult &tr) 1222{ 1223 assert(tr.fault == FAULT_NONE); 1224 1225 unsigned numMasterBeats = request.isWrite ? 1226 (request.size + (smmu.masterPortWidth-1)) 1227 / smmu.masterPortWidth : 1228 1; 1229 1230 doSemaphoreDown(yield, smmu.masterPortSem); 1231 doDelay(yield, Cycles(numMasterBeats)); 1232 doSemaphoreUp(smmu.masterPortSem); 1233 1234 1235 smmu.translationTimeDist.sample(curTick() - recvTick); 1236 if (!request.isAtsRequest && request.isWrite) 1237 ifc.wrBufSlotsRemaining += 1238 (request.size + (ifc.portWidth-1)) / ifc.portWidth; 1239 1240 smmu.scheduleSlaveRetries(); 1241 1242 1243 SMMUAction a; 1244 1245 if (request.isAtsRequest) { 1246 a.type = ACTION_SEND_RESP_ATS; 1247 1248 if (smmu.system.isAtomicMode()) { 1249 request.pkt->makeAtomicResponse(); 1250 } else if (smmu.system.isTimingMode()) { 1251 request.pkt->makeTimingResponse(); 1252 } else { 1253 panic("Not in atomic or timing mode"); 1254 } 1255 } else { 1256 a.type = ACTION_SEND_REQ_FINAL; 1257 a.ifc = &ifc; 1258 } 1259 1260 a.pkt = request.pkt; 1261 a.delay = 0; 1262 1263 a.pkt->setAddr(tr.addr); 1264 a.pkt->req->setPaddr(tr.addr); 1265 1266 yield(a); 1267 1268 if (!request.isAtsRequest) { 1269 PacketPtr pkt = yield.get(); 1270 pkt->setAddr(request.addr); 1271 1272 a.type = ACTION_SEND_RESP; 1273 a.pkt = pkt; 1274 a.ifc = &ifc; 1275 a.delay = 0; 1276 yield(a); 1277 } 1278} 1279 1280void 1281SMMUTranslationProcess::completePrefetch(Yield &yield) 1282{ 1283 SMMUAction a; 1284 a.type = ACTION_TERMINATE; 1285 a.pkt = NULL; 1286 a.ifc = &ifc; 1287 a.delay = 0; 1288 yield(a); 1289} 1290 1291void 1292SMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev) 1293{ 1294 int sizeMask = mask(smmu.regs.eventq_base & Q_BASE_SIZE_MASK); 1295 1296 if (((smmu.regs.eventq_prod+1) & sizeMask) == 1297 (smmu.regs.eventq_cons & sizeMask)) 1298 panic("Event queue full - aborting\n"); 1299 1300 Addr event_addr = 1301 (smmu.regs.eventq_base & Q_BASE_ADDR_MASK) + 1302 (smmu.regs.eventq_prod & sizeMask) * sizeof(ev); 1303 1304 DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): type=%#x stag=%#x " 1305 "flags=%#x sid=%#x ssid=%#x va=%#08x ipa=%#x\n", 1306 event_addr, smmu.regs.eventq_prod, ev.type, ev.stag, 1307 ev.flags, ev.streamId, ev.substreamId, ev.va, ev.ipa); 1308 1309 // This deliberately resets the overflow field in eventq_prod! 1310 smmu.regs.eventq_prod = (smmu.regs.eventq_prod + 1) & sizeMask; 1311 1312 doWrite(yield, event_addr, &ev, sizeof(ev)); 1313 1314 if (!(smmu.regs.eventq_irq_cfg0 & E_BASE_ENABLE_MASK)) 1315 panic("eventq msi not enabled\n"); 1316 1317 doWrite(yield, smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK, 1318 &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1)); 1319} 1320 1321void 1322SMMUTranslationProcess::doReadSTE(Yield &yield, 1323 StreamTableEntry &ste, 1324 uint32_t sid) 1325{ 1326 unsigned max_sid = 1 << (smmu.regs.strtab_base_cfg & ST_CFG_SIZE_MASK); 1327 if (sid >= max_sid) 1328 panic("SID %#x out of range, max=%#x", sid, max_sid); 1329 1330 Addr ste_addr; 1331 1332 if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_2LEVEL) { 1333 unsigned split = 1334 (smmu.regs.strtab_base_cfg & ST_CFG_SPLIT_MASK) >> ST_CFG_SPLIT_SHIFT; 1335 1336 if (split!= 7 && split!=8 && split!=16) 1337 panic("Invalid stream table split %d", split); 1338 1339 uint64_t l2_ptr; 1340 uint64_t l2_addr = 1341 (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + 1342 bits(sid, 32, split) * sizeof(l2_ptr); 1343 1344 DPRINTF(SMMUv3, "Read L1STE at %#x\n", l2_addr); 1345 1346 doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, 0); 1347 1348 DPRINTF(SMMUv3, "Got L1STE L1 at %#x: 0x%016x\n", l2_addr, l2_ptr); 1349 1350 unsigned span = l2_ptr & ST_L2_SPAN_MASK; 1351 if (span == 0) 1352 panic("Invalid level 1 stream table descriptor"); 1353 1354 unsigned index = bits(sid, split-1, 0); 1355 if (index >= (1 << span)) 1356 panic("StreamID %d out of level 1 descriptor range %d", 1357 sid, 1<<span); 1358 1359 ste_addr = (l2_ptr & ST_L2_ADDR_MASK) + index * sizeof(ste); 1360 1361 smmu.steL1Fetches++; 1362 } else if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_LINEAR) { 1363 ste_addr = 1364 (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + sid * sizeof(ste); 1365 } else { 1366 panic("Invalid stream table format"); 1367 } 1368 1369 DPRINTF(SMMUv3, "Read STE at %#x\n", ste_addr); 1370 1371 doReadConfig(yield, ste_addr, &ste, sizeof(ste), sid, 0); 1372 1373 DPRINTF(SMMUv3, "Got STE at %#x [0]: 0x%016x\n", ste_addr, ste.dw0); 1374 DPRINTF(SMMUv3, " STE at %#x [1]: 0x%016x\n", ste_addr, ste.dw1); 1375 DPRINTF(SMMUv3, " STE at %#x [2]: 0x%016x\n", ste_addr, ste.dw2); 1376 DPRINTF(SMMUv3, " STE at %#x [3]: 0x%016x\n", ste_addr, ste.dw3); 1377 DPRINTF(SMMUv3, " STE at %#x [4]: 0x%016x\n", ste_addr, ste._pad[0]); 1378 DPRINTF(SMMUv3, " STE at %#x [5]: 0x%016x\n", ste_addr, ste._pad[1]); 1379 DPRINTF(SMMUv3, " STE at %#x [6]: 0x%016x\n", ste_addr, ste._pad[2]); 1380 DPRINTF(SMMUv3, " STE at %#x [7]: 0x%016x\n", ste_addr, ste._pad[3]); 1381 1382 if (!ste.dw0.valid) 1383 panic("STE @ %#x not valid\n", ste_addr); 1384 1385 smmu.steFetches++; 1386} 1387 1388void 1389SMMUTranslationProcess::doReadCD(Yield &yield, 1390 ContextDescriptor &cd, 1391 const StreamTableEntry &ste, 1392 uint32_t sid, uint32_t ssid) 1393{ 1394 Addr cd_addr; 1395 1396 if (ste.dw0.s1cdmax == 0) { 1397 cd_addr = ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT; 1398 } else { 1399 unsigned max_ssid = 1 << ste.dw0.s1cdmax; 1400 if (ssid >= max_ssid) 1401 panic("SSID %#x out of range, max=%#x", ssid, max_ssid); 1402 1403 if (ste.dw0.s1fmt==STAGE1_CFG_2L_4K || 1404 ste.dw0.s1fmt==STAGE1_CFG_2L_64K) 1405 { 1406 unsigned split = ste.dw0.s1fmt==STAGE1_CFG_2L_4K ? 7 : 11; 1407 1408 uint64_t l2_ptr; 1409 uint64_t l2_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + 1410 bits(ssid, 24, split) * sizeof(l2_ptr); 1411 1412 if (context.stage2Enable) 1413 l2_addr = translateStage2(yield, l2_addr, false).addr; 1414 1415 DPRINTF(SMMUv3, "Read L1CD at %#x\n", l2_addr); 1416 1417 doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, ssid); 1418 1419 DPRINTF(SMMUv3, "Got L1CD at %#x: 0x%016x\n", l2_addr, l2_ptr); 1420 1421 cd_addr = l2_ptr + bits(ssid, split-1, 0) * sizeof(cd); 1422 1423 smmu.cdL1Fetches++; 1424 } else if (ste.dw0.s1fmt == STAGE1_CFG_1L) { 1425 cd_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + ssid*sizeof(cd); 1426 } 1427 } 1428 1429 if (context.stage2Enable) 1430 cd_addr = translateStage2(yield, cd_addr, false).addr; 1431 1432 DPRINTF(SMMUv3, "Read CD at %#x\n", cd_addr); 1433 1434 doReadConfig(yield, cd_addr, &cd, sizeof(cd), sid, ssid); 1435 1436 DPRINTF(SMMUv3, "Got CD at %#x [0]: 0x%016x\n", cd_addr, cd.dw0); 1437 DPRINTF(SMMUv3, " CD at %#x [1]: 0x%016x\n", cd_addr, cd.dw1); 1438 DPRINTF(SMMUv3, " CD at %#x [2]: 0x%016x\n", cd_addr, cd.dw2); 1439 DPRINTF(SMMUv3, " CD at %#x [3]: 0x%016x\n", cd_addr, cd.mair); 1440 DPRINTF(SMMUv3, " CD at %#x [4]: 0x%016x\n", cd_addr, cd.amair); 1441 DPRINTF(SMMUv3, " CD at %#x [5]: 0x%016x\n", cd_addr, cd._pad[0]); 1442 DPRINTF(SMMUv3, " CD at %#x [6]: 0x%016x\n", cd_addr, cd._pad[1]); 1443 DPRINTF(SMMUv3, " CD at %#x [7]: 0x%016x\n", cd_addr, cd._pad[2]); 1444 1445 1446 if (!cd.dw0.valid) 1447 panic("CD @ %#x not valid\n", cd_addr); 1448 1449 smmu.cdFetches++; 1450} 1451 1452void 1453SMMUTranslationProcess::doReadConfig(Yield &yield, Addr addr, 1454 void *ptr, size_t size, 1455 uint32_t sid, uint32_t ssid) 1456{ 1457 doRead(yield, addr, ptr, size); 1458} 1459 1460void 1461SMMUTranslationProcess::doReadPTE(Yield &yield, Addr va, Addr addr, 1462 void *ptr, unsigned stage, 1463 unsigned level) 1464{ 1465 size_t pte_size = sizeof(PageTableOps::pte_t); 1466 1467 Addr mask = pte_size - 1; 1468 Addr base = addr & ~mask; 1469 1470 doRead(yield, base, ptr, pte_size); 1471} 1472