smmu_v3_transl.cc revision 14039
114039Sstacze01@arm.com/* 214039Sstacze01@arm.com * Copyright (c) 2013, 2018-2019 ARM Limited 314039Sstacze01@arm.com * All rights reserved 414039Sstacze01@arm.com * 514039Sstacze01@arm.com * The license below extends only to copyright in the software and shall 614039Sstacze01@arm.com * not be construed as granting a license to any other intellectual 714039Sstacze01@arm.com * property including but not limited to intellectual property relating 814039Sstacze01@arm.com * to a hardware implementation of the functionality of the software 914039Sstacze01@arm.com * licensed hereunder. You may use the software subject to the license 1014039Sstacze01@arm.com * terms below provided that you ensure that this notice is replicated 1114039Sstacze01@arm.com * unmodified and in its entirety in all distributions of the software, 1214039Sstacze01@arm.com * modified or unmodified, in source code or in binary form. 1314039Sstacze01@arm.com * 1414039Sstacze01@arm.com * Redistribution and use in source and binary forms, with or without 1514039Sstacze01@arm.com * modification, are permitted provided that the following conditions are 1614039Sstacze01@arm.com * met: redistributions of source code must retain the above copyright 1714039Sstacze01@arm.com * notice, this list of conditions and the following disclaimer; 1814039Sstacze01@arm.com * redistributions in binary form must reproduce the above copyright 1914039Sstacze01@arm.com * notice, this list of conditions and the following disclaimer in the 2014039Sstacze01@arm.com * documentation and/or other materials provided with the distribution; 2114039Sstacze01@arm.com * neither the name of the copyright holders nor the names of its 2214039Sstacze01@arm.com * contributors may be used to endorse or promote products derived from 2314039Sstacze01@arm.com * this software without specific prior written permission. 2414039Sstacze01@arm.com * 2514039Sstacze01@arm.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2614039Sstacze01@arm.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2714039Sstacze01@arm.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2814039Sstacze01@arm.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2914039Sstacze01@arm.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3014039Sstacze01@arm.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3114039Sstacze01@arm.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3214039Sstacze01@arm.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3314039Sstacze01@arm.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3414039Sstacze01@arm.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3514039Sstacze01@arm.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3614039Sstacze01@arm.com * 3714039Sstacze01@arm.com * Authors: Stan Czerniawski 3814039Sstacze01@arm.com */ 3914039Sstacze01@arm.com 4014039Sstacze01@arm.com#include "dev/arm/smmu_v3_transl.hh" 4114039Sstacze01@arm.com 4214039Sstacze01@arm.com#include "debug/SMMUv3.hh" 4314039Sstacze01@arm.com#include "debug/SMMUv3Hazard.hh" 4414039Sstacze01@arm.com#include "dev/arm/amba.hh" 4514039Sstacze01@arm.com#include "dev/arm/smmu_v3.hh" 4614039Sstacze01@arm.com#include "sim/system.hh" 4714039Sstacze01@arm.com 4814039Sstacze01@arm.comSMMUTranslRequest 4914039Sstacze01@arm.comSMMUTranslRequest::fromPacket(PacketPtr pkt, bool ats) 5014039Sstacze01@arm.com{ 5114039Sstacze01@arm.com SMMUTranslRequest req; 5214039Sstacze01@arm.com req.addr = pkt->getAddr(); 5314039Sstacze01@arm.com req.size = pkt->getSize(); 5414039Sstacze01@arm.com req.sid = pkt->req->streamId(); 5514039Sstacze01@arm.com req.ssid = pkt->req->hasSubstreamId() ? 5614039Sstacze01@arm.com pkt->req->substreamId() : 0; 5714039Sstacze01@arm.com req.isWrite = pkt->isWrite(); 5814039Sstacze01@arm.com req.isPrefetch = false; 5914039Sstacze01@arm.com req.isAtsRequest = ats; 6014039Sstacze01@arm.com req.pkt = pkt; 6114039Sstacze01@arm.com 6214039Sstacze01@arm.com return req; 6314039Sstacze01@arm.com} 6414039Sstacze01@arm.com 6514039Sstacze01@arm.comSMMUTranslRequest 6614039Sstacze01@arm.comSMMUTranslRequest::prefetch(Addr addr, uint32_t sid, uint32_t ssid) 6714039Sstacze01@arm.com{ 6814039Sstacze01@arm.com SMMUTranslRequest req; 6914039Sstacze01@arm.com req.addr = addr; 7014039Sstacze01@arm.com req.size = 0; 7114039Sstacze01@arm.com req.sid = sid; 7214039Sstacze01@arm.com req.ssid = ssid; 7314039Sstacze01@arm.com req.isWrite = false; 7414039Sstacze01@arm.com req.isPrefetch = true; 7514039Sstacze01@arm.com req.isAtsRequest = false; 7614039Sstacze01@arm.com req.pkt = NULL; 7714039Sstacze01@arm.com 7814039Sstacze01@arm.com return req; 7914039Sstacze01@arm.com} 8014039Sstacze01@arm.com 8114039Sstacze01@arm.comvoid 8214039Sstacze01@arm.comSMMUTranslationProcess::beginTransaction(const SMMUTranslRequest &req) 8314039Sstacze01@arm.com{ 8414039Sstacze01@arm.com request = req; 8514039Sstacze01@arm.com 8614039Sstacze01@arm.com reinit(); 8714039Sstacze01@arm.com} 8814039Sstacze01@arm.com 8914039Sstacze01@arm.comvoid 9014039Sstacze01@arm.comSMMUTranslationProcess::resumeTransaction() 9114039Sstacze01@arm.com{ 9214039Sstacze01@arm.com assert(smmu.system.isTimingMode()); 9314039Sstacze01@arm.com 9414039Sstacze01@arm.com assert(!"Stalls are broken"); 9514039Sstacze01@arm.com 9614039Sstacze01@arm.com Tick resumeTick = curTick(); 9714039Sstacze01@arm.com 9814039Sstacze01@arm.com (void) resumeTick; 9914039Sstacze01@arm.com DPRINTF(SMMUv3, "Resume at tick = %d. Fault duration = %d (%.3fus)\n", 10014039Sstacze01@arm.com resumeTick, resumeTick-faultTick, (resumeTick-faultTick) / 1e6); 10114039Sstacze01@arm.com 10214039Sstacze01@arm.com beginTransaction(request); 10314039Sstacze01@arm.com 10414039Sstacze01@arm.com smmu.runProcessTiming(this, request.pkt); 10514039Sstacze01@arm.com} 10614039Sstacze01@arm.com 10714039Sstacze01@arm.comvoid 10814039Sstacze01@arm.comSMMUTranslationProcess::main(Yield &yield) 10914039Sstacze01@arm.com{ 11014039Sstacze01@arm.com // Hack: 11114039Sstacze01@arm.com // The coroutine starts running as soon as it's created. 11214039Sstacze01@arm.com // But we need to wait for request data esp. in atomic mode. 11314039Sstacze01@arm.com SMMUAction a; 11414039Sstacze01@arm.com a.type = ACTION_INITIAL_NOP; 11514039Sstacze01@arm.com a.pkt = NULL; 11614039Sstacze01@arm.com yield(a); 11714039Sstacze01@arm.com 11814039Sstacze01@arm.com const Addr next4k = (request.addr + 0x1000ULL) & ~0xfffULL; 11914039Sstacze01@arm.com 12014039Sstacze01@arm.com if ((request.addr + request.size) > next4k) 12114039Sstacze01@arm.com panic("Transaction crosses 4k boundary (addr=%#x size=%#x)!\n", 12214039Sstacze01@arm.com request.addr, request.size); 12314039Sstacze01@arm.com 12414039Sstacze01@arm.com 12514039Sstacze01@arm.com unsigned numSlaveBeats = request.isWrite ? 12614039Sstacze01@arm.com (request.size + (ifc.portWidth - 1)) / ifc.portWidth : 1; 12714039Sstacze01@arm.com 12814039Sstacze01@arm.com doSemaphoreDown(yield, ifc.slavePortSem); 12914039Sstacze01@arm.com doDelay(yield, Cycles(numSlaveBeats)); 13014039Sstacze01@arm.com doSemaphoreUp(ifc.slavePortSem); 13114039Sstacze01@arm.com 13214039Sstacze01@arm.com 13314039Sstacze01@arm.com recvTick = curTick(); 13414039Sstacze01@arm.com 13514039Sstacze01@arm.com 13614039Sstacze01@arm.com if (!(smmu.regs.cr0 & 0x1)) { 13714039Sstacze01@arm.com // SMMU disabled 13814039Sstacze01@arm.com doDelay(yield, Cycles(1)); 13914039Sstacze01@arm.com completeTransaction(yield, bypass(request.addr)); 14014039Sstacze01@arm.com return; 14114039Sstacze01@arm.com } 14214039Sstacze01@arm.com 14314039Sstacze01@arm.com TranslResult tr; 14414039Sstacze01@arm.com bool wasPrefetched = false; 14514039Sstacze01@arm.com 14614039Sstacze01@arm.com if (request.isPrefetch) { 14714039Sstacze01@arm.com // Abort prefetch if: 14814039Sstacze01@arm.com // - there's already a transaction looking up the same 4k page, OR 14914039Sstacze01@arm.com // - requested address is already in the TLB. 15014039Sstacze01@arm.com if (hazard4kCheck() || ifcTLBLookup(yield, tr, wasPrefetched)) 15114039Sstacze01@arm.com completePrefetch(yield); // this never returns 15214039Sstacze01@arm.com 15314039Sstacze01@arm.com hazard4kRegister(); 15414039Sstacze01@arm.com 15514039Sstacze01@arm.com tr = smmuTranslation(yield); 15614039Sstacze01@arm.com 15714039Sstacze01@arm.com if (tr.fault == FAULT_NONE) 15814039Sstacze01@arm.com ifcTLBUpdate(yield, tr); 15914039Sstacze01@arm.com 16014039Sstacze01@arm.com hazard4kRelease(); 16114039Sstacze01@arm.com 16214039Sstacze01@arm.com completePrefetch(yield); 16314039Sstacze01@arm.com } else { 16414039Sstacze01@arm.com hazardIdRegister(); 16514039Sstacze01@arm.com 16614039Sstacze01@arm.com if (!microTLBLookup(yield, tr)) { 16714039Sstacze01@arm.com bool hit = ifcTLBLookup(yield, tr, wasPrefetched); 16814039Sstacze01@arm.com if (!hit) { 16914039Sstacze01@arm.com while (!hit && hazard4kCheck()) { 17014039Sstacze01@arm.com hazard4kHold(yield); 17114039Sstacze01@arm.com hit = ifcTLBLookup(yield, tr, wasPrefetched); 17214039Sstacze01@arm.com } 17314039Sstacze01@arm.com } 17414039Sstacze01@arm.com 17514039Sstacze01@arm.com // Issue prefetch if: 17614039Sstacze01@arm.com // - there was a TLB hit and the entry was prefetched, OR 17714039Sstacze01@arm.com // - TLB miss was successfully serviced 17814039Sstacze01@arm.com if (hit) { 17914039Sstacze01@arm.com if (wasPrefetched) 18014039Sstacze01@arm.com issuePrefetch(next4k); 18114039Sstacze01@arm.com } else { 18214039Sstacze01@arm.com hazard4kRegister(); 18314039Sstacze01@arm.com 18414039Sstacze01@arm.com tr = smmuTranslation(yield); 18514039Sstacze01@arm.com 18614039Sstacze01@arm.com if (tr.fault == FAULT_NONE) { 18714039Sstacze01@arm.com ifcTLBUpdate(yield, tr); 18814039Sstacze01@arm.com 18914039Sstacze01@arm.com issuePrefetch(next4k); 19014039Sstacze01@arm.com } 19114039Sstacze01@arm.com 19214039Sstacze01@arm.com hazard4kRelease(); 19314039Sstacze01@arm.com } 19414039Sstacze01@arm.com 19514039Sstacze01@arm.com if (tr.fault == FAULT_NONE) 19614039Sstacze01@arm.com microTLBUpdate(yield, tr); 19714039Sstacze01@arm.com } 19814039Sstacze01@arm.com 19914039Sstacze01@arm.com hazardIdHold(yield); 20014039Sstacze01@arm.com hazardIdRelease(); 20114039Sstacze01@arm.com 20214039Sstacze01@arm.com if (tr.fault != FAULT_NONE) 20314039Sstacze01@arm.com panic("fault\n"); 20414039Sstacze01@arm.com 20514039Sstacze01@arm.com completeTransaction(yield, tr); 20614039Sstacze01@arm.com } 20714039Sstacze01@arm.com} 20814039Sstacze01@arm.com 20914039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 21014039Sstacze01@arm.comSMMUTranslationProcess::bypass(Addr addr) const 21114039Sstacze01@arm.com{ 21214039Sstacze01@arm.com TranslResult tr; 21314039Sstacze01@arm.com tr.fault = FAULT_NONE; 21414039Sstacze01@arm.com tr.addr = addr; 21514039Sstacze01@arm.com tr.addrMask = 0; 21614039Sstacze01@arm.com tr.writable = 1; 21714039Sstacze01@arm.com 21814039Sstacze01@arm.com return tr; 21914039Sstacze01@arm.com} 22014039Sstacze01@arm.com 22114039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 22214039Sstacze01@arm.comSMMUTranslationProcess::smmuTranslation(Yield &yield) 22314039Sstacze01@arm.com{ 22414039Sstacze01@arm.com TranslResult tr; 22514039Sstacze01@arm.com 22614039Sstacze01@arm.com // Need SMMU credit to proceed 22714039Sstacze01@arm.com doSemaphoreDown(yield, smmu.transSem); 22814039Sstacze01@arm.com 22914039Sstacze01@arm.com // Simulate pipelined IFC->SMMU link 23014039Sstacze01@arm.com doSemaphoreDown(yield, smmu.ifcSmmuSem); 23114039Sstacze01@arm.com doDelay(yield, Cycles(1)); // serialize transactions 23214039Sstacze01@arm.com doSemaphoreUp(smmu.ifcSmmuSem); 23314039Sstacze01@arm.com doDelay(yield, smmu.ifcSmmuLat - Cycles(1)); // remaining pipeline delay 23414039Sstacze01@arm.com 23514039Sstacze01@arm.com bool haveConfig = true; 23614039Sstacze01@arm.com if (!configCacheLookup(yield, context)) { 23714039Sstacze01@arm.com if(findConfig(yield, context, tr)) { 23814039Sstacze01@arm.com configCacheUpdate(yield, context); 23914039Sstacze01@arm.com } else { 24014039Sstacze01@arm.com haveConfig = false; 24114039Sstacze01@arm.com } 24214039Sstacze01@arm.com } 24314039Sstacze01@arm.com 24414039Sstacze01@arm.com if (haveConfig && !smmuTLBLookup(yield, tr)) { 24514039Sstacze01@arm.com // SMMU main TLB miss 24614039Sstacze01@arm.com 24714039Sstacze01@arm.com // Need PTW slot to proceed 24814039Sstacze01@arm.com doSemaphoreDown(yield, smmu.ptwSem); 24914039Sstacze01@arm.com 25014039Sstacze01@arm.com // Page table walk 25114039Sstacze01@arm.com Tick ptwStartTick = curTick(); 25214039Sstacze01@arm.com 25314039Sstacze01@arm.com if (context.stage1Enable) { 25414039Sstacze01@arm.com tr = translateStage1And2(yield, request.addr); 25514039Sstacze01@arm.com } else if (context.stage2Enable) { 25614039Sstacze01@arm.com tr = translateStage2(yield, request.addr, true); 25714039Sstacze01@arm.com } else { 25814039Sstacze01@arm.com tr = bypass(request.addr); 25914039Sstacze01@arm.com } 26014039Sstacze01@arm.com 26114039Sstacze01@arm.com if (context.stage1Enable || context.stage2Enable) 26214039Sstacze01@arm.com smmu.ptwTimeDist.sample(curTick() - ptwStartTick); 26314039Sstacze01@arm.com 26414039Sstacze01@arm.com // Free PTW slot 26514039Sstacze01@arm.com doSemaphoreUp(smmu.ptwSem); 26614039Sstacze01@arm.com 26714039Sstacze01@arm.com if (tr.fault == FAULT_NONE) 26814039Sstacze01@arm.com smmuTLBUpdate(yield, tr); 26914039Sstacze01@arm.com } 27014039Sstacze01@arm.com 27114039Sstacze01@arm.com // Simulate pipelined SMMU->SLAVE INTERFACE link 27214039Sstacze01@arm.com doSemaphoreDown(yield, smmu.smmuIfcSem); 27314039Sstacze01@arm.com doDelay(yield, Cycles(1)); // serialize transactions 27414039Sstacze01@arm.com doSemaphoreUp(smmu.smmuIfcSem); 27514039Sstacze01@arm.com doDelay(yield, smmu.smmuIfcLat - Cycles(1)); // remaining pipeline delay 27614039Sstacze01@arm.com 27714039Sstacze01@arm.com // return SMMU credit 27814039Sstacze01@arm.com doSemaphoreUp(smmu.transSem); 27914039Sstacze01@arm.com 28014039Sstacze01@arm.com return tr; 28114039Sstacze01@arm.com} 28214039Sstacze01@arm.com 28314039Sstacze01@arm.combool 28414039Sstacze01@arm.comSMMUTranslationProcess::microTLBLookup(Yield &yield, TranslResult &tr) 28514039Sstacze01@arm.com{ 28614039Sstacze01@arm.com if (!ifc.microTLBEnable) 28714039Sstacze01@arm.com return false; 28814039Sstacze01@arm.com 28914039Sstacze01@arm.com doSemaphoreDown(yield, ifc.microTLBSem); 29014039Sstacze01@arm.com doDelay(yield, ifc.microTLBLat); 29114039Sstacze01@arm.com const SMMUTLB::Entry *e = 29214039Sstacze01@arm.com ifc.microTLB->lookup(request.sid, request.ssid, request.addr); 29314039Sstacze01@arm.com doSemaphoreUp(ifc.microTLBSem); 29414039Sstacze01@arm.com 29514039Sstacze01@arm.com if (!e) { 29614039Sstacze01@arm.com DPRINTF(SMMUv3, "micro TLB miss vaddr=%#x sid=%#x ssid=%#x\n", 29714039Sstacze01@arm.com request.addr, request.sid, request.ssid); 29814039Sstacze01@arm.com 29914039Sstacze01@arm.com return false; 30014039Sstacze01@arm.com } 30114039Sstacze01@arm.com 30214039Sstacze01@arm.com DPRINTF(SMMUv3, 30314039Sstacze01@arm.com "micro TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x paddr=%#x\n", 30414039Sstacze01@arm.com request.addr, e->vaMask, request.sid, request.ssid, e->pa); 30514039Sstacze01@arm.com 30614039Sstacze01@arm.com tr.fault = FAULT_NONE; 30714039Sstacze01@arm.com tr.addr = e->pa + (request.addr & ~e->vaMask);; 30814039Sstacze01@arm.com tr.addrMask = e->vaMask; 30914039Sstacze01@arm.com tr.writable = e->permissions; 31014039Sstacze01@arm.com 31114039Sstacze01@arm.com return true; 31214039Sstacze01@arm.com} 31314039Sstacze01@arm.com 31414039Sstacze01@arm.combool 31514039Sstacze01@arm.comSMMUTranslationProcess::ifcTLBLookup(Yield &yield, TranslResult &tr, 31614039Sstacze01@arm.com bool &wasPrefetched) 31714039Sstacze01@arm.com{ 31814039Sstacze01@arm.com if (!ifc.mainTLBEnable) 31914039Sstacze01@arm.com return false; 32014039Sstacze01@arm.com 32114039Sstacze01@arm.com doSemaphoreDown(yield, ifc.mainTLBSem); 32214039Sstacze01@arm.com doDelay(yield, ifc.mainTLBLat); 32314039Sstacze01@arm.com const SMMUTLB::Entry *e = 32414039Sstacze01@arm.com ifc.mainTLB->lookup(request.sid, request.ssid, request.addr); 32514039Sstacze01@arm.com doSemaphoreUp(ifc.mainTLBSem); 32614039Sstacze01@arm.com 32714039Sstacze01@arm.com if (!e) { 32814039Sstacze01@arm.com DPRINTF(SMMUv3, 32914039Sstacze01@arm.com "SLAVE Interface TLB miss vaddr=%#x sid=%#x ssid=%#x\n", 33014039Sstacze01@arm.com request.addr, request.sid, request.ssid); 33114039Sstacze01@arm.com 33214039Sstacze01@arm.com return false; 33314039Sstacze01@arm.com } 33414039Sstacze01@arm.com 33514039Sstacze01@arm.com DPRINTF(SMMUv3, 33614039Sstacze01@arm.com "SLAVE Interface TLB hit vaddr=%#x amask=%#x sid=%#x ssid=%#x " 33714039Sstacze01@arm.com "paddr=%#x\n", request.addr, e->vaMask, request.sid, 33814039Sstacze01@arm.com request.ssid, e->pa); 33914039Sstacze01@arm.com 34014039Sstacze01@arm.com tr.fault = FAULT_NONE; 34114039Sstacze01@arm.com tr.addr = e->pa + (request.addr & ~e->vaMask);; 34214039Sstacze01@arm.com tr.addrMask = e->vaMask; 34314039Sstacze01@arm.com tr.writable = e->permissions; 34414039Sstacze01@arm.com wasPrefetched = e->prefetched; 34514039Sstacze01@arm.com 34614039Sstacze01@arm.com return true; 34714039Sstacze01@arm.com} 34814039Sstacze01@arm.com 34914039Sstacze01@arm.combool 35014039Sstacze01@arm.comSMMUTranslationProcess::smmuTLBLookup(Yield &yield, TranslResult &tr) 35114039Sstacze01@arm.com{ 35214039Sstacze01@arm.com if (!smmu.tlbEnable) 35314039Sstacze01@arm.com return false; 35414039Sstacze01@arm.com 35514039Sstacze01@arm.com doSemaphoreDown(yield, smmu.tlbSem); 35614039Sstacze01@arm.com doDelay(yield, smmu.tlbLat); 35714039Sstacze01@arm.com const ARMArchTLB::Entry *e = 35814039Sstacze01@arm.com smmu.tlb.lookup(request.addr, context.asid, context.vmid); 35914039Sstacze01@arm.com doSemaphoreUp(smmu.tlbSem); 36014039Sstacze01@arm.com 36114039Sstacze01@arm.com if (!e) { 36214039Sstacze01@arm.com DPRINTF(SMMUv3, "SMMU TLB miss vaddr=%#x asid=%#x vmid=%#x\n", 36314039Sstacze01@arm.com request.addr, context.asid, context.vmid); 36414039Sstacze01@arm.com 36514039Sstacze01@arm.com return false; 36614039Sstacze01@arm.com } 36714039Sstacze01@arm.com 36814039Sstacze01@arm.com DPRINTF(SMMUv3, 36914039Sstacze01@arm.com "SMMU TLB hit vaddr=%#x amask=%#x asid=%#x vmid=%#x paddr=%#x\n", 37014039Sstacze01@arm.com request.addr, e->vaMask, context.asid, context.vmid, e->pa); 37114039Sstacze01@arm.com 37214039Sstacze01@arm.com tr.fault = FAULT_NONE; 37314039Sstacze01@arm.com tr.addr = e->pa + (request.addr & ~e->vaMask);; 37414039Sstacze01@arm.com tr.addrMask = e->vaMask; 37514039Sstacze01@arm.com tr.writable = e->permissions; 37614039Sstacze01@arm.com 37714039Sstacze01@arm.com return true; 37814039Sstacze01@arm.com} 37914039Sstacze01@arm.com 38014039Sstacze01@arm.comvoid 38114039Sstacze01@arm.comSMMUTranslationProcess::microTLBUpdate(Yield &yield, 38214039Sstacze01@arm.com const TranslResult &tr) 38314039Sstacze01@arm.com{ 38414039Sstacze01@arm.com assert(tr.fault == FAULT_NONE); 38514039Sstacze01@arm.com 38614039Sstacze01@arm.com if (!ifc.microTLBEnable) 38714039Sstacze01@arm.com return; 38814039Sstacze01@arm.com 38914039Sstacze01@arm.com SMMUTLB::Entry e; 39014039Sstacze01@arm.com e.valid = true; 39114039Sstacze01@arm.com e.prefetched = false; 39214039Sstacze01@arm.com e.sid = request.sid; 39314039Sstacze01@arm.com e.ssid = request.ssid; 39414039Sstacze01@arm.com e.vaMask = tr.addrMask; 39514039Sstacze01@arm.com e.va = request.addr & e.vaMask; 39614039Sstacze01@arm.com e.pa = tr.addr & e.vaMask; 39714039Sstacze01@arm.com e.permissions = tr.writable; 39814039Sstacze01@arm.com e.asid = context.asid; 39914039Sstacze01@arm.com e.vmid = context.vmid; 40014039Sstacze01@arm.com 40114039Sstacze01@arm.com doSemaphoreDown(yield, ifc.microTLBSem); 40214039Sstacze01@arm.com 40314039Sstacze01@arm.com DPRINTF(SMMUv3, 40414039Sstacze01@arm.com "micro TLB upd vaddr=%#x amask=%#x paddr=%#x sid=%#x ssid=%#x\n", 40514039Sstacze01@arm.com e.va, e.vaMask, e.pa, e.sid, e.ssid); 40614039Sstacze01@arm.com 40714039Sstacze01@arm.com ifc.microTLB->store(e, SMMUTLB::ALLOC_ANY_WAY); 40814039Sstacze01@arm.com 40914039Sstacze01@arm.com doSemaphoreUp(ifc.microTLBSem); 41014039Sstacze01@arm.com} 41114039Sstacze01@arm.com 41214039Sstacze01@arm.comvoid 41314039Sstacze01@arm.comSMMUTranslationProcess::ifcTLBUpdate(Yield &yield, 41414039Sstacze01@arm.com const TranslResult &tr) 41514039Sstacze01@arm.com{ 41614039Sstacze01@arm.com assert(tr.fault == FAULT_NONE); 41714039Sstacze01@arm.com 41814039Sstacze01@arm.com if (!ifc.mainTLBEnable) 41914039Sstacze01@arm.com return; 42014039Sstacze01@arm.com 42114039Sstacze01@arm.com SMMUTLB::Entry e; 42214039Sstacze01@arm.com e.valid = true; 42314039Sstacze01@arm.com e.prefetched = request.isPrefetch; 42414039Sstacze01@arm.com e.sid = request.sid; 42514039Sstacze01@arm.com e.ssid = request.ssid; 42614039Sstacze01@arm.com e.vaMask = tr.addrMask; 42714039Sstacze01@arm.com e.va = request.addr & e.vaMask; 42814039Sstacze01@arm.com e.pa = tr.addr & e.vaMask; 42914039Sstacze01@arm.com e.permissions = tr.writable; 43014039Sstacze01@arm.com e.asid = context.asid; 43114039Sstacze01@arm.com e.vmid = context.vmid; 43214039Sstacze01@arm.com 43314039Sstacze01@arm.com SMMUTLB::AllocPolicy alloc = SMMUTLB::ALLOC_ANY_WAY; 43414039Sstacze01@arm.com if (ifc.prefetchEnable && ifc.prefetchReserveLastWay) 43514039Sstacze01@arm.com alloc = request.isPrefetch ? 43614039Sstacze01@arm.com SMMUTLB::ALLOC_LAST_WAY : SMMUTLB::ALLOC_ANY_BUT_LAST_WAY; 43714039Sstacze01@arm.com 43814039Sstacze01@arm.com doSemaphoreDown(yield, ifc.mainTLBSem); 43914039Sstacze01@arm.com 44014039Sstacze01@arm.com DPRINTF(SMMUv3, 44114039Sstacze01@arm.com "SLAVE Interface upd vaddr=%#x amask=%#x paddr=%#x sid=%#x " 44214039Sstacze01@arm.com "ssid=%#x\n", e.va, e.vaMask, e.pa, e.sid, e.ssid); 44314039Sstacze01@arm.com 44414039Sstacze01@arm.com ifc.mainTLB->store(e, alloc); 44514039Sstacze01@arm.com 44614039Sstacze01@arm.com doSemaphoreUp(ifc.mainTLBSem); 44714039Sstacze01@arm.com} 44814039Sstacze01@arm.com 44914039Sstacze01@arm.comvoid 45014039Sstacze01@arm.comSMMUTranslationProcess::smmuTLBUpdate(Yield &yield, 45114039Sstacze01@arm.com const TranslResult &tr) 45214039Sstacze01@arm.com{ 45314039Sstacze01@arm.com assert(tr.fault == FAULT_NONE); 45414039Sstacze01@arm.com 45514039Sstacze01@arm.com if (!smmu.tlbEnable) 45614039Sstacze01@arm.com return; 45714039Sstacze01@arm.com 45814039Sstacze01@arm.com ARMArchTLB::Entry e; 45914039Sstacze01@arm.com e.valid = true; 46014039Sstacze01@arm.com e.vaMask = tr.addrMask; 46114039Sstacze01@arm.com e.va = request.addr & e.vaMask; 46214039Sstacze01@arm.com e.asid = context.asid; 46314039Sstacze01@arm.com e.vmid = context.vmid; 46414039Sstacze01@arm.com e.pa = tr.addr & e.vaMask; 46514039Sstacze01@arm.com e.permissions = tr.writable; 46614039Sstacze01@arm.com 46714039Sstacze01@arm.com doSemaphoreDown(yield, smmu.tlbSem); 46814039Sstacze01@arm.com 46914039Sstacze01@arm.com DPRINTF(SMMUv3, 47014039Sstacze01@arm.com "SMMU TLB upd vaddr=%#x amask=%#x paddr=%#x asid=%#x vmid=%#x\n", 47114039Sstacze01@arm.com e.va, e.vaMask, e.pa, e.asid, e.vmid); 47214039Sstacze01@arm.com 47314039Sstacze01@arm.com smmu.tlb.store(e); 47414039Sstacze01@arm.com 47514039Sstacze01@arm.com doSemaphoreUp(smmu.tlbSem); 47614039Sstacze01@arm.com} 47714039Sstacze01@arm.com 47814039Sstacze01@arm.combool 47914039Sstacze01@arm.comSMMUTranslationProcess::configCacheLookup(Yield &yield, TranslContext &tc) 48014039Sstacze01@arm.com{ 48114039Sstacze01@arm.com if (!smmu.configCacheEnable) 48214039Sstacze01@arm.com return false; 48314039Sstacze01@arm.com 48414039Sstacze01@arm.com doSemaphoreDown(yield, smmu.configSem); 48514039Sstacze01@arm.com doDelay(yield, smmu.configLat); 48614039Sstacze01@arm.com const ConfigCache::Entry *e = 48714039Sstacze01@arm.com smmu.configCache.lookup(request.sid, request.ssid); 48814039Sstacze01@arm.com doSemaphoreUp(smmu.configSem); 48914039Sstacze01@arm.com 49014039Sstacze01@arm.com if (!e) { 49114039Sstacze01@arm.com DPRINTF(SMMUv3, "Config miss sid=%#x ssid=%#x\n", 49214039Sstacze01@arm.com request.sid, request.ssid); 49314039Sstacze01@arm.com 49414039Sstacze01@arm.com return false; 49514039Sstacze01@arm.com } 49614039Sstacze01@arm.com 49714039Sstacze01@arm.com DPRINTF(SMMUv3, "Config hit sid=%#x ssid=%#x ttb=%#08x asid=%#x\n", 49814039Sstacze01@arm.com request.sid, request.ssid, e->ttb0, e->asid); 49914039Sstacze01@arm.com 50014039Sstacze01@arm.com tc.stage1Enable = e->stage1_en; 50114039Sstacze01@arm.com tc.stage2Enable = e->stage2_en; 50214039Sstacze01@arm.com 50314039Sstacze01@arm.com tc.ttb0 = e->ttb0; 50414039Sstacze01@arm.com tc.ttb1 = e->ttb1; 50514039Sstacze01@arm.com tc.asid = e->asid; 50614039Sstacze01@arm.com tc.httb = e->httb; 50714039Sstacze01@arm.com tc.vmid = e->vmid; 50814039Sstacze01@arm.com 50914039Sstacze01@arm.com tc.stage1TranslGranule = e->stage1_tg; 51014039Sstacze01@arm.com tc.stage2TranslGranule = e->stage2_tg; 51114039Sstacze01@arm.com 51214039Sstacze01@arm.com return true; 51314039Sstacze01@arm.com} 51414039Sstacze01@arm.com 51514039Sstacze01@arm.comvoid 51614039Sstacze01@arm.comSMMUTranslationProcess::configCacheUpdate(Yield &yield, 51714039Sstacze01@arm.com const TranslContext &tc) 51814039Sstacze01@arm.com{ 51914039Sstacze01@arm.com if (!smmu.configCacheEnable) 52014039Sstacze01@arm.com return; 52114039Sstacze01@arm.com 52214039Sstacze01@arm.com ConfigCache::Entry e; 52314039Sstacze01@arm.com e.valid = true; 52414039Sstacze01@arm.com e.sid = request.sid; 52514039Sstacze01@arm.com e.ssid = request.ssid; 52614039Sstacze01@arm.com e.stage1_en = tc.stage1Enable; 52714039Sstacze01@arm.com e.stage2_en = tc.stage2Enable; 52814039Sstacze01@arm.com e.ttb0 = tc.ttb0; 52914039Sstacze01@arm.com e.ttb1 = tc.ttb1; 53014039Sstacze01@arm.com e.asid = tc.asid; 53114039Sstacze01@arm.com e.httb = tc.httb; 53214039Sstacze01@arm.com e.vmid = tc.vmid; 53314039Sstacze01@arm.com e.stage1_tg = tc.stage1TranslGranule; 53414039Sstacze01@arm.com e.stage2_tg = tc.stage2TranslGranule; 53514039Sstacze01@arm.com 53614039Sstacze01@arm.com doSemaphoreDown(yield, smmu.configSem); 53714039Sstacze01@arm.com 53814039Sstacze01@arm.com DPRINTF(SMMUv3, "Config upd sid=%#x ssid=%#x\n", e.sid, e.ssid); 53914039Sstacze01@arm.com 54014039Sstacze01@arm.com smmu.configCache.store(e); 54114039Sstacze01@arm.com 54214039Sstacze01@arm.com doSemaphoreUp(smmu.configSem); 54314039Sstacze01@arm.com} 54414039Sstacze01@arm.com 54514039Sstacze01@arm.combool 54614039Sstacze01@arm.comSMMUTranslationProcess::findConfig(Yield &yield, 54714039Sstacze01@arm.com TranslContext &tc, 54814039Sstacze01@arm.com TranslResult &tr) 54914039Sstacze01@arm.com{ 55014039Sstacze01@arm.com tc.stage1Enable = false; 55114039Sstacze01@arm.com tc.stage2Enable = false; 55214039Sstacze01@arm.com 55314039Sstacze01@arm.com StreamTableEntry ste; 55414039Sstacze01@arm.com doReadSTE(yield, ste, request.sid); 55514039Sstacze01@arm.com 55614039Sstacze01@arm.com switch (ste.dw0.config) { 55714039Sstacze01@arm.com case STE_CONFIG_BYPASS: 55814039Sstacze01@arm.com break; 55914039Sstacze01@arm.com 56014039Sstacze01@arm.com case STE_CONFIG_STAGE1_ONLY: 56114039Sstacze01@arm.com tc.stage1Enable = true; 56214039Sstacze01@arm.com break; 56314039Sstacze01@arm.com 56414039Sstacze01@arm.com case STE_CONFIG_STAGE2_ONLY: 56514039Sstacze01@arm.com tc.stage2Enable = true; 56614039Sstacze01@arm.com break; 56714039Sstacze01@arm.com 56814039Sstacze01@arm.com case STE_CONFIG_STAGE1_AND_2: 56914039Sstacze01@arm.com tc.stage1Enable = true; 57014039Sstacze01@arm.com tc.stage2Enable = true; 57114039Sstacze01@arm.com break; 57214039Sstacze01@arm.com 57314039Sstacze01@arm.com default: 57414039Sstacze01@arm.com panic("Bad or unimplemented STE config %d\n", 57514039Sstacze01@arm.com ste.dw0.config); 57614039Sstacze01@arm.com } 57714039Sstacze01@arm.com 57814039Sstacze01@arm.com 57914039Sstacze01@arm.com // Establish stage 2 context first since 58014039Sstacze01@arm.com // Context Descriptors can be in IPA space. 58114039Sstacze01@arm.com if (tc.stage2Enable) { 58214039Sstacze01@arm.com tc.httb = ste.dw3.s2ttb << STE_S2TTB_SHIFT; 58314039Sstacze01@arm.com tc.vmid = ste.dw2.s2vmid; 58414039Sstacze01@arm.com tc.stage2TranslGranule = ste.dw2.s2tg; 58514039Sstacze01@arm.com } else { 58614039Sstacze01@arm.com tc.httb = 0xdeadbeef; 58714039Sstacze01@arm.com tc.vmid = 0; 58814039Sstacze01@arm.com tc.stage2TranslGranule = TRANS_GRANULE_INVALID; 58914039Sstacze01@arm.com } 59014039Sstacze01@arm.com 59114039Sstacze01@arm.com 59214039Sstacze01@arm.com // Now fetch stage 1 config. 59314039Sstacze01@arm.com if (context.stage1Enable) { 59414039Sstacze01@arm.com ContextDescriptor cd; 59514039Sstacze01@arm.com doReadCD(yield, cd, ste, request.sid, request.ssid); 59614039Sstacze01@arm.com 59714039Sstacze01@arm.com tc.ttb0 = cd.dw1.ttb0 << CD_TTB_SHIFT; 59814039Sstacze01@arm.com tc.ttb1 = cd.dw2.ttb1 << CD_TTB_SHIFT; 59914039Sstacze01@arm.com tc.asid = cd.dw0.asid; 60014039Sstacze01@arm.com tc.stage1TranslGranule = cd.dw0.tg0; 60114039Sstacze01@arm.com } else { 60214039Sstacze01@arm.com tc.ttb0 = 0xcafebabe; 60314039Sstacze01@arm.com tc.ttb1 = 0xcafed00d; 60414039Sstacze01@arm.com tc.asid = 0; 60514039Sstacze01@arm.com tc.stage1TranslGranule = TRANS_GRANULE_INVALID; 60614039Sstacze01@arm.com } 60714039Sstacze01@arm.com 60814039Sstacze01@arm.com return true; 60914039Sstacze01@arm.com} 61014039Sstacze01@arm.com 61114039Sstacze01@arm.comvoid 61214039Sstacze01@arm.comSMMUTranslationProcess::walkCacheLookup( 61314039Sstacze01@arm.com Yield &yield, 61414039Sstacze01@arm.com const WalkCache::Entry *&walkEntry, 61514039Sstacze01@arm.com Addr addr, uint16_t asid, uint16_t vmid, 61614039Sstacze01@arm.com unsigned stage, unsigned level) 61714039Sstacze01@arm.com{ 61814039Sstacze01@arm.com const char *indent = stage==2 ? " " : ""; 61914039Sstacze01@arm.com (void) indent; // this is only used in DPRINTFs 62014039Sstacze01@arm.com 62114039Sstacze01@arm.com const PageTableOps *pt_ops = 62214039Sstacze01@arm.com stage == 1 ? 62314039Sstacze01@arm.com smmu.getPageTableOps(context.stage1TranslGranule) : 62414039Sstacze01@arm.com smmu.getPageTableOps(context.stage2TranslGranule); 62514039Sstacze01@arm.com 62614039Sstacze01@arm.com unsigned walkCacheLevels = 62714039Sstacze01@arm.com smmu.walkCacheEnable ? 62814039Sstacze01@arm.com (stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels) : 62914039Sstacze01@arm.com 0; 63014039Sstacze01@arm.com 63114039Sstacze01@arm.com if ((1 << level) & walkCacheLevels) { 63214039Sstacze01@arm.com doSemaphoreDown(yield, smmu.walkSem); 63314039Sstacze01@arm.com doDelay(yield, smmu.walkLat); 63414039Sstacze01@arm.com 63514039Sstacze01@arm.com walkEntry = smmu.walkCache.lookup(addr, pt_ops->walkMask(level), 63614039Sstacze01@arm.com asid, vmid, stage, level); 63714039Sstacze01@arm.com 63814039Sstacze01@arm.com if (walkEntry) { 63914039Sstacze01@arm.com DPRINTF(SMMUv3, "%sWalkCache hit va=%#x asid=%#x vmid=%#x " 64014039Sstacze01@arm.com "base=%#x (S%d, L%d)\n", 64114039Sstacze01@arm.com indent, addr, asid, vmid, walkEntry->pa, stage, level); 64214039Sstacze01@arm.com } else { 64314039Sstacze01@arm.com DPRINTF(SMMUv3, "%sWalkCache miss va=%#x asid=%#x vmid=%#x " 64414039Sstacze01@arm.com "(S%d, L%d)\n", 64514039Sstacze01@arm.com indent, addr, asid, vmid, stage, level); 64614039Sstacze01@arm.com } 64714039Sstacze01@arm.com 64814039Sstacze01@arm.com doSemaphoreUp(smmu.walkSem); 64914039Sstacze01@arm.com } 65014039Sstacze01@arm.com} 65114039Sstacze01@arm.com 65214039Sstacze01@arm.comvoid 65314039Sstacze01@arm.comSMMUTranslationProcess::walkCacheUpdate(Yield &yield, Addr va, 65414039Sstacze01@arm.com Addr vaMask, Addr pa, 65514039Sstacze01@arm.com unsigned stage, unsigned level, 65614039Sstacze01@arm.com bool leaf, uint8_t permissions) 65714039Sstacze01@arm.com{ 65814039Sstacze01@arm.com unsigned walkCacheLevels = 65914039Sstacze01@arm.com stage == 1 ? smmu.walkCacheS1Levels : smmu.walkCacheS2Levels; 66014039Sstacze01@arm.com 66114039Sstacze01@arm.com if (smmu.walkCacheEnable && ((1<<level) & walkCacheLevels)) { 66214039Sstacze01@arm.com WalkCache::Entry e; 66314039Sstacze01@arm.com e.valid = true; 66414039Sstacze01@arm.com e.va = va; 66514039Sstacze01@arm.com e.vaMask = vaMask; 66614039Sstacze01@arm.com e.asid = stage==1 ? context.asid : 0; 66714039Sstacze01@arm.com e.vmid = context.vmid; 66814039Sstacze01@arm.com e.stage = stage; 66914039Sstacze01@arm.com e.level = level; 67014039Sstacze01@arm.com e.leaf = leaf; 67114039Sstacze01@arm.com e.pa = pa; 67214039Sstacze01@arm.com e.permissions = permissions; 67314039Sstacze01@arm.com 67414039Sstacze01@arm.com doSemaphoreDown(yield, smmu.walkSem); 67514039Sstacze01@arm.com 67614039Sstacze01@arm.com DPRINTF(SMMUv3, "%sWalkCache upd va=%#x mask=%#x asid=%#x vmid=%#x " 67714039Sstacze01@arm.com "tpa=%#x leaf=%s (S%d, L%d)\n", 67814039Sstacze01@arm.com e.stage==2 ? " " : "", 67914039Sstacze01@arm.com e.va, e.vaMask, e.asid, e.vmid, 68014039Sstacze01@arm.com e.pa, e.leaf, e.stage, e.level); 68114039Sstacze01@arm.com 68214039Sstacze01@arm.com smmu.walkCache.store(e); 68314039Sstacze01@arm.com 68414039Sstacze01@arm.com doSemaphoreUp(smmu.walkSem); 68514039Sstacze01@arm.com } 68614039Sstacze01@arm.com} 68714039Sstacze01@arm.com 68814039Sstacze01@arm.com/* 68914039Sstacze01@arm.com * Please note: 69014039Sstacze01@arm.com * This does not deal with the case where stage 1 page size 69114039Sstacze01@arm.com * is larger than stage 2 page size. 69214039Sstacze01@arm.com */ 69314039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 69414039Sstacze01@arm.comSMMUTranslationProcess::walkStage1And2(Yield &yield, Addr addr, 69514039Sstacze01@arm.com const PageTableOps *pt_ops, 69614039Sstacze01@arm.com unsigned level, Addr walkPtr) 69714039Sstacze01@arm.com{ 69814039Sstacze01@arm.com PageTableOps::pte_t pte = 0; 69914039Sstacze01@arm.com 70014039Sstacze01@arm.com doSemaphoreDown(yield, smmu.cycleSem); 70114039Sstacze01@arm.com doDelay(yield, Cycles(1)); 70214039Sstacze01@arm.com doSemaphoreUp(smmu.cycleSem); 70314039Sstacze01@arm.com 70414039Sstacze01@arm.com for (; level <= pt_ops->lastLevel(); level++) { 70514039Sstacze01@arm.com Addr pte_addr = walkPtr + pt_ops->index(addr, level); 70614039Sstacze01@arm.com 70714039Sstacze01@arm.com DPRINTF(SMMUv3, "Fetching S1 L%d PTE from pa=%#08x\n", 70814039Sstacze01@arm.com level, pte_addr); 70914039Sstacze01@arm.com 71014039Sstacze01@arm.com doReadPTE(yield, addr, pte_addr, &pte, 1, level); 71114039Sstacze01@arm.com 71214039Sstacze01@arm.com DPRINTF(SMMUv3, "Got S1 L%d PTE=%#x from pa=%#08x\n", 71314039Sstacze01@arm.com level, pte, pte_addr); 71414039Sstacze01@arm.com 71514039Sstacze01@arm.com doSemaphoreDown(yield, smmu.cycleSem); 71614039Sstacze01@arm.com doDelay(yield, Cycles(1)); 71714039Sstacze01@arm.com doSemaphoreUp(smmu.cycleSem); 71814039Sstacze01@arm.com 71914039Sstacze01@arm.com bool valid = pt_ops->isValid(pte, level); 72014039Sstacze01@arm.com bool leaf = pt_ops->isLeaf(pte, level); 72114039Sstacze01@arm.com 72214039Sstacze01@arm.com if (!valid) { 72314039Sstacze01@arm.com DPRINTF(SMMUv3, "S1 PTE not valid - fault\n"); 72414039Sstacze01@arm.com 72514039Sstacze01@arm.com TranslResult tr; 72614039Sstacze01@arm.com tr.fault = FAULT_TRANSLATION; 72714039Sstacze01@arm.com return tr; 72814039Sstacze01@arm.com } 72914039Sstacze01@arm.com 73014039Sstacze01@arm.com if (valid && leaf && request.isWrite && 73114039Sstacze01@arm.com !pt_ops->isWritable(pte, level, false)) 73214039Sstacze01@arm.com { 73314039Sstacze01@arm.com DPRINTF(SMMUv3, "S1 page not writable - fault\n"); 73414039Sstacze01@arm.com 73514039Sstacze01@arm.com TranslResult tr; 73614039Sstacze01@arm.com tr.fault = FAULT_PERMISSION; 73714039Sstacze01@arm.com return tr; 73814039Sstacze01@arm.com } 73914039Sstacze01@arm.com 74014039Sstacze01@arm.com walkPtr = pt_ops->nextLevelPointer(pte, level); 74114039Sstacze01@arm.com 74214039Sstacze01@arm.com if (leaf) 74314039Sstacze01@arm.com break; 74414039Sstacze01@arm.com 74514039Sstacze01@arm.com if (context.stage2Enable) { 74614039Sstacze01@arm.com TranslResult s2tr = translateStage2(yield, walkPtr, false); 74714039Sstacze01@arm.com if (s2tr.fault != FAULT_NONE) 74814039Sstacze01@arm.com return s2tr; 74914039Sstacze01@arm.com 75014039Sstacze01@arm.com walkPtr = s2tr.addr; 75114039Sstacze01@arm.com } 75214039Sstacze01@arm.com 75314039Sstacze01@arm.com walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, 75414039Sstacze01@arm.com 1, level, leaf, 0); 75514039Sstacze01@arm.com } 75614039Sstacze01@arm.com 75714039Sstacze01@arm.com TranslResult tr; 75814039Sstacze01@arm.com tr.fault = FAULT_NONE; 75914039Sstacze01@arm.com tr.addrMask = pt_ops->pageMask(pte, level); 76014039Sstacze01@arm.com tr.addr = walkPtr + (addr & ~tr.addrMask); 76114039Sstacze01@arm.com tr.writable = pt_ops->isWritable(pte, level, false); 76214039Sstacze01@arm.com 76314039Sstacze01@arm.com if (context.stage2Enable) { 76414039Sstacze01@arm.com TranslResult s2tr = translateStage2(yield, tr.addr, true); 76514039Sstacze01@arm.com if (s2tr.fault != FAULT_NONE) 76614039Sstacze01@arm.com return s2tr; 76714039Sstacze01@arm.com 76814039Sstacze01@arm.com tr = combineTranslations(tr, s2tr); 76914039Sstacze01@arm.com } 77014039Sstacze01@arm.com 77114039Sstacze01@arm.com walkCacheUpdate(yield, addr, tr.addrMask, tr.addr, 77214039Sstacze01@arm.com 1, level, true, tr.writable); 77314039Sstacze01@arm.com 77414039Sstacze01@arm.com return tr; 77514039Sstacze01@arm.com} 77614039Sstacze01@arm.com 77714039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 77814039Sstacze01@arm.comSMMUTranslationProcess::walkStage2(Yield &yield, Addr addr, bool final_tr, 77914039Sstacze01@arm.com const PageTableOps *pt_ops, 78014039Sstacze01@arm.com unsigned level, Addr walkPtr) 78114039Sstacze01@arm.com{ 78214039Sstacze01@arm.com PageTableOps::pte_t pte; 78314039Sstacze01@arm.com 78414039Sstacze01@arm.com doSemaphoreDown(yield, smmu.cycleSem); 78514039Sstacze01@arm.com doDelay(yield, Cycles(1)); 78614039Sstacze01@arm.com doSemaphoreUp(smmu.cycleSem); 78714039Sstacze01@arm.com 78814039Sstacze01@arm.com for (; level <= pt_ops->lastLevel(); level++) { 78914039Sstacze01@arm.com Addr pte_addr = walkPtr + pt_ops->index(addr, level); 79014039Sstacze01@arm.com 79114039Sstacze01@arm.com DPRINTF(SMMUv3, " Fetching S2 L%d PTE from pa=%#08x\n", 79214039Sstacze01@arm.com level, pte_addr); 79314039Sstacze01@arm.com 79414039Sstacze01@arm.com doReadPTE(yield, addr, pte_addr, &pte, 2, level); 79514039Sstacze01@arm.com 79614039Sstacze01@arm.com DPRINTF(SMMUv3, " Got S2 L%d PTE=%#x from pa=%#08x\n", 79714039Sstacze01@arm.com level, pte, pte_addr); 79814039Sstacze01@arm.com 79914039Sstacze01@arm.com doSemaphoreDown(yield, smmu.cycleSem); 80014039Sstacze01@arm.com doDelay(yield, Cycles(1)); 80114039Sstacze01@arm.com doSemaphoreUp(smmu.cycleSem); 80214039Sstacze01@arm.com 80314039Sstacze01@arm.com bool valid = pt_ops->isValid(pte, level); 80414039Sstacze01@arm.com bool leaf = pt_ops->isLeaf(pte, level); 80514039Sstacze01@arm.com 80614039Sstacze01@arm.com if (!valid) { 80714039Sstacze01@arm.com DPRINTF(SMMUv3, " S2 PTE not valid - fault\n"); 80814039Sstacze01@arm.com 80914039Sstacze01@arm.com TranslResult tr; 81014039Sstacze01@arm.com tr.fault = FAULT_TRANSLATION; 81114039Sstacze01@arm.com return tr; 81214039Sstacze01@arm.com } 81314039Sstacze01@arm.com 81414039Sstacze01@arm.com if (valid && leaf && request.isWrite && 81514039Sstacze01@arm.com !pt_ops->isWritable(pte, level, true)) 81614039Sstacze01@arm.com { 81714039Sstacze01@arm.com DPRINTF(SMMUv3, " S2 PTE not writable = fault\n"); 81814039Sstacze01@arm.com 81914039Sstacze01@arm.com TranslResult tr; 82014039Sstacze01@arm.com tr.fault = FAULT_PERMISSION; 82114039Sstacze01@arm.com return tr; 82214039Sstacze01@arm.com } 82314039Sstacze01@arm.com 82414039Sstacze01@arm.com walkPtr = pt_ops->nextLevelPointer(pte, level); 82514039Sstacze01@arm.com 82614039Sstacze01@arm.com if (final_tr || smmu.walkCacheNonfinalEnable) 82714039Sstacze01@arm.com walkCacheUpdate(yield, addr, pt_ops->walkMask(level), walkPtr, 82814039Sstacze01@arm.com 2, level, leaf, 82914039Sstacze01@arm.com leaf ? pt_ops->isWritable(pte, level, true) : 0); 83014039Sstacze01@arm.com if (leaf) 83114039Sstacze01@arm.com break; 83214039Sstacze01@arm.com } 83314039Sstacze01@arm.com 83414039Sstacze01@arm.com TranslResult tr; 83514039Sstacze01@arm.com tr.fault = FAULT_NONE; 83614039Sstacze01@arm.com tr.addrMask = pt_ops->pageMask(pte, level); 83714039Sstacze01@arm.com tr.addr = walkPtr + (addr & ~tr.addrMask); 83814039Sstacze01@arm.com tr.writable = pt_ops->isWritable(pte, level, true); 83914039Sstacze01@arm.com 84014039Sstacze01@arm.com return tr; 84114039Sstacze01@arm.com} 84214039Sstacze01@arm.com 84314039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 84414039Sstacze01@arm.comSMMUTranslationProcess::translateStage1And2(Yield &yield, Addr addr) 84514039Sstacze01@arm.com{ 84614039Sstacze01@arm.com const PageTableOps *pt_ops = 84714039Sstacze01@arm.com smmu.getPageTableOps(context.stage1TranslGranule); 84814039Sstacze01@arm.com 84914039Sstacze01@arm.com const WalkCache::Entry *walk_ep = NULL; 85014039Sstacze01@arm.com unsigned level; 85114039Sstacze01@arm.com 85214039Sstacze01@arm.com // Level here is actually (level+1) so we can count down 85314039Sstacze01@arm.com // to 0 using unsigned int. 85414039Sstacze01@arm.com for (level = pt_ops->lastLevel() + 1; 85514039Sstacze01@arm.com level > pt_ops->firstLevel(); 85614039Sstacze01@arm.com level--) 85714039Sstacze01@arm.com { 85814039Sstacze01@arm.com walkCacheLookup(yield, walk_ep, addr, 85914039Sstacze01@arm.com context.asid, context.vmid, 1, level-1); 86014039Sstacze01@arm.com 86114039Sstacze01@arm.com if (walk_ep) 86214039Sstacze01@arm.com break; 86314039Sstacze01@arm.com } 86414039Sstacze01@arm.com 86514039Sstacze01@arm.com // Correct level (see above). 86614039Sstacze01@arm.com level -= 1; 86714039Sstacze01@arm.com 86814039Sstacze01@arm.com TranslResult tr; 86914039Sstacze01@arm.com if (walk_ep) { 87014039Sstacze01@arm.com if (walk_ep->leaf) { 87114039Sstacze01@arm.com tr.fault = FAULT_NONE; 87214039Sstacze01@arm.com tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); 87314039Sstacze01@arm.com tr.addrMask = walk_ep->vaMask; 87414039Sstacze01@arm.com tr.writable = walk_ep->permissions; 87514039Sstacze01@arm.com } else { 87614039Sstacze01@arm.com tr = walkStage1And2(yield, addr, pt_ops, level+1, walk_ep->pa); 87714039Sstacze01@arm.com } 87814039Sstacze01@arm.com } else { 87914039Sstacze01@arm.com Addr table_addr = context.ttb0; 88014039Sstacze01@arm.com if (context.stage2Enable) { 88114039Sstacze01@arm.com TranslResult s2tr = translateStage2(yield, table_addr, false); 88214039Sstacze01@arm.com if (s2tr.fault != FAULT_NONE) 88314039Sstacze01@arm.com return s2tr; 88414039Sstacze01@arm.com 88514039Sstacze01@arm.com table_addr = s2tr.addr; 88614039Sstacze01@arm.com } 88714039Sstacze01@arm.com 88814039Sstacze01@arm.com tr = walkStage1And2(yield, addr, pt_ops, pt_ops->firstLevel(), 88914039Sstacze01@arm.com table_addr); 89014039Sstacze01@arm.com } 89114039Sstacze01@arm.com 89214039Sstacze01@arm.com if (tr.fault == FAULT_NONE) 89314039Sstacze01@arm.com DPRINTF(SMMUv3, "Translated vaddr %#x to paddr %#x\n", addr, tr.addr); 89414039Sstacze01@arm.com 89514039Sstacze01@arm.com return tr; 89614039Sstacze01@arm.com} 89714039Sstacze01@arm.com 89814039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 89914039Sstacze01@arm.comSMMUTranslationProcess::translateStage2(Yield &yield, Addr addr, bool final_tr) 90014039Sstacze01@arm.com{ 90114039Sstacze01@arm.com const PageTableOps *pt_ops = 90214039Sstacze01@arm.com smmu.getPageTableOps(context.stage2TranslGranule); 90314039Sstacze01@arm.com 90414039Sstacze01@arm.com const IPACache::Entry *ipa_ep = NULL; 90514039Sstacze01@arm.com if (smmu.ipaCacheEnable) { 90614039Sstacze01@arm.com doSemaphoreDown(yield, smmu.ipaSem); 90714039Sstacze01@arm.com doDelay(yield, smmu.ipaLat); 90814039Sstacze01@arm.com ipa_ep = smmu.ipaCache.lookup(addr, context.vmid); 90914039Sstacze01@arm.com doSemaphoreUp(smmu.ipaSem); 91014039Sstacze01@arm.com } 91114039Sstacze01@arm.com 91214039Sstacze01@arm.com if (ipa_ep) { 91314039Sstacze01@arm.com TranslResult tr; 91414039Sstacze01@arm.com tr.fault = FAULT_NONE; 91514039Sstacze01@arm.com tr.addr = ipa_ep->pa + (addr & ~ipa_ep->ipaMask); 91614039Sstacze01@arm.com tr.addrMask = ipa_ep->ipaMask; 91714039Sstacze01@arm.com tr.writable = ipa_ep->permissions; 91814039Sstacze01@arm.com 91914039Sstacze01@arm.com DPRINTF(SMMUv3, " IPACache hit ipa=%#x vmid=%#x pa=%#x\n", 92014039Sstacze01@arm.com addr, context.vmid, tr.addr); 92114039Sstacze01@arm.com 92214039Sstacze01@arm.com return tr; 92314039Sstacze01@arm.com } else if (smmu.ipaCacheEnable) { 92414039Sstacze01@arm.com DPRINTF(SMMUv3, " IPACache miss ipa=%#x vmid=%#x\n", 92514039Sstacze01@arm.com addr, context.vmid); 92614039Sstacze01@arm.com } 92714039Sstacze01@arm.com 92814039Sstacze01@arm.com const WalkCache::Entry *walk_ep = NULL; 92914039Sstacze01@arm.com unsigned level = pt_ops->firstLevel(); 93014039Sstacze01@arm.com 93114039Sstacze01@arm.com if (final_tr || smmu.walkCacheNonfinalEnable) { 93214039Sstacze01@arm.com // Level here is actually (level+1) so we can count down 93314039Sstacze01@arm.com // to 0 using unsigned int. 93414039Sstacze01@arm.com for (level = pt_ops->lastLevel() + 1; 93514039Sstacze01@arm.com level > pt_ops->firstLevel(); 93614039Sstacze01@arm.com level--) 93714039Sstacze01@arm.com { 93814039Sstacze01@arm.com walkCacheLookup(yield, walk_ep, addr, 93914039Sstacze01@arm.com 0, context.vmid, 2, level-1); 94014039Sstacze01@arm.com 94114039Sstacze01@arm.com if (walk_ep) 94214039Sstacze01@arm.com break; 94314039Sstacze01@arm.com } 94414039Sstacze01@arm.com 94514039Sstacze01@arm.com // Correct level (see above). 94614039Sstacze01@arm.com level -= 1; 94714039Sstacze01@arm.com } 94814039Sstacze01@arm.com 94914039Sstacze01@arm.com TranslResult tr; 95014039Sstacze01@arm.com if (walk_ep) { 95114039Sstacze01@arm.com if (walk_ep->leaf) { 95214039Sstacze01@arm.com tr.fault = FAULT_NONE; 95314039Sstacze01@arm.com tr.addr = walk_ep->pa + (addr & ~walk_ep->vaMask); 95414039Sstacze01@arm.com tr.addrMask = walk_ep->vaMask; 95514039Sstacze01@arm.com tr.writable = walk_ep->permissions; 95614039Sstacze01@arm.com } else { 95714039Sstacze01@arm.com tr = walkStage2(yield, addr, final_tr, pt_ops, 95814039Sstacze01@arm.com level + 1, walk_ep->pa); 95914039Sstacze01@arm.com } 96014039Sstacze01@arm.com } else { 96114039Sstacze01@arm.com tr = walkStage2(yield, addr, final_tr, pt_ops, pt_ops->firstLevel(), 96214039Sstacze01@arm.com context.httb); 96314039Sstacze01@arm.com } 96414039Sstacze01@arm.com 96514039Sstacze01@arm.com if (tr.fault == FAULT_NONE) 96614039Sstacze01@arm.com DPRINTF(SMMUv3, " Translated %saddr %#x to paddr %#x\n", 96714039Sstacze01@arm.com context.stage1Enable ? "ip" : "v", addr, tr.addr); 96814039Sstacze01@arm.com 96914039Sstacze01@arm.com if (smmu.ipaCacheEnable) { 97014039Sstacze01@arm.com IPACache::Entry e; 97114039Sstacze01@arm.com e.valid = true; 97214039Sstacze01@arm.com e.ipaMask = tr.addrMask; 97314039Sstacze01@arm.com e.ipa = addr & e.ipaMask; 97414039Sstacze01@arm.com e.pa = tr.addr & tr.addrMask; 97514039Sstacze01@arm.com e.permissions = tr.writable; 97614039Sstacze01@arm.com e.vmid = context.vmid; 97714039Sstacze01@arm.com 97814039Sstacze01@arm.com doSemaphoreDown(yield, smmu.ipaSem); 97914039Sstacze01@arm.com smmu.ipaCache.store(e); 98014039Sstacze01@arm.com doSemaphoreUp(smmu.ipaSem); 98114039Sstacze01@arm.com } 98214039Sstacze01@arm.com 98314039Sstacze01@arm.com return tr; 98414039Sstacze01@arm.com} 98514039Sstacze01@arm.com 98614039Sstacze01@arm.comSMMUTranslationProcess::TranslResult 98714039Sstacze01@arm.comSMMUTranslationProcess::combineTranslations(const TranslResult &s1tr, 98814039Sstacze01@arm.com const TranslResult &s2tr) const 98914039Sstacze01@arm.com{ 99014039Sstacze01@arm.com if (s2tr.fault != FAULT_NONE) 99114039Sstacze01@arm.com return s2tr; 99214039Sstacze01@arm.com 99314039Sstacze01@arm.com assert(s1tr.fault == FAULT_NONE); 99414039Sstacze01@arm.com 99514039Sstacze01@arm.com TranslResult tr; 99614039Sstacze01@arm.com tr.fault = FAULT_NONE; 99714039Sstacze01@arm.com tr.addr = s2tr.addr; 99814039Sstacze01@arm.com tr.addrMask = s1tr.addrMask | s2tr.addrMask; 99914039Sstacze01@arm.com tr.writable = s1tr.writable & s2tr.writable; 100014039Sstacze01@arm.com 100114039Sstacze01@arm.com return tr; 100214039Sstacze01@arm.com} 100314039Sstacze01@arm.com 100414039Sstacze01@arm.combool 100514039Sstacze01@arm.comSMMUTranslationProcess::hazard4kCheck() 100614039Sstacze01@arm.com{ 100714039Sstacze01@arm.com Addr addr4k = request.addr & ~0xfffULL; 100814039Sstacze01@arm.com 100914039Sstacze01@arm.com for (auto it = ifc.duplicateReqs.begin(); 101014039Sstacze01@arm.com it != ifc.duplicateReqs.end(); 101114039Sstacze01@arm.com ++it) 101214039Sstacze01@arm.com { 101314039Sstacze01@arm.com Addr other4k = (*it)->request.addr & ~0xfffULL; 101414039Sstacze01@arm.com if (addr4k == other4k) 101514039Sstacze01@arm.com return true; 101614039Sstacze01@arm.com } 101714039Sstacze01@arm.com 101814039Sstacze01@arm.com return false; 101914039Sstacze01@arm.com} 102014039Sstacze01@arm.com 102114039Sstacze01@arm.comvoid 102214039Sstacze01@arm.comSMMUTranslationProcess::hazard4kRegister() 102314039Sstacze01@arm.com{ 102414039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "4kReg: p=%p a4k=%#x\n", 102514039Sstacze01@arm.com this, request.addr & ~0xfffULL); 102614039Sstacze01@arm.com 102714039Sstacze01@arm.com ifc.duplicateReqs.push_back(this); 102814039Sstacze01@arm.com} 102914039Sstacze01@arm.com 103014039Sstacze01@arm.comvoid 103114039Sstacze01@arm.comSMMUTranslationProcess::hazard4kHold(Yield &yield) 103214039Sstacze01@arm.com{ 103314039Sstacze01@arm.com Addr addr4k = request.addr & ~0xfffULL; 103414039Sstacze01@arm.com 103514039Sstacze01@arm.com bool found_hazard; 103614039Sstacze01@arm.com 103714039Sstacze01@arm.com do { 103814039Sstacze01@arm.com found_hazard = false; 103914039Sstacze01@arm.com 104014039Sstacze01@arm.com for (auto it = ifc.duplicateReqs.begin(); 104114039Sstacze01@arm.com it!=ifc.duplicateReqs.end() && *it!=this; 104214039Sstacze01@arm.com ++it) 104314039Sstacze01@arm.com { 104414039Sstacze01@arm.com Addr other4k = (*it)->request.addr & ~0xfffULL; 104514039Sstacze01@arm.com 104614039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x Q: p=%p a4k=%#x\n", 104714039Sstacze01@arm.com this, addr4k, *it, other4k); 104814039Sstacze01@arm.com 104914039Sstacze01@arm.com if (addr4k == other4k) { 105014039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, 105114039Sstacze01@arm.com "4kHold: p=%p a4k=%#x WAIT on p=%p a4k=%#x\n", 105214039Sstacze01@arm.com this, addr4k, *it, other4k); 105314039Sstacze01@arm.com 105414039Sstacze01@arm.com doWaitForSignal(yield, ifc.duplicateReqRemoved); 105514039Sstacze01@arm.com 105614039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "4kHold: p=%p a4k=%#x RESUME\n", 105714039Sstacze01@arm.com this, addr4k); 105814039Sstacze01@arm.com 105914039Sstacze01@arm.com // This is to avoid checking *it!=this after doWaitForSignal() 106014039Sstacze01@arm.com // since it could have been deleted. 106114039Sstacze01@arm.com found_hazard = true; 106214039Sstacze01@arm.com break; 106314039Sstacze01@arm.com } 106414039Sstacze01@arm.com } 106514039Sstacze01@arm.com } while (found_hazard); 106614039Sstacze01@arm.com} 106714039Sstacze01@arm.com 106814039Sstacze01@arm.comvoid 106914039Sstacze01@arm.comSMMUTranslationProcess::hazard4kRelease() 107014039Sstacze01@arm.com{ 107114039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "4kRel: p=%p a4k=%#x\n", 107214039Sstacze01@arm.com this, request.addr & ~0xfffULL); 107314039Sstacze01@arm.com 107414039Sstacze01@arm.com std::list<SMMUTranslationProcess *>::iterator it; 107514039Sstacze01@arm.com 107614039Sstacze01@arm.com for (it = ifc.duplicateReqs.begin(); it != ifc.duplicateReqs.end(); ++it) 107714039Sstacze01@arm.com if (*it == this) 107814039Sstacze01@arm.com break; 107914039Sstacze01@arm.com 108014039Sstacze01@arm.com if (it == ifc.duplicateReqs.end()) 108114039Sstacze01@arm.com panic("hazard4kRelease: request not found"); 108214039Sstacze01@arm.com 108314039Sstacze01@arm.com ifc.duplicateReqs.erase(it); 108414039Sstacze01@arm.com 108514039Sstacze01@arm.com doBroadcastSignal(ifc.duplicateReqRemoved); 108614039Sstacze01@arm.com} 108714039Sstacze01@arm.com 108814039Sstacze01@arm.comvoid 108914039Sstacze01@arm.comSMMUTranslationProcess::hazardIdRegister() 109014039Sstacze01@arm.com{ 109114039Sstacze01@arm.com auto orderId = AMBA::orderId(request.pkt); 109214039Sstacze01@arm.com 109314039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdReg: p=%p oid=%d\n", this, orderId); 109414039Sstacze01@arm.com 109514039Sstacze01@arm.com assert(orderId < SMMU_MAX_TRANS_ID); 109614039Sstacze01@arm.com 109714039Sstacze01@arm.com std::list<SMMUTranslationProcess *> &depReqs = 109814039Sstacze01@arm.com request.isWrite ? 109914039Sstacze01@arm.com ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 110014039Sstacze01@arm.com depReqs.push_back(this); 110114039Sstacze01@arm.com} 110214039Sstacze01@arm.com 110314039Sstacze01@arm.comvoid 110414039Sstacze01@arm.comSMMUTranslationProcess::hazardIdHold(Yield &yield) 110514039Sstacze01@arm.com{ 110614039Sstacze01@arm.com auto orderId = AMBA::orderId(request.pkt); 110714039Sstacze01@arm.com 110814039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d\n", this, orderId); 110914039Sstacze01@arm.com 111014039Sstacze01@arm.com std::list<SMMUTranslationProcess *> &depReqs = 111114039Sstacze01@arm.com request.isWrite ? 111214039Sstacze01@arm.com ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 111314039Sstacze01@arm.com std::list<SMMUTranslationProcess *>::iterator it; 111414039Sstacze01@arm.com 111514039Sstacze01@arm.com bool found_hazard; 111614039Sstacze01@arm.com 111714039Sstacze01@arm.com do { 111814039Sstacze01@arm.com found_hazard = false; 111914039Sstacze01@arm.com 112014039Sstacze01@arm.com for (auto it = depReqs.begin(); it!=depReqs.end() && *it!=this; ++it) { 112114039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d Q: %p\n", 112214039Sstacze01@arm.com this, orderId, *it); 112314039Sstacze01@arm.com 112414039Sstacze01@arm.com if (AMBA::orderId((*it)->request.pkt) == orderId) { 112514039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d WAIT on=%p\n", 112614039Sstacze01@arm.com this, orderId, *it); 112714039Sstacze01@arm.com 112814039Sstacze01@arm.com doWaitForSignal(yield, ifc.dependentReqRemoved); 112914039Sstacze01@arm.com 113014039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdHold: p=%p oid=%d RESUME\n", 113114039Sstacze01@arm.com this, orderId); 113214039Sstacze01@arm.com 113314039Sstacze01@arm.com // This is to avoid checking *it!=this after doWaitForSignal() 113414039Sstacze01@arm.com // since it could have been deleted. 113514039Sstacze01@arm.com found_hazard = true; 113614039Sstacze01@arm.com break; 113714039Sstacze01@arm.com } 113814039Sstacze01@arm.com } 113914039Sstacze01@arm.com } while (found_hazard); 114014039Sstacze01@arm.com} 114114039Sstacze01@arm.com 114214039Sstacze01@arm.comvoid 114314039Sstacze01@arm.comSMMUTranslationProcess::hazardIdRelease() 114414039Sstacze01@arm.com{ 114514039Sstacze01@arm.com auto orderId = AMBA::orderId(request.pkt); 114614039Sstacze01@arm.com 114714039Sstacze01@arm.com DPRINTF(SMMUv3Hazard, "IdRel: p=%p oid=%d\n", this, orderId); 114814039Sstacze01@arm.com 114914039Sstacze01@arm.com std::list<SMMUTranslationProcess *> &depReqs = 115014039Sstacze01@arm.com request.isWrite ? 115114039Sstacze01@arm.com ifc.dependentWrites[orderId] : ifc.dependentReads[orderId]; 115214039Sstacze01@arm.com std::list<SMMUTranslationProcess *>::iterator it; 115314039Sstacze01@arm.com 115414039Sstacze01@arm.com for (it = depReqs.begin(); it != depReqs.end(); ++it) { 115514039Sstacze01@arm.com if (*it == this) 115614039Sstacze01@arm.com break; 115714039Sstacze01@arm.com } 115814039Sstacze01@arm.com 115914039Sstacze01@arm.com if (it == depReqs.end()) 116014039Sstacze01@arm.com panic("hazardIdRelease: request not found"); 116114039Sstacze01@arm.com 116214039Sstacze01@arm.com depReqs.erase(it); 116314039Sstacze01@arm.com 116414039Sstacze01@arm.com doBroadcastSignal(ifc.dependentReqRemoved); 116514039Sstacze01@arm.com} 116614039Sstacze01@arm.com 116714039Sstacze01@arm.comvoid 116814039Sstacze01@arm.comSMMUTranslationProcess::issuePrefetch(Addr addr) 116914039Sstacze01@arm.com{ 117014039Sstacze01@arm.com if (!smmu.system.isTimingMode()) 117114039Sstacze01@arm.com return; 117214039Sstacze01@arm.com 117314039Sstacze01@arm.com if (!ifc.prefetchEnable || ifc.xlateSlotsRemaining == 0) 117414039Sstacze01@arm.com return; 117514039Sstacze01@arm.com 117614039Sstacze01@arm.com ifc.xlateSlotsRemaining--; 117714039Sstacze01@arm.com 117814039Sstacze01@arm.com std::string proc_name = csprintf("%sprf", name()); 117914039Sstacze01@arm.com SMMUTranslationProcess *proc = 118014039Sstacze01@arm.com new SMMUTranslationProcess(proc_name, smmu, ifc); 118114039Sstacze01@arm.com 118214039Sstacze01@arm.com proc->beginTransaction( 118314039Sstacze01@arm.com SMMUTranslRequest::prefetch(addr, request.sid, request.ssid)); 118414039Sstacze01@arm.com proc->scheduleWakeup(smmu.clockEdge(Cycles(1))); 118514039Sstacze01@arm.com} 118614039Sstacze01@arm.com 118714039Sstacze01@arm.comvoid 118814039Sstacze01@arm.comSMMUTranslationProcess::completeTransaction(Yield &yield, 118914039Sstacze01@arm.com const TranslResult &tr) 119014039Sstacze01@arm.com{ 119114039Sstacze01@arm.com assert(tr.fault == FAULT_NONE); 119214039Sstacze01@arm.com 119314039Sstacze01@arm.com unsigned numMasterBeats = request.isWrite ? 119414039Sstacze01@arm.com (request.size + (smmu.masterPortWidth-1)) 119514039Sstacze01@arm.com / smmu.masterPortWidth : 119614039Sstacze01@arm.com 1; 119714039Sstacze01@arm.com 119814039Sstacze01@arm.com doSemaphoreDown(yield, smmu.masterPortSem); 119914039Sstacze01@arm.com doDelay(yield, Cycles(numMasterBeats)); 120014039Sstacze01@arm.com doSemaphoreUp(smmu.masterPortSem); 120114039Sstacze01@arm.com 120214039Sstacze01@arm.com 120314039Sstacze01@arm.com smmu.translationTimeDist.sample(curTick() - recvTick); 120414039Sstacze01@arm.com ifc.xlateSlotsRemaining++; 120514039Sstacze01@arm.com if (!request.isAtsRequest && request.isWrite) 120614039Sstacze01@arm.com ifc.wrBufSlotsRemaining += 120714039Sstacze01@arm.com (request.size + (ifc.portWidth-1)) / ifc.portWidth; 120814039Sstacze01@arm.com 120914039Sstacze01@arm.com smmu.scheduleSlaveRetries(); 121014039Sstacze01@arm.com 121114039Sstacze01@arm.com 121214039Sstacze01@arm.com SMMUAction a; 121314039Sstacze01@arm.com 121414039Sstacze01@arm.com if (request.isAtsRequest) { 121514039Sstacze01@arm.com a.type = ACTION_SEND_RESP_ATS; 121614039Sstacze01@arm.com 121714039Sstacze01@arm.com if (smmu.system.isAtomicMode()) { 121814039Sstacze01@arm.com request.pkt->makeAtomicResponse(); 121914039Sstacze01@arm.com } else if (smmu.system.isTimingMode()) { 122014039Sstacze01@arm.com request.pkt->makeTimingResponse(); 122114039Sstacze01@arm.com } else { 122214039Sstacze01@arm.com panic("Not in atomic or timing mode"); 122314039Sstacze01@arm.com } 122414039Sstacze01@arm.com } else { 122514039Sstacze01@arm.com a.type = ACTION_SEND_REQ_FINAL; 122614039Sstacze01@arm.com a.ifc = &ifc; 122714039Sstacze01@arm.com } 122814039Sstacze01@arm.com 122914039Sstacze01@arm.com a.pkt = request.pkt; 123014039Sstacze01@arm.com a.delay = 0; 123114039Sstacze01@arm.com 123214039Sstacze01@arm.com a.pkt->setAddr(tr.addr); 123314039Sstacze01@arm.com a.pkt->req->setPaddr(tr.addr); 123414039Sstacze01@arm.com 123514039Sstacze01@arm.com yield(a); 123614039Sstacze01@arm.com 123714039Sstacze01@arm.com if (!request.isAtsRequest) { 123814039Sstacze01@arm.com PacketPtr pkt = yield.get(); 123914039Sstacze01@arm.com pkt->setAddr(request.addr); 124014039Sstacze01@arm.com 124114039Sstacze01@arm.com a.type = ACTION_SEND_RESP; 124214039Sstacze01@arm.com a.pkt = pkt; 124314039Sstacze01@arm.com a.ifc = &ifc; 124414039Sstacze01@arm.com a.delay = 0; 124514039Sstacze01@arm.com yield(a); 124614039Sstacze01@arm.com } 124714039Sstacze01@arm.com} 124814039Sstacze01@arm.com 124914039Sstacze01@arm.comvoid 125014039Sstacze01@arm.comSMMUTranslationProcess::completePrefetch(Yield &yield) 125114039Sstacze01@arm.com{ 125214039Sstacze01@arm.com ifc.xlateSlotsRemaining++; 125314039Sstacze01@arm.com 125414039Sstacze01@arm.com SMMUAction a; 125514039Sstacze01@arm.com a.type = ACTION_TERMINATE; 125614039Sstacze01@arm.com a.pkt = NULL; 125714039Sstacze01@arm.com a.ifc = &ifc; 125814039Sstacze01@arm.com a.delay = 0; 125914039Sstacze01@arm.com yield(a); 126014039Sstacze01@arm.com} 126114039Sstacze01@arm.com 126214039Sstacze01@arm.comvoid 126314039Sstacze01@arm.comSMMUTranslationProcess::sendEvent(Yield &yield, const SMMUEvent &ev) 126414039Sstacze01@arm.com{ 126514039Sstacze01@arm.com int sizeMask = mask(smmu.regs.eventq_base & Q_BASE_SIZE_MASK) & 126614039Sstacze01@arm.com Q_CONS_PROD_MASK; 126714039Sstacze01@arm.com 126814039Sstacze01@arm.com if (((smmu.regs.eventq_prod+1) & sizeMask) == 126914039Sstacze01@arm.com (smmu.regs.eventq_cons & sizeMask)) 127014039Sstacze01@arm.com panic("Event queue full - aborting\n"); 127114039Sstacze01@arm.com 127214039Sstacze01@arm.com Addr event_addr = 127314039Sstacze01@arm.com (smmu.regs.eventq_base & Q_BASE_ADDR_MASK) + 127414039Sstacze01@arm.com (smmu.regs.eventq_prod & sizeMask) * sizeof(ev); 127514039Sstacze01@arm.com 127614039Sstacze01@arm.com DPRINTF(SMMUv3, "Sending event to addr=%#08x (pos=%d): type=%#x stag=%#x " 127714039Sstacze01@arm.com "flags=%#x sid=%#x ssid=%#x va=%#08x ipa=%#x\n", 127814039Sstacze01@arm.com event_addr, smmu.regs.eventq_prod, ev.type, ev.stag, 127914039Sstacze01@arm.com ev.flags, ev.streamId, ev.substreamId, ev.va, ev.ipa); 128014039Sstacze01@arm.com 128114039Sstacze01@arm.com // This deliberately resets the overflow field in eventq_prod! 128214039Sstacze01@arm.com smmu.regs.eventq_prod = (smmu.regs.eventq_prod + 1) & sizeMask; 128314039Sstacze01@arm.com 128414039Sstacze01@arm.com doWrite(yield, event_addr, &ev, sizeof(ev)); 128514039Sstacze01@arm.com 128614039Sstacze01@arm.com if (!(smmu.regs.eventq_irq_cfg0 & E_BASE_ENABLE_MASK)) 128714039Sstacze01@arm.com panic("eventq msi not enabled\n"); 128814039Sstacze01@arm.com 128914039Sstacze01@arm.com doWrite(yield, smmu.regs.eventq_irq_cfg0 & E_BASE_ADDR_MASK, 129014039Sstacze01@arm.com &smmu.regs.eventq_irq_cfg1, sizeof(smmu.regs.eventq_irq_cfg1)); 129114039Sstacze01@arm.com} 129214039Sstacze01@arm.com 129314039Sstacze01@arm.comvoid 129414039Sstacze01@arm.comSMMUTranslationProcess::doReadSTE(Yield &yield, 129514039Sstacze01@arm.com StreamTableEntry &ste, 129614039Sstacze01@arm.com uint32_t sid) 129714039Sstacze01@arm.com{ 129814039Sstacze01@arm.com unsigned max_sid = 1 << (smmu.regs.strtab_base_cfg & ST_CFG_SIZE_MASK); 129914039Sstacze01@arm.com if (sid >= max_sid) 130014039Sstacze01@arm.com panic("SID %#x out of range, max=%#x", sid, max_sid); 130114039Sstacze01@arm.com 130214039Sstacze01@arm.com Addr ste_addr; 130314039Sstacze01@arm.com 130414039Sstacze01@arm.com if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_2LEVEL) { 130514039Sstacze01@arm.com unsigned split = 130614039Sstacze01@arm.com (smmu.regs.strtab_base_cfg & ST_CFG_SPLIT_MASK) >> ST_CFG_SPLIT_SHIFT; 130714039Sstacze01@arm.com 130814039Sstacze01@arm.com if (split!= 7 && split!=8 && split!=16) 130914039Sstacze01@arm.com panic("Invalid stream table split %d", split); 131014039Sstacze01@arm.com 131114039Sstacze01@arm.com uint64_t l2_ptr; 131214039Sstacze01@arm.com uint64_t l2_addr = 131314039Sstacze01@arm.com (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + 131414039Sstacze01@arm.com bits(sid, 32, split) * sizeof(l2_ptr); 131514039Sstacze01@arm.com 131614039Sstacze01@arm.com DPRINTF(SMMUv3, "Read L1STE at %#x\n", l2_addr); 131714039Sstacze01@arm.com 131814039Sstacze01@arm.com doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, 0); 131914039Sstacze01@arm.com 132014039Sstacze01@arm.com DPRINTF(SMMUv3, "Got L1STE L1 at %#x: 0x%016x\n", l2_addr, l2_ptr); 132114039Sstacze01@arm.com 132214039Sstacze01@arm.com unsigned span = l2_ptr & ST_L2_SPAN_MASK; 132314039Sstacze01@arm.com if (span == 0) 132414039Sstacze01@arm.com panic("Invalid level 1 stream table descriptor"); 132514039Sstacze01@arm.com 132614039Sstacze01@arm.com unsigned index = bits(sid, split-1, 0); 132714039Sstacze01@arm.com if (index >= (1 << span)) 132814039Sstacze01@arm.com panic("StreamID %d out of level 1 descriptor range %d", 132914039Sstacze01@arm.com sid, 1<<span); 133014039Sstacze01@arm.com 133114039Sstacze01@arm.com ste_addr = (l2_ptr & ST_L2_ADDR_MASK) + index * sizeof(ste); 133214039Sstacze01@arm.com 133314039Sstacze01@arm.com smmu.steL1Fetches++; 133414039Sstacze01@arm.com } else if ((smmu.regs.strtab_base_cfg & ST_CFG_FMT_MASK) == ST_CFG_FMT_LINEAR) { 133514039Sstacze01@arm.com ste_addr = 133614039Sstacze01@arm.com (smmu.regs.strtab_base & VMT_BASE_ADDR_MASK) + sid * sizeof(ste); 133714039Sstacze01@arm.com } else { 133814039Sstacze01@arm.com panic("Invalid stream table format"); 133914039Sstacze01@arm.com } 134014039Sstacze01@arm.com 134114039Sstacze01@arm.com DPRINTF(SMMUv3, "Read STE at %#x\n", ste_addr); 134214039Sstacze01@arm.com 134314039Sstacze01@arm.com doReadConfig(yield, ste_addr, &ste, sizeof(ste), sid, 0); 134414039Sstacze01@arm.com 134514039Sstacze01@arm.com DPRINTF(SMMUv3, "Got STE at %#x [0]: 0x%016x\n", ste_addr, ste.dw0); 134614039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [1]: 0x%016x\n", ste_addr, ste.dw1); 134714039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [2]: 0x%016x\n", ste_addr, ste.dw2); 134814039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [3]: 0x%016x\n", ste_addr, ste.dw3); 134914039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [4]: 0x%016x\n", ste_addr, ste._pad[0]); 135014039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [5]: 0x%016x\n", ste_addr, ste._pad[1]); 135114039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [6]: 0x%016x\n", ste_addr, ste._pad[2]); 135214039Sstacze01@arm.com DPRINTF(SMMUv3, " STE at %#x [7]: 0x%016x\n", ste_addr, ste._pad[3]); 135314039Sstacze01@arm.com 135414039Sstacze01@arm.com if (!ste.dw0.valid) 135514039Sstacze01@arm.com panic("STE @ %#x not valid\n", ste_addr); 135614039Sstacze01@arm.com 135714039Sstacze01@arm.com smmu.steFetches++; 135814039Sstacze01@arm.com} 135914039Sstacze01@arm.com 136014039Sstacze01@arm.comvoid 136114039Sstacze01@arm.comSMMUTranslationProcess::doReadCD(Yield &yield, 136214039Sstacze01@arm.com ContextDescriptor &cd, 136314039Sstacze01@arm.com const StreamTableEntry &ste, 136414039Sstacze01@arm.com uint32_t sid, uint32_t ssid) 136514039Sstacze01@arm.com{ 136614039Sstacze01@arm.com Addr cd_addr; 136714039Sstacze01@arm.com 136814039Sstacze01@arm.com if (ste.dw0.s1cdmax == 0) { 136914039Sstacze01@arm.com cd_addr = ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT; 137014039Sstacze01@arm.com } else { 137114039Sstacze01@arm.com unsigned max_ssid = 1 << ste.dw0.s1cdmax; 137214039Sstacze01@arm.com if (ssid >= max_ssid) 137314039Sstacze01@arm.com panic("SSID %#x out of range, max=%#x", ssid, max_ssid); 137414039Sstacze01@arm.com 137514039Sstacze01@arm.com if (ste.dw0.s1fmt==STAGE1_CFG_2L_4K || 137614039Sstacze01@arm.com ste.dw0.s1fmt==STAGE1_CFG_2L_64K) 137714039Sstacze01@arm.com { 137814039Sstacze01@arm.com unsigned split = ste.dw0.s1fmt==STAGE1_CFG_2L_4K ? 7 : 11; 137914039Sstacze01@arm.com 138014039Sstacze01@arm.com uint64_t l2_ptr; 138114039Sstacze01@arm.com uint64_t l2_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + 138214039Sstacze01@arm.com bits(ssid, 24, split) * sizeof(l2_ptr); 138314039Sstacze01@arm.com 138414039Sstacze01@arm.com if (context.stage2Enable) 138514039Sstacze01@arm.com l2_addr = translateStage2(yield, l2_addr, false).addr; 138614039Sstacze01@arm.com 138714039Sstacze01@arm.com DPRINTF(SMMUv3, "Read L1CD at %#x\n", l2_addr); 138814039Sstacze01@arm.com 138914039Sstacze01@arm.com doReadConfig(yield, l2_addr, &l2_ptr, sizeof(l2_ptr), sid, ssid); 139014039Sstacze01@arm.com 139114039Sstacze01@arm.com DPRINTF(SMMUv3, "Got L1CD at %#x: 0x%016x\n", l2_addr, l2_ptr); 139214039Sstacze01@arm.com 139314039Sstacze01@arm.com cd_addr = l2_ptr + bits(ssid, split-1, 0) * sizeof(cd); 139414039Sstacze01@arm.com 139514039Sstacze01@arm.com smmu.cdL1Fetches++; 139614039Sstacze01@arm.com } else if (ste.dw0.s1fmt == STAGE1_CFG_1L) { 139714039Sstacze01@arm.com cd_addr = (ste.dw0.s1ctxptr << ST_CD_ADDR_SHIFT) + ssid*sizeof(cd); 139814039Sstacze01@arm.com } 139914039Sstacze01@arm.com } 140014039Sstacze01@arm.com 140114039Sstacze01@arm.com if (context.stage2Enable) 140214039Sstacze01@arm.com cd_addr = translateStage2(yield, cd_addr, false).addr; 140314039Sstacze01@arm.com 140414039Sstacze01@arm.com DPRINTF(SMMUv3, "Read CD at %#x\n", cd_addr); 140514039Sstacze01@arm.com 140614039Sstacze01@arm.com doReadConfig(yield, cd_addr, &cd, sizeof(cd), sid, ssid); 140714039Sstacze01@arm.com 140814039Sstacze01@arm.com DPRINTF(SMMUv3, "Got CD at %#x [0]: 0x%016x\n", cd_addr, cd.dw0); 140914039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [1]: 0x%016x\n", cd_addr, cd.dw1); 141014039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [2]: 0x%016x\n", cd_addr, cd.dw2); 141114039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [3]: 0x%016x\n", cd_addr, cd.mair); 141214039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [4]: 0x%016x\n", cd_addr, cd.amair); 141314039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [5]: 0x%016x\n", cd_addr, cd._pad[0]); 141414039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [6]: 0x%016x\n", cd_addr, cd._pad[1]); 141514039Sstacze01@arm.com DPRINTF(SMMUv3, " CD at %#x [7]: 0x%016x\n", cd_addr, cd._pad[2]); 141614039Sstacze01@arm.com 141714039Sstacze01@arm.com 141814039Sstacze01@arm.com if (!cd.dw0.valid) 141914039Sstacze01@arm.com panic("CD @ %#x not valid\n", cd_addr); 142014039Sstacze01@arm.com 142114039Sstacze01@arm.com smmu.cdFetches++; 142214039Sstacze01@arm.com} 142314039Sstacze01@arm.com 142414039Sstacze01@arm.comvoid 142514039Sstacze01@arm.comSMMUTranslationProcess::doReadConfig(Yield &yield, Addr addr, 142614039Sstacze01@arm.com void *ptr, size_t size, 142714039Sstacze01@arm.com uint32_t sid, uint32_t ssid) 142814039Sstacze01@arm.com{ 142914039Sstacze01@arm.com doRead(yield, addr, ptr, size); 143014039Sstacze01@arm.com} 143114039Sstacze01@arm.com 143214039Sstacze01@arm.comvoid 143314039Sstacze01@arm.comSMMUTranslationProcess::doReadPTE(Yield &yield, Addr va, Addr addr, 143414039Sstacze01@arm.com void *ptr, unsigned stage, 143514039Sstacze01@arm.com unsigned level) 143614039Sstacze01@arm.com{ 143714039Sstacze01@arm.com size_t pte_size = sizeof(PageTableOps::pte_t); 143814039Sstacze01@arm.com 143914039Sstacze01@arm.com Addr mask = pte_size - 1; 144014039Sstacze01@arm.com Addr base = addr & ~mask; 144114039Sstacze01@arm.com 144214039Sstacze01@arm.com doRead(yield, base, ptr, pte_size); 144314039Sstacze01@arm.com} 1444