smmu_v3.cc revision 14132
19243SN/A/* 210206Sandreas.hansson@arm.com * Copyright (c) 2013, 2018-2019 ARM Limited 39243SN/A * All rights reserved 49243SN/A * 59243SN/A * The license below extends only to copyright in the software and shall 69243SN/A * not be construed as granting a license to any other intellectual 79243SN/A * property including but not limited to intellectual property relating 89243SN/A * to a hardware implementation of the functionality of the software 99243SN/A * licensed hereunder. You may use the software subject to the license 109243SN/A * terms below provided that you ensure that this notice is replicated 119243SN/A * unmodified and in its entirety in all distributions of the software, 129243SN/A * modified or unmodified, in source code or in binary form. 139243SN/A * 149831SN/A * Redistribution and use in source and binary forms, with or without 159831SN/A * modification, are permitted provided that the following conditions are 169831SN/A * met: redistributions of source code must retain the above copyright 179243SN/A * notice, this list of conditions and the following disclaimer; 189243SN/A * redistributions in binary form must reproduce the above copyright 199243SN/A * notice, this list of conditions and the following disclaimer in the 209243SN/A * documentation and/or other materials provided with the distribution; 219243SN/A * neither the name of the copyright holders nor the names of its 229243SN/A * contributors may be used to endorse or promote products derived from 239243SN/A * this software without specific prior written permission. 249243SN/A * 259243SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 269243SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 279243SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 289243SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 299243SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 309243SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 319243SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 329243SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 339243SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 349243SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 359243SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 369243SN/A * 379243SN/A * Authors: Stan Czerniawski 389243SN/A */ 399243SN/A 409243SN/A#include "dev/arm/smmu_v3.hh" 419243SN/A 429967SN/A#include <cstddef> 439243SN/A#include <cstring> 449243SN/A 4510146Sandreas.hansson@arm.com#include "base/bitfield.hh" 469356SN/A#include "base/cast.hh" 4710146Sandreas.hansson@arm.com#include "base/logging.hh" 4810247Sandreas.hansson@arm.com#include "base/trace.hh" 4910208Sandreas.hansson@arm.com#include "base/types.hh" 509352SN/A#include "debug/Checkpoint.hh" 5110146Sandreas.hansson@arm.com#include "debug/SMMUv3.hh" 529814SN/A#include "dev/arm/smmu_v3_transl.hh" 539243SN/A#include "mem/packet_access.hh" 549243SN/A#include "sim/system.hh" 559243SN/A 5610146Sandreas.hansson@arm.comSMMUv3::SMMUv3(SMMUv3Params *params) : 579243SN/A MemObject(params), 589243SN/A system(*params->system), 599243SN/A masterId(params->system->getMasterId(this)), 6010211Sandreas.hansson@arm.com masterPort(name() + ".master", *this), 6110208Sandreas.hansson@arm.com masterTableWalkPort(name() + ".master_walker", *this), 6210208Sandreas.hansson@arm.com controlPort(name() + ".control", *this, params->reg_map), 6310208Sandreas.hansson@arm.com tlb(params->tlb_entries, params->tlb_assoc, params->tlb_policy), 649831SN/A configCache(params->cfg_entries, params->cfg_assoc, params->cfg_policy), 659831SN/A ipaCache(params->ipa_entries, params->ipa_assoc, params->ipa_policy), 669831SN/A walkCache({ { params->walk_S1L0, params->walk_S1L1, 679831SN/A params->walk_S1L2, params->walk_S1L3, 689831SN/A params->walk_S2L0, params->walk_S2L1, 6910140SN/A params->walk_S2L2, params->walk_S2L3 } }, 7010286Sandreas.hansson@arm.com params->walk_assoc, params->walk_policy), 719243SN/A tlbEnable(params->tlb_enable), 729566SN/A configCacheEnable(params->cfg_enable), 739243SN/A ipaCacheEnable(params->ipa_enable), 749243SN/A walkCacheEnable(params->walk_enable), 7510140SN/A tableWalkPortEnable(false), 7610140SN/A walkCacheNonfinalEnable(params->wc_nonfinal_enable), 7710147Sandreas.hansson@arm.com walkCacheS1Levels(params->wc_s1_levels), 7810147Sandreas.hansson@arm.com walkCacheS2Levels(params->wc_s2_levels), 7910216Sandreas.hansson@arm.com masterPortWidth(params->master_port_width), 8010210Sandreas.hansson@arm.com tlbSem(params->tlb_slots), 8110212Sandreas.hansson@arm.com ifcSmmuSem(1), 829488SN/A smmuIfcSem(1), 839243SN/A configSem(params->cfg_slots), 849243SN/A ipaSem(params->ipa_slots), 8510141SN/A walkSem(params->walk_slots), 869726SN/A masterPortSem(1), 879726SN/A transSem(params->xlate_slots), 8810208Sandreas.hansson@arm.com ptwSem(params->ptw_slots), 8910208Sandreas.hansson@arm.com cycleSem(1), 9010208Sandreas.hansson@arm.com tlbLat(params->tlb_lat), 919243SN/A ifcSmmuLat(params->ifc_smmu_lat), 929243SN/A smmuIfcLat(params->smmu_ifc_lat), 939243SN/A configLat(params->cfg_lat), 949243SN/A ipaLat(params->ipa_lat), 959969SN/A walkLat(params->walk_lat), 969243SN/A slaveInterfaces(params->slave_interfaces), 979243SN/A commandExecutor(name() + ".cmd_exec", *this), 989969SN/A regsMap(params->reg_map), 999243SN/A processCommandsEvent(this) 1009243SN/A{ 10110246Sandreas.hansson@arm.com fatal_if(regsMap.size() != SMMU_REG_SIZE, 10210246Sandreas.hansson@arm.com "Invalid register map size: %#x different than SMMU_REG_SIZE = %#x\n", 10310246Sandreas.hansson@arm.com regsMap.size(), SMMU_REG_SIZE); 10410246Sandreas.hansson@arm.com 10510246Sandreas.hansson@arm.com // Init smmu registers to 0 10610246Sandreas.hansson@arm.com memset(®s, 0, sizeof(regs)); 10710246Sandreas.hansson@arm.com 10810246Sandreas.hansson@arm.com // Setup RO ID registers 10910140SN/A regs.idr0 = params->smmu_idr0; 11010140SN/A regs.idr1 = params->smmu_idr1; 11110140SN/A regs.idr2 = params->smmu_idr2; 11210140SN/A regs.idr3 = params->smmu_idr3; 11310140SN/A regs.idr4 = params->smmu_idr4; 1149243SN/A regs.idr5 = params->smmu_idr5; 1159243SN/A regs.iidr = params->smmu_iidr; 1169567SN/A regs.aidr = params->smmu_aidr; 1179243SN/A 1189243SN/A // TODO: At the moment it possible to set the ID registers to hold 1199243SN/A // any possible value. It would be nice to have a sanity check here 1209831SN/A // at construction time in case some idx registers are programmed to 1219831SN/A // store an unallowed values or if the are configuration conflicts. 1229831SN/A warn("SMMUv3 IDx register values unchecked\n"); 1239831SN/A 1249831SN/A for (auto ifc : slaveInterfaces) 1259243SN/A ifc->setSMMU(this); 12610286Sandreas.hansson@arm.com} 1279566SN/A 1289566SN/Abool 12910143SN/ASMMUv3::masterRecvTimingResp(PacketPtr pkt) 1309566SN/A{ 1319566SN/A DPRINTF(SMMUv3, "[t] master resp addr=%#x size=%#x\n", 13210136SN/A pkt->getAddr(), pkt->getSize()); 1339831SN/A 13410286Sandreas.hansson@arm.com // @todo: We need to pay for this and not just zero it out 13510136SN/A pkt->headerDelay = pkt->payloadDelay = 0; 1369566SN/A 13710286Sandreas.hansson@arm.com SMMUProcess *proc = 13810286Sandreas.hansson@arm.com safe_cast<SMMUProcess *>(pkt->popSenderState()); 13910286Sandreas.hansson@arm.com 14010286Sandreas.hansson@arm.com runProcessTiming(proc, pkt); 14110286Sandreas.hansson@arm.com 14210286Sandreas.hansson@arm.com return true; 14310286Sandreas.hansson@arm.com} 14410286Sandreas.hansson@arm.com 14510286Sandreas.hansson@arm.comvoid 14610286Sandreas.hansson@arm.comSMMUv3::masterRecvReqRetry() 14710286Sandreas.hansson@arm.com{ 14810286Sandreas.hansson@arm.com assert(!packetsToRetry.empty()); 14910286Sandreas.hansson@arm.com 15010286Sandreas.hansson@arm.com while (!packetsToRetry.empty()) { 15110286Sandreas.hansson@arm.com SMMUAction a = packetsToRetry.front(); 15210286Sandreas.hansson@arm.com 1539669SN/A assert(a.type==ACTION_SEND_REQ || a.type==ACTION_SEND_REQ_FINAL); 15410286Sandreas.hansson@arm.com 15510286Sandreas.hansson@arm.com DPRINTF(SMMUv3, "[t] master retr addr=%#x size=%#x\n", 15610286Sandreas.hansson@arm.com a.pkt->getAddr(), a.pkt->getSize()); 15710286Sandreas.hansson@arm.com 15810286Sandreas.hansson@arm.com if (!masterPort.sendTimingReq(a.pkt)) 15910286Sandreas.hansson@arm.com break; 16010286Sandreas.hansson@arm.com 16110286Sandreas.hansson@arm.com packetsToRetry.pop(); 1629566SN/A 1639566SN/A /* 16410207Sandreas.hansson@arm.com * ACTION_SEND_REQ_FINAL means that we have just forwarded the packet 16510207Sandreas.hansson@arm.com * on the master interface; this means that we no longer hold on to 16610207Sandreas.hansson@arm.com * that transaction and therefore can accept a new one. 16710207Sandreas.hansson@arm.com * If the slave port was stalled then unstall it (send retry). 16810207Sandreas.hansson@arm.com */ 16910207Sandreas.hansson@arm.com if (a.type == ACTION_SEND_REQ_FINAL) 1709243SN/A scheduleSlaveRetries(); 1719243SN/A } 1729243SN/A} 17310146Sandreas.hansson@arm.com 17410140SN/Abool 17510140SN/ASMMUv3::masterTableWalkRecvTimingResp(PacketPtr pkt) 17610146Sandreas.hansson@arm.com{ 17710140SN/A DPRINTF(SMMUv3, "[t] master HWTW resp addr=%#x size=%#x\n", 17810140SN/A pkt->getAddr(), pkt->getSize()); 17910140SN/A 18010140SN/A // @todo: We need to pay for this and not just zero it out 18110140SN/A pkt->headerDelay = pkt->payloadDelay = 0; 18210140SN/A 18310146Sandreas.hansson@arm.com SMMUProcess *proc = 1849243SN/A safe_cast<SMMUProcess *>(pkt->popSenderState()); 18510143SN/A 18610143SN/A runProcessTiming(proc, pkt); 18710208Sandreas.hansson@arm.com 18810143SN/A return true; 18910206Sandreas.hansson@arm.com} 19010206Sandreas.hansson@arm.com 19110206Sandreas.hansson@arm.comvoid 19210206Sandreas.hansson@arm.comSMMUv3::masterTableWalkRecvReqRetry() 19310206Sandreas.hansson@arm.com{ 19410206Sandreas.hansson@arm.com assert(tableWalkPortEnable); 19510207Sandreas.hansson@arm.com assert(!packetsTableWalkToRetry.empty()); 19610207Sandreas.hansson@arm.com 19710207Sandreas.hansson@arm.com while (!packetsTableWalkToRetry.empty()) { 1989243SN/A SMMUAction a = packetsTableWalkToRetry.front(); 1999243SN/A 2009243SN/A assert(a.type==ACTION_SEND_REQ); 20110146Sandreas.hansson@arm.com 2029243SN/A DPRINTF(SMMUv3, "[t] master HWTW retr addr=%#x size=%#x\n", 2039243SN/A a.pkt->getAddr(), a.pkt->getSize()); 2049243SN/A 2059243SN/A if (!masterTableWalkPort.sendTimingReq(a.pkt)) 2069243SN/A break; 2079243SN/A 2089243SN/A packetsTableWalkToRetry.pop(); 2099243SN/A } 2109243SN/A} 2119243SN/A 2129243SN/Avoid 2139243SN/ASMMUv3::scheduleSlaveRetries() 2149243SN/A{ 2159243SN/A for (auto ifc : slaveInterfaces) { 2169243SN/A ifc->scheduleDeviceRetry(); 2179243SN/A } 21810146Sandreas.hansson@arm.com} 2199243SN/A 2209831SN/ASMMUAction 2219831SN/ASMMUv3::runProcess(SMMUProcess *proc, PacketPtr pkt) 2229831SN/A{ 2239243SN/A if (system.isAtomicMode()) { 2249831SN/A return runProcessAtomic(proc, pkt); 2259831SN/A } else if (system.isTimingMode()) { 2269243SN/A return runProcessTiming(proc, pkt); 2279243SN/A } else { 2289243SN/A panic("Not in timing or atomic mode!"); 22910146Sandreas.hansson@arm.com } 2309243SN/A} 2319831SN/A 2329831SN/ASMMUAction 2339831SN/ASMMUv3::runProcessAtomic(SMMUProcess *proc, PacketPtr pkt) 2349243SN/A{ 2359243SN/A SMMUAction action; 23610146Sandreas.hansson@arm.com Tick delay = 0; 23710146Sandreas.hansson@arm.com bool finished = false; 23810143SN/A 2399243SN/A do { 2409669SN/A action = proc->run(pkt); 24110136SN/A 24210136SN/A switch (action.type) { 2439243SN/A case ACTION_SEND_REQ: 2449967SN/A // Send an MMU initiated request on the table walk port if it is 24510245Sandreas.hansson@arm.com // enabled. Otherwise, fall through and handle same as the final 24610245Sandreas.hansson@arm.com // ACTION_SEND_REQ_FINAL request. 24710245Sandreas.hansson@arm.com if (tableWalkPortEnable) { 2489243SN/A delay += masterTableWalkPort.sendAtomic(action.pkt); 24910286Sandreas.hansson@arm.com pkt = action.pkt; 25010286Sandreas.hansson@arm.com break; 2519831SN/A } 2529243SN/A M5_FALLTHROUGH; 2539491SN/A case ACTION_SEND_REQ_FINAL: 2549831SN/A delay += masterPort.sendAtomic(action.pkt); 25510136SN/A pkt = action.pkt; 2569491SN/A break; 2579491SN/A 2589831SN/A case ACTION_SEND_RESP: 2599243SN/A case ACTION_SEND_RESP_ATS: 2609669SN/A case ACTION_SLEEP: 2619566SN/A finished = true; 2629566SN/A break; 2639669SN/A 2649669SN/A case ACTION_DELAY: 2659669SN/A delay += action.delay; 2669669SN/A break; 2679669SN/A 2689669SN/A case ACTION_TERMINATE: 2699669SN/A panic("ACTION_TERMINATE in atomic mode\n"); 2709669SN/A 2719669SN/A default: 2729669SN/A panic("Unknown action\n"); 2739669SN/A } 2749669SN/A } while (!finished); 2759669SN/A 27610136SN/A action.delay = delay; 27710286Sandreas.hansson@arm.com 27810286Sandreas.hansson@arm.com return action; 27910286Sandreas.hansson@arm.com} 2809669SN/A 2819669SN/ASMMUAction 2829669SN/ASMMUv3::runProcessTiming(SMMUProcess *proc, PacketPtr pkt) 28310286Sandreas.hansson@arm.com{ 28410286Sandreas.hansson@arm.com SMMUAction action = proc->run(pkt); 2859669SN/A 2869669SN/A switch (action.type) { 2879491SN/A case ACTION_SEND_REQ: 2889243SN/A // Send an MMU initiated request on the table walk port if it is 2899243SN/A // enabled. Otherwise, fall through and handle same as the final 2909243SN/A // ACTION_SEND_REQ_FINAL request. 2919491SN/A if (tableWalkPortEnable) { 2929491SN/A action.pkt->pushSenderState(proc); 2939243SN/A 2949243SN/A DPRINTF(SMMUv3, "[t] master HWTW req addr=%#x size=%#x\n", 2959243SN/A action.pkt->getAddr(), action.pkt->getSize()); 2969491SN/A 2979243SN/A if (packetsTableWalkToRetry.empty() 2989243SN/A && masterTableWalkPort.sendTimingReq(action.pkt)) { 29910136SN/A scheduleSlaveRetries(); 3009491SN/A } else { 3019491SN/A DPRINTF(SMMUv3, "[t] master HWTW req needs retry," 3029491SN/A " qlen=%d\n", packetsTableWalkToRetry.size()); 30310286Sandreas.hansson@arm.com packetsTableWalkToRetry.push(action); 30410286Sandreas.hansson@arm.com } 30510286Sandreas.hansson@arm.com 3069566SN/A break; 3079566SN/A } 3089566SN/A M5_FALLTHROUGH; 3099566SN/A case ACTION_SEND_REQ_FINAL: 3109566SN/A action.pkt->pushSenderState(proc); 3119491SN/A 3129491SN/A DPRINTF(SMMUv3, "[t] master req addr=%#x size=%#x\n", 3139243SN/A action.pkt->getAddr(), action.pkt->getSize()); 3149243SN/A 3159243SN/A if (packetsToRetry.empty() && masterPort.sendTimingReq(action.pkt)) { 3169491SN/A scheduleSlaveRetries(); 3179243SN/A } else { 3189243SN/A DPRINTF(SMMUv3, "[t] master req needs retry, qlen=%d\n", 3199243SN/A packetsToRetry.size()); 32010286Sandreas.hansson@arm.com packetsToRetry.push(action); 32110286Sandreas.hansson@arm.com } 3229243SN/A 3239491SN/A break; 3249243SN/A 3259243SN/A case ACTION_SEND_RESP: 3269243SN/A // @todo: We need to pay for this and not just zero it out 3279243SN/A action.pkt->headerDelay = action.pkt->payloadDelay = 0; 3289243SN/A 3299243SN/A DPRINTF(SMMUv3, "[t] slave resp addr=%#x size=%#x\n", 3309243SN/A action.pkt->getAddr(), 3319243SN/A action.pkt->getSize()); 33210245Sandreas.hansson@arm.com 3339243SN/A assert(action.ifc); 3349243SN/A action.ifc->schedTimingResp(action.pkt); 3359831SN/A 3369243SN/A delete proc; 3379243SN/A break; 3389567SN/A 3399567SN/A case ACTION_SEND_RESP_ATS: 3409967SN/A // @todo: We need to pay for this and not just zero it out 3419967SN/A action.pkt->headerDelay = action.pkt->payloadDelay = 0; 3429967SN/A 3439243SN/A DPRINTF(SMMUv3, "[t] ATS slave resp addr=%#x size=%#x\n", 3449243SN/A action.pkt->getAddr(), action.pkt->getSize()); 3459243SN/A 34610146Sandreas.hansson@arm.com assert(action.ifc); 3479243SN/A action.ifc->schedAtsTimingResp(action.pkt); 3489243SN/A 3499243SN/A delete proc; 3509243SN/A break; 3519243SN/A 3529831SN/A case ACTION_DELAY: 3539831SN/A case ACTION_SLEEP: 3549831SN/A break; 3559831SN/A 3569831SN/A case ACTION_TERMINATE: 3579831SN/A delete proc; 3589831SN/A break; 3599831SN/A 3609243SN/A default: 3619831SN/A panic("Unknown action\n"); 3629831SN/A } 3639831SN/A 3649831SN/A return action; 3659831SN/A} 3669831SN/A 3679831SN/Avoid 3689243SN/ASMMUv3::processCommands() 3699831SN/A{ 3709831SN/A DPRINTF(SMMUv3, "processCommands()\n"); 3719831SN/A 3729833SN/A if (system.isAtomicMode()) { 3739832SN/A SMMUAction a = runProcessAtomic(&commandExecutor, NULL); 3749832SN/A (void) a; 3759832SN/A } else if (system.isTimingMode()) { 3769832SN/A if (!commandExecutor.isBusy()) 3779831SN/A runProcessTiming(&commandExecutor, NULL); 3789831SN/A } else { 3799831SN/A panic("Not in timing or atomic mode!"); 3809831SN/A } 3819831SN/A} 3829975SN/A 3839831SN/Avoid 3849831SN/ASMMUv3::processCommand(const SMMUCommand &cmd) 3859243SN/A{ 3869831SN/A switch (cmd.dw0.type) { 3879831SN/A case CMD_PRF_CONFIG: 3889831SN/A DPRINTF(SMMUv3, "CMD_PREFETCH_CONFIG - ignored\n"); 3899831SN/A break; 3909831SN/A 3919831SN/A case CMD_PRF_ADDR: 3929831SN/A DPRINTF(SMMUv3, "CMD_PREFETCH_ADDR - ignored\n"); 3939831SN/A break; 3949831SN/A 3959831SN/A case CMD_CFGI_STE: { 3969831SN/A DPRINTF(SMMUv3, "CMD_CFGI_STE sid=%#x\n", cmd.dw0.sid); 3979831SN/A configCache.invalidateSID(cmd.dw0.sid); 3989966SN/A 3999831SN/A for (auto slave_interface : slaveInterfaces) { 4009831SN/A slave_interface->microTLB->invalidateSID(cmd.dw0.sid); 4019831SN/A slave_interface->mainTLB->invalidateSID(cmd.dw0.sid); 4029831SN/A } 4039831SN/A break; 4049831SN/A } 4059831SN/A 4069831SN/A case CMD_CFGI_STE_RANGE: { 4079831SN/A const auto range = cmd.dw1.range; 4089831SN/A if (range == 31) { 4099831SN/A // CMD_CFGI_ALL is an alias of CMD_CFGI_STE_RANGE with 4109831SN/A // range = 31 4119831SN/A DPRINTF(SMMUv3, "CMD_CFGI_ALL\n"); 4129831SN/A configCache.invalidateAll(); 4139831SN/A 4149243SN/A for (auto slave_interface : slaveInterfaces) { 4159243SN/A slave_interface->microTLB->invalidateAll(); 4169831SN/A slave_interface->mainTLB->invalidateAll(); 4179831SN/A } 4189831SN/A } else { 4199831SN/A DPRINTF(SMMUv3, "CMD_CFGI_STE_RANGE\n"); 4209831SN/A const auto start_sid = cmd.dw0.sid & ~((1 << (range + 1)) - 1); 4219243SN/A const auto end_sid = start_sid + (1 << (range + 1)) - 1; 4229831SN/A for (auto sid = start_sid; sid <= end_sid; sid++) { 4239831SN/A configCache.invalidateSID(sid); 4249831SN/A 4259243SN/A for (auto slave_interface : slaveInterfaces) { 42610206Sandreas.hansson@arm.com slave_interface->microTLB->invalidateSID(sid); 42710206Sandreas.hansson@arm.com slave_interface->mainTLB->invalidateSID(sid); 42810206Sandreas.hansson@arm.com } 4299567SN/A } 4309567SN/A } 4319243SN/A break; 4329243SN/A } 4339243SN/A 4349243SN/A case CMD_CFGI_CD: { 43510146Sandreas.hansson@arm.com DPRINTF(SMMUv3, "CMD_CFGI_CD sid=%#x ssid=%#x\n", 4369243SN/A cmd.dw0.sid, cmd.dw0.ssid); 4379243SN/A configCache.invalidateSSID(cmd.dw0.sid, cmd.dw0.ssid); 4389243SN/A 4399243SN/A for (auto slave_interface : slaveInterfaces) { 4409243SN/A slave_interface->microTLB->invalidateSSID( 4419831SN/A cmd.dw0.sid, cmd.dw0.ssid); 4429831SN/A slave_interface->mainTLB->invalidateSSID( 4439831SN/A cmd.dw0.sid, cmd.dw0.ssid); 4449831SN/A } 4459831SN/A break; 4469831SN/A } 4479831SN/A 4489831SN/A case CMD_CFGI_CD_ALL: { 4499243SN/A DPRINTF(SMMUv3, "CMD_CFGI_CD_ALL sid=%#x\n", cmd.dw0.sid); 4509832SN/A configCache.invalidateSID(cmd.dw0.sid); 4519838SN/A 4529838SN/A for (auto slave_interface : slaveInterfaces) { 4539838SN/A slave_interface->microTLB->invalidateSID(cmd.dw0.sid); 4549832SN/A slave_interface->mainTLB->invalidateSID(cmd.dw0.sid); 4559832SN/A } 4569243SN/A break; 4579832SN/A } 4589832SN/A 4599832SN/A case CMD_TLBI_NH_ALL: { 4609832SN/A DPRINTF(SMMUv3, "CMD_TLBI_NH_ALL vmid=%#x\n", cmd.dw0.vmid); 4619838SN/A for (auto slave_interface : slaveInterfaces) { 4629838SN/A slave_interface->microTLB->invalidateVMID(cmd.dw0.vmid); 4639838SN/A slave_interface->mainTLB->invalidateVMID(cmd.dw0.vmid); 4649832SN/A } 4659832SN/A tlb.invalidateVMID(cmd.dw0.vmid); 4669832SN/A walkCache.invalidateVMID(cmd.dw0.vmid); 4679832SN/A break; 4689832SN/A } 4699832SN/A 4709832SN/A case CMD_TLBI_NH_ASID: { 4719832SN/A DPRINTF(SMMUv3, "CMD_TLBI_NH_ASID asid=%#x vmid=%#x\n", 4729832SN/A cmd.dw0.asid, cmd.dw0.vmid); 4739832SN/A for (auto slave_interface : slaveInterfaces) { 4749832SN/A slave_interface->microTLB->invalidateASID( 4759832SN/A cmd.dw0.asid, cmd.dw0.vmid); 4769832SN/A slave_interface->mainTLB->invalidateASID( 4779832SN/A cmd.dw0.asid, cmd.dw0.vmid); 4789832SN/A } 4799832SN/A tlb.invalidateASID(cmd.dw0.asid, cmd.dw0.vmid); 4809832SN/A walkCache.invalidateASID(cmd.dw0.asid, cmd.dw0.vmid); 48110047SN/A break; 4829832SN/A } 4839832SN/A 4849832SN/A case CMD_TLBI_NH_VAA: { 4859838SN/A const Addr addr = cmd.addr(); 4869838SN/A DPRINTF(SMMUv3, "CMD_TLBI_NH_VAA va=%#08x vmid=%#x\n", 4879838SN/A addr, cmd.dw0.vmid); 4889832SN/A for (auto slave_interface : slaveInterfaces) { 4899832SN/A slave_interface->microTLB->invalidateVAA( 4909832SN/A addr, cmd.dw0.vmid); 4919832SN/A slave_interface->mainTLB->invalidateVAA( 4929832SN/A addr, cmd.dw0.vmid); 4939832SN/A } 4949832SN/A tlb.invalidateVAA(addr, cmd.dw0.vmid); 4959832SN/A 4969832SN/A if (!cmd.dw1.leaf) 4979832SN/A walkCache.invalidateVAA(addr, cmd.dw0.vmid); 4989832SN/A break; 4999832SN/A } 5009832SN/A 5019832SN/A case CMD_TLBI_NH_VA: { 5029832SN/A const Addr addr = cmd.addr(); 5039832SN/A DPRINTF(SMMUv3, "CMD_TLBI_NH_VA va=%#08x asid=%#x vmid=%#x\n", 5049832SN/A addr, cmd.dw0.asid, cmd.dw0.vmid); 5059832SN/A for (auto slave_interface : slaveInterfaces) { 5069832SN/A slave_interface->microTLB->invalidateVA( 5079832SN/A addr, cmd.dw0.asid, cmd.dw0.vmid); 5089243SN/A slave_interface->mainTLB->invalidateVA( 5099832SN/A addr, cmd.dw0.asid, cmd.dw0.vmid); 5109832SN/A } 5119832SN/A tlb.invalidateVA(addr, cmd.dw0.asid, cmd.dw0.vmid); 5129966SN/A 5139243SN/A if (!cmd.dw1.leaf) 5149832SN/A walkCache.invalidateVA(addr, cmd.dw0.asid, cmd.dw0.vmid); 5159832SN/A break; 5169243SN/A } 5179832SN/A 5189831SN/A case CMD_TLBI_S2_IPA: { 5199832SN/A const Addr addr = cmd.addr(); 5209831SN/A DPRINTF(SMMUv3, "CMD_TLBI_S2_IPA ipa=%#08x vmid=%#x\n", 5219832SN/A addr, cmd.dw0.vmid); 5229832SN/A // This does not invalidate TLBs containing 5239977SN/A // combined Stage1 + Stage2 translations, as per the spec. 5249977SN/A ipaCache.invalidateIPA(addr, cmd.dw0.vmid); 5259977SN/A 5269977SN/A if (!cmd.dw1.leaf) 5279832SN/A walkCache.invalidateVMID(cmd.dw0.vmid); 5289832SN/A break; 5299831SN/A } 5309831SN/A 5319831SN/A case CMD_TLBI_S12_VMALL: { 5329243SN/A DPRINTF(SMMUv3, "CMD_TLBI_S12_VMALL vmid=%#x\n", cmd.dw0.vmid); 5339243SN/A for (auto slave_interface : slaveInterfaces) { 5349243SN/A slave_interface->microTLB->invalidateVMID(cmd.dw0.vmid); 5359243SN/A slave_interface->mainTLB->invalidateVMID(cmd.dw0.vmid); 5369831SN/A } 5379831SN/A tlb.invalidateVMID(cmd.dw0.vmid); 5389726SN/A ipaCache.invalidateVMID(cmd.dw0.vmid); 5399243SN/A walkCache.invalidateVMID(cmd.dw0.vmid); 54010206Sandreas.hansson@arm.com break; 54110206Sandreas.hansson@arm.com } 54210206Sandreas.hansson@arm.com 54310206Sandreas.hansson@arm.com case CMD_TLBI_NSNH_ALL: { 54410206Sandreas.hansson@arm.com DPRINTF(SMMUv3, "CMD_TLBI_NSNH_ALL\n"); 5459243SN/A for (auto slave_interface : slaveInterfaces) { 5469243SN/A slave_interface->microTLB->invalidateAll(); 5479243SN/A slave_interface->mainTLB->invalidateAll(); 5489243SN/A } 54910146Sandreas.hansson@arm.com tlb.invalidateAll(); 5509243SN/A ipaCache.invalidateAll(); 5519833SN/A walkCache.invalidateAll(); 5529243SN/A break; 5539243SN/A } 5549243SN/A 5559833SN/A case CMD_RESUME: 5569243SN/A DPRINTF(SMMUv3, "CMD_RESUME\n"); 5579243SN/A panic("resume unimplemented"); 5589243SN/A break; 5599833SN/A 5609243SN/A default: 5619243SN/A warn("Unimplemented command %#x\n", cmd.dw0.type); 5629243SN/A break; 5639243SN/A } 5649243SN/A} 56510146Sandreas.hansson@arm.com 5669243SN/Aconst PageTableOps* 5679349SN/ASMMUv3::getPageTableOps(uint8_t trans_granule) 5689349SN/A{ 5699349SN/A static V8PageTableOps4k ptOps4k; 5709349SN/A static V8PageTableOps16k ptOps16k; 5719349SN/A static V8PageTableOps64k ptOps64k; 5729349SN/A 5739243SN/A switch (trans_granule) { 5749567SN/A case TRANS_GRANULE_4K: return &ptOps4k; 5759831SN/A case TRANS_GRANULE_16K: return &ptOps16k; 5769243SN/A case TRANS_GRANULE_64K: return &ptOps64k; 5779567SN/A default: 5789567SN/A panic("Unknown translation granule size %d", trans_granule); 57910143SN/A } 5809567SN/A} 5819567SN/A 5829567SN/ATick 5839243SN/ASMMUv3::readControl(PacketPtr pkt) 5849243SN/A{ 5859243SN/A DPRINTF(SMMUv3, "readControl: addr=%08x size=%d\n", 5869243SN/A pkt->getAddr(), pkt->getSize()); 5879243SN/A 5889243SN/A int offset = pkt->getAddr() - regsMap.start(); 5899243SN/A assert(offset >= 0 && offset < SMMU_REG_SIZE); 5909831SN/A 5919831SN/A if (inSecureBlock(offset)) { 5929831SN/A warn("smmu: secure registers (0x%x) are not implemented\n", 5939831SN/A offset); 5949831SN/A } 5959243SN/A 5969831SN/A auto reg_ptr = regs.data + offset; 5979831SN/A 5989243SN/A switch (pkt->getSize()) { 5999243SN/A case sizeof(uint32_t): 6009243SN/A pkt->setLE<uint32_t>(*reinterpret_cast<uint32_t *>(reg_ptr)); 6019567SN/A break; 6029831SN/A case sizeof(uint64_t): 6039567SN/A pkt->setLE<uint64_t>(*reinterpret_cast<uint64_t *>(reg_ptr)); 6049243SN/A break; 6059243SN/A default: 6069243SN/A panic("smmu: unallowed access size: %d bytes\n", pkt->getSize()); 6079243SN/A break; 6089243SN/A } 6099831SN/A 6109243SN/A pkt->makeAtomicResponse(); 6119977SN/A 6129243SN/A return 0; 6139243SN/A} 6149567SN/A 6159831SN/ATick 6169567SN/ASMMUv3::writeControl(PacketPtr pkt) 6179243SN/A{ 6189243SN/A int offset = pkt->getAddr() - regsMap.start(); 6199243SN/A assert(offset >= 0 && offset < SMMU_REG_SIZE); 6209243SN/A 6219243SN/A DPRINTF(SMMUv3, "writeControl: addr=%08x size=%d data=%16x\n", 6229831SN/A pkt->getAddr(), pkt->getSize(), 6239243SN/A pkt->getSize() == sizeof(uint64_t) ? 6249977SN/A pkt->getLE<uint64_t>() : pkt->getLE<uint32_t>()); 6259243SN/A 6269243SN/A switch (offset) { 6279243SN/A case offsetof(SMMURegs, cr0): 6289243SN/A assert(pkt->getSize() == sizeof(uint32_t)); 6299726SN/A regs.cr0 = regs.cr0ack = pkt->getLE<uint32_t>(); 6309243SN/A break; 6319243SN/A 6329243SN/A case offsetof(SMMURegs, cr1): 6339243SN/A case offsetof(SMMURegs, cr2): 6349243SN/A case offsetof(SMMURegs, strtab_base_cfg): 6359243SN/A case offsetof(SMMURegs, eventq_cons): 63610146Sandreas.hansson@arm.com case offsetof(SMMURegs, eventq_irq_cfg1): 6379243SN/A case offsetof(SMMURegs, priq_cons): 6389243SN/A assert(pkt->getSize() == sizeof(uint32_t)); 6399243SN/A *reinterpret_cast<uint32_t *>(regs.data + offset) = 6409243SN/A pkt->getLE<uint32_t>(); 6419831SN/A break; 6429243SN/A 6439831SN/A case offsetof(SMMURegs, cmdq_cons): 6449831SN/A assert(pkt->getSize() == sizeof(uint32_t)); 6459831SN/A if (regs.cr0 & CR0_CMDQEN_MASK) { 6469831SN/A warn("CMDQ is enabled: ignoring write to CMDQ_CONS\n"); 64710143SN/A } else { 6489831SN/A *reinterpret_cast<uint32_t *>(regs.data + offset) = 6499831SN/A pkt->getLE<uint32_t>(); 6509831SN/A } 6519831SN/A break; 6529831SN/A 6539831SN/A case offsetof(SMMURegs, cmdq_prod): 6549831SN/A assert(pkt->getSize() == sizeof(uint32_t)); 6559831SN/A *reinterpret_cast<uint32_t *>(regs.data + offset) = 6569831SN/A pkt->getLE<uint32_t>(); 6579831SN/A schedule(processCommandsEvent, nextCycle()); 6589831SN/A break; 6599831SN/A 6609243SN/A case offsetof(SMMURegs, strtab_base): 6619831SN/A case offsetof(SMMURegs, eventq_irq_cfg0): 6629831SN/A assert(pkt->getSize() == sizeof(uint64_t)); 6639243SN/A *reinterpret_cast<uint64_t *>(regs.data + offset) = 6649831SN/A pkt->getLE<uint64_t>(); 6659831SN/A break; 6669831SN/A 6679831SN/A case offsetof(SMMURegs, cmdq_base): 6689831SN/A assert(pkt->getSize() == sizeof(uint64_t)); 6699831SN/A if (regs.cr0 & CR0_CMDQEN_MASK) { 6709831SN/A warn("CMDQ is enabled: ignoring write to CMDQ_BASE\n"); 6719831SN/A } else { 6729831SN/A *reinterpret_cast<uint64_t *>(regs.data + offset) = 6739831SN/A pkt->getLE<uint64_t>(); 6749831SN/A regs.cmdq_cons = 0; 6759831SN/A regs.cmdq_prod = 0; 6769567SN/A } 6779831SN/A break; 6789831SN/A 6799831SN/A case offsetof(SMMURegs, eventq_base): 6809831SN/A assert(pkt->getSize() == sizeof(uint64_t)); 6819831SN/A *reinterpret_cast<uint64_t *>(regs.data + offset) = 6829831SN/A pkt->getLE<uint64_t>(); 6839243SN/A regs.eventq_cons = 0; 6849243SN/A regs.eventq_prod = 0; 6859243SN/A break; 68610206Sandreas.hansson@arm.com 6879243SN/A case offsetof(SMMURegs, priq_base): 68810206Sandreas.hansson@arm.com assert(pkt->getSize() == sizeof(uint64_t)); 68910206Sandreas.hansson@arm.com *reinterpret_cast<uint64_t *>(regs.data + offset) = 69010206Sandreas.hansson@arm.com pkt->getLE<uint64_t>(); 69110206Sandreas.hansson@arm.com regs.priq_cons = 0; 69210206Sandreas.hansson@arm.com regs.priq_prod = 0; 6939243SN/A break; 69410206Sandreas.hansson@arm.com 69510206Sandreas.hansson@arm.com default: 6969243SN/A if (inSecureBlock(offset)) { 6979243SN/A warn("smmu: secure registers (0x%x) are not implemented\n", 6989243SN/A offset); 6999243SN/A } else { 7009243SN/A warn("smmu: write to read-only/undefined register at 0x%x\n", 7019243SN/A offset); 70210206Sandreas.hansson@arm.com } 7039243SN/A } 7049243SN/A 7059243SN/A pkt->makeAtomicResponse(); 7069243SN/A 7079243SN/A return 0; 70810146Sandreas.hansson@arm.com} 7099974SN/A 7109974SN/Abool 7119974SN/ASMMUv3::inSecureBlock(uint32_t offs) const 7129974SN/A{ 7139974SN/A if (offs >= offsetof(SMMURegs, _secure_regs) && offs < SMMU_SECURE_SZ) 7149974SN/A return true; 7159974SN/A else 7169974SN/A return false; 7179974SN/A} 7189974SN/A 7199974SN/Avoid 7209974SN/ASMMUv3::init() 7219974SN/A{ 7229974SN/A // make sure both sides are connected and have the same block size 72310211Sandreas.hansson@arm.com if (!masterPort.isConnected()) 7249974SN/A fatal("Master port is not connected.\n"); 7259974SN/A 7269974SN/A // If the second master port is connected for the table walks, enable 7279974SN/A // the mode to send table walks through this port instead 7289974SN/A if (masterTableWalkPort.isConnected()) 7299974SN/A tableWalkPortEnable = true; 73010211Sandreas.hansson@arm.com 73110211Sandreas.hansson@arm.com // notify the master side of our address ranges 73210211Sandreas.hansson@arm.com for (auto ifc : slaveInterfaces) { 73310211Sandreas.hansson@arm.com ifc->sendRange(); 73410211Sandreas.hansson@arm.com } 73510211Sandreas.hansson@arm.com 73610211Sandreas.hansson@arm.com if (controlPort.isConnected()) 73710211Sandreas.hansson@arm.com controlPort.sendRangeChange(); 7389974SN/A} 7399974SN/A 74010211Sandreas.hansson@arm.comvoid 7419974SN/ASMMUv3::regStats() 7429974SN/A{ 74310211Sandreas.hansson@arm.com MemObject::regStats(); 7449974SN/A 7459974SN/A using namespace Stats; 7469974SN/A 7479974SN/A for (size_t i = 0; i < slaveInterfaces.size(); i++) { 7489974SN/A slaveInterfaces[i]->microTLB->regStats( 7499974SN/A csprintf("%s.utlb%d", name(), i)); 7509974SN/A slaveInterfaces[i]->mainTLB->regStats( 7519974SN/A csprintf("%s.maintlb%d", name(), i)); 7529974SN/A } 7539974SN/A 7549974SN/A tlb.regStats(name() + ".tlb"); 7559974SN/A configCache.regStats(name() + ".cfg"); 75610146Sandreas.hansson@arm.com ipaCache.regStats(name() + ".ipa"); 7579243SN/A walkCache.regStats(name() + ".walk"); 7589243SN/A 7599243SN/A steL1Fetches 7609243SN/A .name(name() + ".steL1Fetches") 7619243SN/A .desc("STE L1 fetches") 7629243SN/A .flags(pdf); 7639243SN/A 7649243SN/A steFetches 7659243SN/A .name(name() + ".steFetches") 7669243SN/A .desc("STE fetches") 7679243SN/A .flags(pdf); 7689243SN/A 7699243SN/A cdL1Fetches 7709549SN/A .name(name() + ".cdL1Fetches") 7719549SN/A .desc("CD L1 fetches") 7729549SN/A .flags(pdf); 7739726SN/A 7749726SN/A cdFetches 7759726SN/A .name(name() + ".cdFetches") 7769243SN/A .desc("CD fetches") 7779587SN/A .flags(pdf); 7789587SN/A 7799587SN/A translationTimeDist 7809243SN/A .init(0, 2000000, 2000) 7819243SN/A .name(name() + ".translationTimeDist") 7829243SN/A .desc("Time to translate address") 7839243SN/A .flags(pdf); 7849243SN/A 7859243SN/A ptwTimeDist 7869243SN/A .init(0, 2000000, 2000) 7879243SN/A .name(name() + ".ptwTimeDist") 78810246Sandreas.hansson@arm.com .desc("Time to walk page tables") 7899488SN/A .flags(pdf); 79010246Sandreas.hansson@arm.com} 79110246Sandreas.hansson@arm.com 79210246Sandreas.hansson@arm.comDrainState 7939969SN/ASMMUv3::drain() 7949488SN/A{ 7959488SN/A // Wait until the Command Executor is not busy 7969488SN/A if (commandExecutor.isBusy()) { 79710207Sandreas.hansson@arm.com return DrainState::Draining; 79810246Sandreas.hansson@arm.com } 79910246Sandreas.hansson@arm.com return DrainState::Drained; 80010207Sandreas.hansson@arm.com} 80110207Sandreas.hansson@arm.com 80210207Sandreas.hansson@arm.comvoid 80310207Sandreas.hansson@arm.comSMMUv3::serialize(CheckpointOut &cp) const 80410246Sandreas.hansson@arm.com{ 80510246Sandreas.hansson@arm.com DPRINTF(Checkpoint, "Serializing SMMUv3\n"); 80610207Sandreas.hansson@arm.com 80710207Sandreas.hansson@arm.com SERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0])); 80810207Sandreas.hansson@arm.com} 80910207Sandreas.hansson@arm.com 81010247Sandreas.hansson@arm.comvoid 81110247Sandreas.hansson@arm.comSMMUv3::unserialize(CheckpointIn &cp) 81210247Sandreas.hansson@arm.com{ 81310247Sandreas.hansson@arm.com DPRINTF(Checkpoint, "Unserializing SMMUv3\n"); 81410247Sandreas.hansson@arm.com 8159975SN/A UNSERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0])); 81610211Sandreas.hansson@arm.com} 81710246Sandreas.hansson@arm.com 81810211Sandreas.hansson@arm.comPort& 81910211Sandreas.hansson@arm.comSMMUv3::getPort(const std::string &name, PortID id) 82010246Sandreas.hansson@arm.com{ 82110211Sandreas.hansson@arm.com if (name == "master") { 8229971SN/A return masterPort; 8239971SN/A } else if (name == "master_walker") { 82410210Sandreas.hansson@arm.com return masterTableWalkPort; 82510210Sandreas.hansson@arm.com } else if (name == "control") { 82610210Sandreas.hansson@arm.com return controlPort; 82710210Sandreas.hansson@arm.com } else { 8289971SN/A return MemObject::getPort(name, id); 82910208Sandreas.hansson@arm.com } 8309971SN/A} 8319971SN/A 8329969SN/ASMMUv3* 8339824SN/ASMMUv3Params::create() 8349824SN/A{ 8359488SN/A return new SMMUv3(this); 8369969SN/A} 83710210Sandreas.hansson@arm.com