tlb.cc revision 13889
112837Sgabeblack@google.com/* 212837Sgabeblack@google.com * Copyright (c) 2010-2013, 2016-2019 ARM Limited 312837Sgabeblack@google.com * All rights reserved 412837Sgabeblack@google.com * 512837Sgabeblack@google.com * The license below extends only to copyright in the software and shall 612837Sgabeblack@google.com * not be construed as granting a license to any other intellectual 712837Sgabeblack@google.com * property including but not limited to intellectual property relating 812837Sgabeblack@google.com * to a hardware implementation of the functionality of the software 912837Sgabeblack@google.com * licensed hereunder. You may use the software subject to the license 1012837Sgabeblack@google.com * terms below provided that you ensure that this notice is replicated 1112837Sgabeblack@google.com * unmodified and in its entirety in all distributions of the software, 1212837Sgabeblack@google.com * modified or unmodified, in source code or in binary form. 1312837Sgabeblack@google.com * 1412837Sgabeblack@google.com * Copyright (c) 2001-2005 The Regents of The University of Michigan 1512837Sgabeblack@google.com * All rights reserved. 1612837Sgabeblack@google.com * 1712837Sgabeblack@google.com * Redistribution and use in source and binary forms, with or without 1812837Sgabeblack@google.com * modification, are permitted provided that the following conditions are 1912837Sgabeblack@google.com * met: redistributions of source code must retain the above copyright 2012837Sgabeblack@google.com * notice, this list of conditions and the following disclaimer; 2112837Sgabeblack@google.com * redistributions in binary form must reproduce the above copyright 2212837Sgabeblack@google.com * notice, this list of conditions and the following disclaimer in the 2312837Sgabeblack@google.com * documentation and/or other materials provided with the distribution; 2412837Sgabeblack@google.com * neither the name of the copyright holders nor the names of its 2512837Sgabeblack@google.com * contributors may be used to endorse or promote products derived from 2612837Sgabeblack@google.com * this software without specific prior written permission. 2712837Sgabeblack@google.com * 2812837Sgabeblack@google.com * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2912837Sgabeblack@google.com * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 3012837Sgabeblack@google.com * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 3112837Sgabeblack@google.com * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 3212837Sgabeblack@google.com * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 3313051Sgabeblack@google.com * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 3412955Sgabeblack@google.com * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 3513294Sgabeblack@google.com * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 3612837Sgabeblack@google.com * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3712837Sgabeblack@google.com * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3813294Sgabeblack@google.com * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3913051Sgabeblack@google.com * 4012837Sgabeblack@google.com * Authors: Ali Saidi 4112837Sgabeblack@google.com * Nathan Binkert 4212955Sgabeblack@google.com * Steve Reinhardt 4312955Sgabeblack@google.com */ 4412955Sgabeblack@google.com 4512955Sgabeblack@google.com#include "arch/arm/tlb.hh" 4613206Sgabeblack@google.com 4713206Sgabeblack@google.com#include <memory> 4813303Sgabeblack@google.com#include <string> 4912955Sgabeblack@google.com#include <vector> 5012955Sgabeblack@google.com 5112955Sgabeblack@google.com#include "arch/arm/faults.hh" 5212837Sgabeblack@google.com#include "arch/arm/pagetable.hh" 5312837Sgabeblack@google.com#include "arch/arm/stage2_lookup.hh" 5412837Sgabeblack@google.com#include "arch/arm/stage2_mmu.hh" 5512837Sgabeblack@google.com#include "arch/arm/system.hh" 5612837Sgabeblack@google.com#include "arch/arm/table_walker.hh" 5712837Sgabeblack@google.com#include "arch/arm/utility.hh" 5812924Sgabeblack@google.com#include "arch/generic/mmapped_ipr.hh" 5912837Sgabeblack@google.com#include "base/inifile.hh" 6012837Sgabeblack@google.com#include "base/str.hh" 6112837Sgabeblack@google.com#include "base/trace.hh" 6212837Sgabeblack@google.com#include "cpu/base.hh" 6312837Sgabeblack@google.com#include "cpu/thread_context.hh" 6412837Sgabeblack@google.com#include "debug/Checkpoint.hh" 6512837Sgabeblack@google.com#include "debug/TLB.hh" 6612837Sgabeblack@google.com#include "debug/TLBVerbose.hh" 6712837Sgabeblack@google.com#include "mem/page_table.hh" 6812837Sgabeblack@google.com#include "mem/request.hh" 6912955Sgabeblack@google.com#include "params/ArmTLB.hh" 7012837Sgabeblack@google.com#include "sim/full_system.hh" 7112837Sgabeblack@google.com#include "sim/process.hh" 7212837Sgabeblack@google.com 7312837Sgabeblack@google.comusing namespace std; 7412837Sgabeblack@google.comusing namespace ArmISA; 7512837Sgabeblack@google.com 7612837Sgabeblack@google.comTLB::TLB(const ArmTLBParams *p) 7712837Sgabeblack@google.com : BaseTLB(p), table(new TlbEntry[p->size]), size(p->size), 7812837Sgabeblack@google.com isStage2(p->is_stage2), stage2Req(false), stage2DescReq(false), _attr(0), 7912955Sgabeblack@google.com directToStage2(false), tableWalker(p->walker), stage2Tlb(NULL), 8012955Sgabeblack@google.com stage2Mmu(NULL), test(nullptr), rangeMRU(1), 8112955Sgabeblack@google.com aarch64(false), aarch64EL(EL0), isPriv(false), isSecure(false), 8213206Sgabeblack@google.com isHyp(false), asid(0), vmid(0), hcr(0), dacr(0), 8312955Sgabeblack@google.com miscRegValid(false), miscRegContext(0), curTranType(NormalTran) 8412955Sgabeblack@google.com{ 8512955Sgabeblack@google.com const ArmSystem *sys = dynamic_cast<const ArmSystem *>(p->sys); 8612955Sgabeblack@google.com 8712955Sgabeblack@google.com tableWalker->setTlb(this); 8812955Sgabeblack@google.com 8912955Sgabeblack@google.com // Cache system-level properties 9012955Sgabeblack@google.com haveLPAE = tableWalker->haveLPAE(); 9112955Sgabeblack@google.com haveVirtualization = tableWalker->haveVirtualization(); 9212837Sgabeblack@google.com haveLargeAsid64 = tableWalker->haveLargeAsid64(); 9312837Sgabeblack@google.com 9412837Sgabeblack@google.com if (sys) 9512837Sgabeblack@google.com m5opRange = sys->m5opRange(); 9612837Sgabeblack@google.com} 9712837Sgabeblack@google.com 9812837Sgabeblack@google.comTLB::~TLB() 9912837Sgabeblack@google.com{ 10012837Sgabeblack@google.com delete[] table; 10112837Sgabeblack@google.com} 10212837Sgabeblack@google.com 10312837Sgabeblack@google.comvoid 10412837Sgabeblack@google.comTLB::init() 10512837Sgabeblack@google.com{ 10612837Sgabeblack@google.com if (stage2Mmu && !isStage2) 10712837Sgabeblack@google.com stage2Tlb = stage2Mmu->stage2Tlb(); 10812837Sgabeblack@google.com} 10912837Sgabeblack@google.com 11012837Sgabeblack@google.comvoid 11112955Sgabeblack@google.comTLB::setMMU(Stage2MMU *m, MasterID master_id) 11212955Sgabeblack@google.com{ 11312955Sgabeblack@google.com stage2Mmu = m; 11413206Sgabeblack@google.com tableWalker->setMMU(m, master_id); 11512955Sgabeblack@google.com} 11612955Sgabeblack@google.com 11712955Sgabeblack@google.combool 11812955Sgabeblack@google.comTLB::translateFunctional(ThreadContext *tc, Addr va, Addr &pa) 11912955Sgabeblack@google.com{ 12012955Sgabeblack@google.com updateMiscReg(tc); 12112955Sgabeblack@google.com 12212955Sgabeblack@google.com if (directToStage2) { 12312955Sgabeblack@google.com assert(stage2Tlb); 12412837Sgabeblack@google.com return stage2Tlb->translateFunctional(tc, va, pa); 12512837Sgabeblack@google.com } 12612837Sgabeblack@google.com 12712837Sgabeblack@google.com TlbEntry *e = lookup(va, asid, vmid, isHyp, isSecure, true, false, 12812837Sgabeblack@google.com aarch64 ? aarch64EL : EL1); 12912955Sgabeblack@google.com if (!e) 13012837Sgabeblack@google.com return false; 13112955Sgabeblack@google.com pa = e->pAddr(va); 13212955Sgabeblack@google.com return true; 13312955Sgabeblack@google.com} 13412955Sgabeblack@google.com 13512955Sgabeblack@google.comFault 13612955Sgabeblack@google.comTLB::finalizePhysical(const RequestPtr &req, 13712955Sgabeblack@google.com ThreadContext *tc, Mode mode) const 13812955Sgabeblack@google.com{ 13912955Sgabeblack@google.com const Addr paddr = req->getPaddr(); 14012955Sgabeblack@google.com 14112955Sgabeblack@google.com if (m5opRange.contains(paddr)) { 14212955Sgabeblack@google.com req->setFlags(Request::MMAPPED_IPR | Request::GENERIC_IPR); 14312837Sgabeblack@google.com req->setPaddr(GenericISA::iprAddressPseudoInst( 14412837Sgabeblack@google.com (paddr >> 8) & 0xFF, 14512837Sgabeblack@google.com paddr & 0xFF)); 14612837Sgabeblack@google.com } 14712837Sgabeblack@google.com 14812837Sgabeblack@google.com return NoFault; 14912837Sgabeblack@google.com} 15012837Sgabeblack@google.com 15112955Sgabeblack@google.comTlbEntry* 15212837Sgabeblack@google.comTLB::lookup(Addr va, uint16_t asn, uint8_t vmid, bool hyp, bool secure, 15312955Sgabeblack@google.com bool functional, bool ignore_asn, uint8_t target_el) 15412955Sgabeblack@google.com{ 15512955Sgabeblack@google.com 15612955Sgabeblack@google.com TlbEntry *retval = NULL; 15712955Sgabeblack@google.com 15812955Sgabeblack@google.com // Maintaining LRU array 15912955Sgabeblack@google.com int x = 0; 16012955Sgabeblack@google.com while (retval == NULL && x < size) { 16112955Sgabeblack@google.com if ((!ignore_asn && table[x].match(va, asn, vmid, hyp, secure, false, 16212955Sgabeblack@google.com target_el)) || 16312955Sgabeblack@google.com (ignore_asn && table[x].match(va, vmid, hyp, secure, target_el))) { 16412955Sgabeblack@google.com // We only move the hit entry ahead when the position is higher 16512837Sgabeblack@google.com // than rangeMRU 16612837Sgabeblack@google.com if (x > rangeMRU && !functional) { 16712837Sgabeblack@google.com TlbEntry tmp_entry = table[x]; 16812837Sgabeblack@google.com for (int i = x; i > 0; i--) 16912837Sgabeblack@google.com table[i] = table[i - 1]; 17012837Sgabeblack@google.com table[0] = tmp_entry; 17112837Sgabeblack@google.com retval = &table[0]; 17212837Sgabeblack@google.com } else { 17312837Sgabeblack@google.com retval = &table[x]; 17412837Sgabeblack@google.com } 17512837Sgabeblack@google.com break; 17612837Sgabeblack@google.com } 17712837Sgabeblack@google.com ++x; 17812837Sgabeblack@google.com } 17912837Sgabeblack@google.com 18012837Sgabeblack@google.com DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d " 18112837Sgabeblack@google.com "ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d " 18212837Sgabeblack@google.com "el: %d\n", 18312837Sgabeblack@google.com va, asn, retval ? "hit" : "miss", vmid, hyp, secure, 18412837Sgabeblack@google.com retval ? retval->pfn : 0, retval ? retval->size : 0, 18512837Sgabeblack@google.com retval ? retval->pAddr(va) : 0, retval ? retval->ap : 0, 18612837Sgabeblack@google.com retval ? retval->ns : 0, retval ? retval->nstid : 0, 18712900Sgabeblack@google.com retval ? retval->global : 0, retval ? retval->asid : 0, 18812900Sgabeblack@google.com retval ? retval->el : 0); 18912900Sgabeblack@google.com 19012900Sgabeblack@google.com return retval; 19112915Sgabeblack@google.com} 19212915Sgabeblack@google.com 19312915Sgabeblack@google.com// insert a new TLB entry 19412915Sgabeblack@google.comvoid 19512837Sgabeblack@google.comTLB::insert(Addr addr, TlbEntry &entry) 19612837Sgabeblack@google.com{ 19712837Sgabeblack@google.com DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x" 19812837Sgabeblack@google.com " asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d" 19912837Sgabeblack@google.com " ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn, 20013303Sgabeblack@google.com entry.size, entry.vpn, entry.asid, entry.vmid, entry.N, 20113303Sgabeblack@google.com entry.global, entry.valid, entry.nonCacheable, entry.xn, 20213303Sgabeblack@google.com entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid, 20313303Sgabeblack@google.com entry.isHyp); 20412837Sgabeblack@google.com 20512837Sgabeblack@google.com if (table[size - 1].valid) 20612837Sgabeblack@google.com DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x " 20712837Sgabeblack@google.com "size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n", 20812955Sgabeblack@google.com table[size-1].vpn << table[size-1].N, table[size-1].asid, 20912955Sgabeblack@google.com table[size-1].vmid, table[size-1].pfn << table[size-1].N, 21012955Sgabeblack@google.com table[size-1].size, table[size-1].ap, table[size-1].ns, 21112837Sgabeblack@google.com table[size-1].nstid, table[size-1].global, table[size-1].isHyp, 21212837Sgabeblack@google.com table[size-1].el); 21313294Sgabeblack@google.com 21413294Sgabeblack@google.com //inserting to MRU position and evicting the LRU one 21513294Sgabeblack@google.com 21613294Sgabeblack@google.com for (int i = size - 1; i > 0; --i) 21713294Sgabeblack@google.com table[i] = table[i-1]; 21813294Sgabeblack@google.com table[0] = entry; 21913294Sgabeblack@google.com 22013294Sgabeblack@google.com inserts++; 22113294Sgabeblack@google.com ppRefills->notify(1); 22213294Sgabeblack@google.com} 22313294Sgabeblack@google.com 22413294Sgabeblack@google.comvoid 22513294Sgabeblack@google.comTLB::printTlb() const 22613294Sgabeblack@google.com{ 22713294Sgabeblack@google.com int x = 0; 22813294Sgabeblack@google.com TlbEntry *te; 22913294Sgabeblack@google.com DPRINTF(TLB, "Current TLB contents:\n"); 23013294Sgabeblack@google.com while (x < size) { 23113294Sgabeblack@google.com te = &table[x]; 23213294Sgabeblack@google.com if (te->valid) 23313294Sgabeblack@google.com DPRINTF(TLB, " * %s\n", te->print()); 23413294Sgabeblack@google.com ++x; 23513294Sgabeblack@google.com } 23613294Sgabeblack@google.com} 23713294Sgabeblack@google.com 23813294Sgabeblack@google.comvoid 23913294Sgabeblack@google.comTLB::flushAllSecurity(bool secure_lookup, uint8_t target_el, bool ignore_el) 24013294Sgabeblack@google.com{ 24113294Sgabeblack@google.com DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n", 24213294Sgabeblack@google.com (secure_lookup ? "secure" : "non-secure")); 24313303Sgabeblack@google.com int x = 0; 24413294Sgabeblack@google.com TlbEntry *te; 24513294Sgabeblack@google.com while (x < size) { 24613294Sgabeblack@google.com te = &table[x]; 24713294Sgabeblack@google.com if (te->valid && secure_lookup == !te->nstid && 24813294Sgabeblack@google.com (te->vmid == vmid || secure_lookup) && 24913294Sgabeblack@google.com checkELMatch(target_el, te->el, ignore_el)) { 25013294Sgabeblack@google.com 25113294Sgabeblack@google.com DPRINTF(TLB, " - %s\n", te->print()); 25213294Sgabeblack@google.com te->valid = false; 25313294Sgabeblack@google.com flushedEntries++; 25413294Sgabeblack@google.com } 25513294Sgabeblack@google.com ++x; 25613294Sgabeblack@google.com } 25713294Sgabeblack@google.com 25813294Sgabeblack@google.com flushTlb++; 25913294Sgabeblack@google.com 26013294Sgabeblack@google.com // If there's a second stage TLB (and we're not it) then flush it as well 26112837Sgabeblack@google.com // if we're currently in hyp mode 26212837Sgabeblack@google.com if (!isStage2 && isHyp) { 26312837Sgabeblack@google.com stage2Tlb->flushAllSecurity(secure_lookup, true); 26412837Sgabeblack@google.com } 26512837Sgabeblack@google.com} 26613303Sgabeblack@google.com 26713303Sgabeblack@google.comvoid 26813303Sgabeblack@google.comTLB::flushAllNs(uint8_t target_el, bool ignore_el) 26913303Sgabeblack@google.com{ 27013303Sgabeblack@google.com bool hyp = target_el == EL2; 27113303Sgabeblack@google.com 27213303Sgabeblack@google.com DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n", 27313303Sgabeblack@google.com (hyp ? "hyp" : "non-hyp")); 27413303Sgabeblack@google.com int x = 0; 27513303Sgabeblack@google.com TlbEntry *te; 27613303Sgabeblack@google.com while (x < size) { 27713303Sgabeblack@google.com te = &table[x]; 27812837Sgabeblack@google.com if (te->valid && te->nstid && te->isHyp == hyp && 279 checkELMatch(target_el, te->el, ignore_el)) { 280 281 DPRINTF(TLB, " - %s\n", te->print()); 282 flushedEntries++; 283 te->valid = false; 284 } 285 ++x; 286 } 287 288 flushTlb++; 289 290 // If there's a second stage TLB (and we're not it) then flush it as well 291 if (!isStage2 && !hyp) { 292 stage2Tlb->flushAllNs(false, true); 293 } 294} 295 296void 297TLB::flushMvaAsid(Addr mva, uint64_t asn, bool secure_lookup, uint8_t target_el) 298{ 299 DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x " 300 "(%s lookup)\n", mva, asn, (secure_lookup ? 301 "secure" : "non-secure")); 302 _flushMva(mva, asn, secure_lookup, false, target_el); 303 flushTlbMvaAsid++; 304} 305 306void 307TLB::flushAsid(uint64_t asn, bool secure_lookup, uint8_t target_el) 308{ 309 DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n", asn, 310 (secure_lookup ? "secure" : "non-secure")); 311 312 int x = 0 ; 313 TlbEntry *te; 314 315 while (x < size) { 316 te = &table[x]; 317 if (te->valid && te->asid == asn && secure_lookup == !te->nstid && 318 (te->vmid == vmid || secure_lookup) && 319 checkELMatch(target_el, te->el, false)) { 320 321 te->valid = false; 322 DPRINTF(TLB, " - %s\n", te->print()); 323 flushedEntries++; 324 } 325 ++x; 326 } 327 flushTlbAsid++; 328} 329 330void 331TLB::flushMva(Addr mva, bool secure_lookup, uint8_t target_el) 332{ 333 DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n", mva, 334 (secure_lookup ? "secure" : "non-secure")); 335 _flushMva(mva, 0xbeef, secure_lookup, true, target_el); 336 flushTlbMva++; 337} 338 339void 340TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup, 341 bool ignore_asn, uint8_t target_el) 342{ 343 TlbEntry *te; 344 // D5.7.2: Sign-extend address to 64 bits 345 mva = sext<56>(mva); 346 347 bool hyp = target_el == EL2; 348 349 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn, 350 target_el); 351 while (te != NULL) { 352 if (secure_lookup == !te->nstid) { 353 DPRINTF(TLB, " - %s\n", te->print()); 354 te->valid = false; 355 flushedEntries++; 356 } 357 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn, 358 target_el); 359 } 360} 361 362void 363TLB::flushIpaVmid(Addr ipa, bool secure_lookup, uint8_t target_el) 364{ 365 assert(!isStage2); 366 stage2Tlb->_flushMva(ipa, 0xbeef, secure_lookup, true, target_el); 367} 368 369bool 370TLB::checkELMatch(uint8_t target_el, uint8_t tentry_el, bool ignore_el) 371{ 372 bool elMatch = true; 373 if (!ignore_el) { 374 if (target_el == 2 || target_el == 3) { 375 elMatch = (tentry_el == target_el); 376 } else { 377 elMatch = (tentry_el == 0) || (tentry_el == 1); 378 } 379 } 380 return elMatch; 381} 382 383void 384TLB::drainResume() 385{ 386 // We might have unserialized something or switched CPUs, so make 387 // sure to re-read the misc regs. 388 miscRegValid = false; 389} 390 391void 392TLB::takeOverFrom(BaseTLB *_otlb) 393{ 394 TLB *otlb = dynamic_cast<TLB*>(_otlb); 395 /* Make sure we actually have a valid type */ 396 if (otlb) { 397 _attr = otlb->_attr; 398 haveLPAE = otlb->haveLPAE; 399 directToStage2 = otlb->directToStage2; 400 stage2Req = otlb->stage2Req; 401 stage2DescReq = otlb->stage2DescReq; 402 403 /* Sync the stage2 MMU if they exist in both 404 * the old CPU and the new 405 */ 406 if (!isStage2 && 407 stage2Tlb && otlb->stage2Tlb) { 408 stage2Tlb->takeOverFrom(otlb->stage2Tlb); 409 } 410 } else { 411 panic("Incompatible TLB type!"); 412 } 413} 414 415void 416TLB::serialize(CheckpointOut &cp) const 417{ 418 DPRINTF(Checkpoint, "Serializing Arm TLB\n"); 419 420 SERIALIZE_SCALAR(_attr); 421 SERIALIZE_SCALAR(haveLPAE); 422 SERIALIZE_SCALAR(directToStage2); 423 SERIALIZE_SCALAR(stage2Req); 424 SERIALIZE_SCALAR(stage2DescReq); 425 426 int num_entries = size; 427 SERIALIZE_SCALAR(num_entries); 428 for (int i = 0; i < size; i++) 429 table[i].serializeSection(cp, csprintf("TlbEntry%d", i)); 430} 431 432void 433TLB::unserialize(CheckpointIn &cp) 434{ 435 DPRINTF(Checkpoint, "Unserializing Arm TLB\n"); 436 437 UNSERIALIZE_SCALAR(_attr); 438 UNSERIALIZE_SCALAR(haveLPAE); 439 UNSERIALIZE_SCALAR(directToStage2); 440 UNSERIALIZE_SCALAR(stage2Req); 441 UNSERIALIZE_SCALAR(stage2DescReq); 442 443 int num_entries; 444 UNSERIALIZE_SCALAR(num_entries); 445 for (int i = 0; i < min(size, num_entries); i++) 446 table[i].unserializeSection(cp, csprintf("TlbEntry%d", i)); 447} 448 449void 450TLB::regStats() 451{ 452 BaseTLB::regStats(); 453 instHits 454 .name(name() + ".inst_hits") 455 .desc("ITB inst hits") 456 ; 457 458 instMisses 459 .name(name() + ".inst_misses") 460 .desc("ITB inst misses") 461 ; 462 463 instAccesses 464 .name(name() + ".inst_accesses") 465 .desc("ITB inst accesses") 466 ; 467 468 readHits 469 .name(name() + ".read_hits") 470 .desc("DTB read hits") 471 ; 472 473 readMisses 474 .name(name() + ".read_misses") 475 .desc("DTB read misses") 476 ; 477 478 readAccesses 479 .name(name() + ".read_accesses") 480 .desc("DTB read accesses") 481 ; 482 483 writeHits 484 .name(name() + ".write_hits") 485 .desc("DTB write hits") 486 ; 487 488 writeMisses 489 .name(name() + ".write_misses") 490 .desc("DTB write misses") 491 ; 492 493 writeAccesses 494 .name(name() + ".write_accesses") 495 .desc("DTB write accesses") 496 ; 497 498 hits 499 .name(name() + ".hits") 500 .desc("DTB hits") 501 ; 502 503 misses 504 .name(name() + ".misses") 505 .desc("DTB misses") 506 ; 507 508 accesses 509 .name(name() + ".accesses") 510 .desc("DTB accesses") 511 ; 512 513 flushTlb 514 .name(name() + ".flush_tlb") 515 .desc("Number of times complete TLB was flushed") 516 ; 517 518 flushTlbMva 519 .name(name() + ".flush_tlb_mva") 520 .desc("Number of times TLB was flushed by MVA") 521 ; 522 523 flushTlbMvaAsid 524 .name(name() + ".flush_tlb_mva_asid") 525 .desc("Number of times TLB was flushed by MVA & ASID") 526 ; 527 528 flushTlbAsid 529 .name(name() + ".flush_tlb_asid") 530 .desc("Number of times TLB was flushed by ASID") 531 ; 532 533 flushedEntries 534 .name(name() + ".flush_entries") 535 .desc("Number of entries that have been flushed from TLB") 536 ; 537 538 alignFaults 539 .name(name() + ".align_faults") 540 .desc("Number of TLB faults due to alignment restrictions") 541 ; 542 543 prefetchFaults 544 .name(name() + ".prefetch_faults") 545 .desc("Number of TLB faults due to prefetch") 546 ; 547 548 domainFaults 549 .name(name() + ".domain_faults") 550 .desc("Number of TLB faults due to domain restrictions") 551 ; 552 553 permsFaults 554 .name(name() + ".perms_faults") 555 .desc("Number of TLB faults due to permissions restrictions") 556 ; 557 558 instAccesses = instHits + instMisses; 559 readAccesses = readHits + readMisses; 560 writeAccesses = writeHits + writeMisses; 561 hits = readHits + writeHits + instHits; 562 misses = readMisses + writeMisses + instMisses; 563 accesses = readAccesses + writeAccesses + instAccesses; 564} 565 566void 567TLB::regProbePoints() 568{ 569 ppRefills.reset(new ProbePoints::PMU(getProbeManager(), "Refills")); 570} 571 572Fault 573TLB::translateSe(const RequestPtr &req, ThreadContext *tc, Mode mode, 574 Translation *translation, bool &delay, bool timing) 575{ 576 updateMiscReg(tc); 577 Addr vaddr_tainted = req->getVaddr(); 578 Addr vaddr = 0; 579 if (aarch64) 580 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr); 581 else 582 vaddr = vaddr_tainted; 583 Request::Flags flags = req->getFlags(); 584 585 bool is_fetch = (mode == Execute); 586 bool is_write = (mode == Write); 587 588 if (!is_fetch) { 589 assert(flags & MustBeOne); 590 if (sctlr.a || !(flags & AllowUnaligned)) { 591 if (vaddr & mask(flags & AlignmentMask)) { 592 // LPAE is always disabled in SE mode 593 return std::make_shared<DataAbort>( 594 vaddr_tainted, 595 TlbEntry::DomainType::NoAccess, is_write, 596 ArmFault::AlignmentFault, isStage2, 597 ArmFault::VmsaTran); 598 } 599 } 600 } 601 602 Addr paddr; 603 Process *p = tc->getProcessPtr(); 604 605 if (!p->pTable->translate(vaddr, paddr)) 606 return std::make_shared<GenericPageTableFault>(vaddr_tainted); 607 req->setPaddr(paddr); 608 609 return finalizePhysical(req, tc, mode); 610} 611 612Fault 613TLB::checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode) 614{ 615 // a data cache maintenance instruction that operates by MVA does 616 // not generate a Data Abort exeception due to a Permission fault 617 if (req->isCacheMaintenance()) { 618 return NoFault; 619 } 620 621 Addr vaddr = req->getVaddr(); // 32-bit don't have to purify 622 Request::Flags flags = req->getFlags(); 623 bool is_fetch = (mode == Execute); 624 bool is_write = (mode == Write); 625 bool is_priv = isPriv && !(flags & UserMode); 626 627 // Get the translation type from the actuall table entry 628 ArmFault::TranMethod tranMethod = te->longDescFormat ? ArmFault::LpaeTran 629 : ArmFault::VmsaTran; 630 631 // If this is the second stage of translation and the request is for a 632 // stage 1 page table walk then we need to check the HCR.PTW bit. This 633 // allows us to generate a fault if the request targets an area marked 634 // as a device or strongly ordered. 635 if (isStage2 && req->isPTWalk() && hcr.ptw && 636 (te->mtype != TlbEntry::MemoryType::Normal)) { 637 return std::make_shared<DataAbort>( 638 vaddr, te->domain, is_write, 639 ArmFault::PermissionLL + te->lookupLevel, 640 isStage2, tranMethod); 641 } 642 643 // Generate an alignment fault for unaligned data accesses to device or 644 // strongly ordered memory 645 if (!is_fetch) { 646 if (te->mtype != TlbEntry::MemoryType::Normal) { 647 if (vaddr & mask(flags & AlignmentMask)) { 648 alignFaults++; 649 return std::make_shared<DataAbort>( 650 vaddr, TlbEntry::DomainType::NoAccess, is_write, 651 ArmFault::AlignmentFault, isStage2, 652 tranMethod); 653 } 654 } 655 } 656 657 if (te->nonCacheable) { 658 // Prevent prefetching from I/O devices. 659 if (req->isPrefetch()) { 660 // Here we can safely use the fault status for the short 661 // desc. format in all cases 662 return std::make_shared<PrefetchAbort>( 663 vaddr, ArmFault::PrefetchUncacheable, 664 isStage2, tranMethod); 665 } 666 } 667 668 if (!te->longDescFormat) { 669 switch ((dacr >> (static_cast<uint8_t>(te->domain) * 2)) & 0x3) { 670 case 0: 671 domainFaults++; 672 DPRINTF(TLB, "TLB Fault: Data abort on domain. DACR: %#x" 673 " domain: %#x write:%d\n", dacr, 674 static_cast<uint8_t>(te->domain), is_write); 675 if (is_fetch) { 676 // Use PC value instead of vaddr because vaddr might 677 // be aligned to cache line and should not be the 678 // address reported in FAR 679 return std::make_shared<PrefetchAbort>( 680 req->getPC(), 681 ArmFault::DomainLL + te->lookupLevel, 682 isStage2, tranMethod); 683 } else 684 return std::make_shared<DataAbort>( 685 vaddr, te->domain, is_write, 686 ArmFault::DomainLL + te->lookupLevel, 687 isStage2, tranMethod); 688 case 1: 689 // Continue with permissions check 690 break; 691 case 2: 692 panic("UNPRED domain\n"); 693 case 3: 694 return NoFault; 695 } 696 } 697 698 // The 'ap' variable is AP[2:0] or {AP[2,1],1b'0}, i.e. always three bits 699 uint8_t ap = te->longDescFormat ? te->ap << 1 : te->ap; 700 uint8_t hap = te->hap; 701 702 if (sctlr.afe == 1 || te->longDescFormat) 703 ap |= 1; 704 705 bool abt; 706 bool isWritable = true; 707 // If this is a stage 2 access (eg for reading stage 1 page table entries) 708 // then don't perform the AP permissions check, we stil do the HAP check 709 // below. 710 if (isStage2) { 711 abt = false; 712 } else { 713 switch (ap) { 714 case 0: 715 DPRINTF(TLB, "Access permissions 0, checking rs:%#x\n", 716 (int)sctlr.rs); 717 if (!sctlr.xp) { 718 switch ((int)sctlr.rs) { 719 case 2: 720 abt = is_write; 721 break; 722 case 1: 723 abt = is_write || !is_priv; 724 break; 725 case 0: 726 case 3: 727 default: 728 abt = true; 729 break; 730 } 731 } else { 732 abt = true; 733 } 734 break; 735 case 1: 736 abt = !is_priv; 737 break; 738 case 2: 739 abt = !is_priv && is_write; 740 isWritable = is_priv; 741 break; 742 case 3: 743 abt = false; 744 break; 745 case 4: 746 panic("UNPRED premissions\n"); 747 case 5: 748 abt = !is_priv || is_write; 749 isWritable = false; 750 break; 751 case 6: 752 case 7: 753 abt = is_write; 754 isWritable = false; 755 break; 756 default: 757 panic("Unknown permissions %#x\n", ap); 758 } 759 } 760 761 bool hapAbt = is_write ? !(hap & 2) : !(hap & 1); 762 bool xn = te->xn || (isWritable && sctlr.wxn) || 763 (ap == 3 && sctlr.uwxn && is_priv); 764 if (is_fetch && (abt || xn || 765 (te->longDescFormat && te->pxn && is_priv) || 766 (isSecure && te->ns && scr.sif))) { 767 permsFaults++; 768 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. AP:%d " 769 "priv:%d write:%d ns:%d sif:%d sctlr.afe: %d \n", 770 ap, is_priv, is_write, te->ns, scr.sif,sctlr.afe); 771 // Use PC value instead of vaddr because vaddr might be aligned to 772 // cache line and should not be the address reported in FAR 773 return std::make_shared<PrefetchAbort>( 774 req->getPC(), 775 ArmFault::PermissionLL + te->lookupLevel, 776 isStage2, tranMethod); 777 } else if (abt | hapAbt) { 778 permsFaults++; 779 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d priv:%d" 780 " write:%d\n", ap, is_priv, is_write); 781 return std::make_shared<DataAbort>( 782 vaddr, te->domain, is_write, 783 ArmFault::PermissionLL + te->lookupLevel, 784 isStage2 | !abt, tranMethod); 785 } 786 return NoFault; 787} 788 789 790Fault 791TLB::checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode, 792 ThreadContext *tc) 793{ 794 assert(aarch64); 795 796 // A data cache maintenance instruction that operates by VA does 797 // not generate a Permission fault unless: 798 // * It is a data cache invalidate (dc ivac) which requires write 799 // permissions to the VA, or 800 // * It is executed from EL0 801 if (req->isCacheClean() && aarch64EL != EL0 && !isStage2) { 802 return NoFault; 803 } 804 805 Addr vaddr_tainted = req->getVaddr(); 806 Addr vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr); 807 808 Request::Flags flags = req->getFlags(); 809 bool is_fetch = (mode == Execute); 810 // Cache clean operations require read permissions to the specified VA 811 bool is_write = !req->isCacheClean() && mode == Write; 812 bool is_priv M5_VAR_USED = isPriv && !(flags & UserMode); 813 814 updateMiscReg(tc, curTranType); 815 816 // If this is the second stage of translation and the request is for a 817 // stage 1 page table walk then we need to check the HCR.PTW bit. This 818 // allows us to generate a fault if the request targets an area marked 819 // as a device or strongly ordered. 820 if (isStage2 && req->isPTWalk() && hcr.ptw && 821 (te->mtype != TlbEntry::MemoryType::Normal)) { 822 return std::make_shared<DataAbort>( 823 vaddr_tainted, te->domain, is_write, 824 ArmFault::PermissionLL + te->lookupLevel, 825 isStage2, ArmFault::LpaeTran); 826 } 827 828 // Generate an alignment fault for unaligned accesses to device or 829 // strongly ordered memory 830 if (!is_fetch) { 831 if (te->mtype != TlbEntry::MemoryType::Normal) { 832 if (vaddr & mask(flags & AlignmentMask)) { 833 alignFaults++; 834 return std::make_shared<DataAbort>( 835 vaddr_tainted, 836 TlbEntry::DomainType::NoAccess, is_write, 837 ArmFault::AlignmentFault, isStage2, 838 ArmFault::LpaeTran); 839 } 840 } 841 } 842 843 if (te->nonCacheable) { 844 // Prevent prefetching from I/O devices. 845 if (req->isPrefetch()) { 846 // Here we can safely use the fault status for the short 847 // desc. format in all cases 848 return std::make_shared<PrefetchAbort>( 849 vaddr_tainted, 850 ArmFault::PrefetchUncacheable, 851 isStage2, ArmFault::LpaeTran); 852 } 853 } 854 855 uint8_t ap = 0x3 & (te->ap); // 2-bit access protection field 856 bool grant = false; 857 858 uint8_t xn = te->xn; 859 uint8_t pxn = te->pxn; 860 bool r = !is_write && !is_fetch; 861 bool w = is_write; 862 bool x = is_fetch; 863 DPRINTF(TLBVerbose, "Checking permissions: ap:%d, xn:%d, pxn:%d, r:%d, " 864 "w:%d, x:%d\n", ap, xn, pxn, r, w, x); 865 866 if (isStage2) { 867 assert(ArmSystem::haveVirtualization(tc) && aarch64EL != EL2); 868 // In stage 2 we use the hypervisor access permission bits. 869 // The following permissions are described in ARM DDI 0487A.f 870 // D4-1802 871 uint8_t hap = 0x3 & te->hap; 872 if (is_fetch) { 873 // sctlr.wxn overrides the xn bit 874 grant = !sctlr.wxn && !xn; 875 } else if (is_write) { 876 grant = hap & 0x2; 877 } else { // is_read 878 grant = hap & 0x1; 879 } 880 } else { 881 switch (aarch64EL) { 882 case EL0: 883 { 884 uint8_t perm = (ap << 2) | (xn << 1) | pxn; 885 switch (perm) { 886 case 0: 887 case 1: 888 case 8: 889 case 9: 890 grant = x; 891 break; 892 case 4: 893 case 5: 894 grant = r || w || (x && !sctlr.wxn); 895 break; 896 case 6: 897 case 7: 898 grant = r || w; 899 break; 900 case 12: 901 case 13: 902 grant = r || x; 903 break; 904 case 14: 905 case 15: 906 grant = r; 907 break; 908 default: 909 grant = false; 910 } 911 } 912 break; 913 case EL1: 914 { 915 uint8_t perm = (ap << 2) | (xn << 1) | pxn; 916 switch (perm) { 917 case 0: 918 case 2: 919 grant = r || w || (x && !sctlr.wxn); 920 break; 921 case 1: 922 case 3: 923 case 4: 924 case 5: 925 case 6: 926 case 7: 927 // regions that are writeable at EL0 should not be 928 // executable at EL1 929 grant = r || w; 930 break; 931 case 8: 932 case 10: 933 case 12: 934 case 14: 935 grant = r || x; 936 break; 937 case 9: 938 case 11: 939 case 13: 940 case 15: 941 grant = r; 942 break; 943 default: 944 grant = false; 945 } 946 } 947 break; 948 case EL2: 949 case EL3: 950 { 951 uint8_t perm = (ap & 0x2) | xn; 952 switch (perm) { 953 case 0: 954 grant = r || w || (x && !sctlr.wxn) ; 955 break; 956 case 1: 957 grant = r || w; 958 break; 959 case 2: 960 grant = r || x; 961 break; 962 case 3: 963 grant = r; 964 break; 965 default: 966 grant = false; 967 } 968 } 969 break; 970 } 971 } 972 973 if (!grant) { 974 if (is_fetch) { 975 permsFaults++; 976 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. " 977 "AP:%d priv:%d write:%d ns:%d sif:%d " 978 "sctlr.afe: %d\n", 979 ap, is_priv, is_write, te->ns, scr.sif, sctlr.afe); 980 // Use PC value instead of vaddr because vaddr might be aligned to 981 // cache line and should not be the address reported in FAR 982 return std::make_shared<PrefetchAbort>( 983 req->getPC(), 984 ArmFault::PermissionLL + te->lookupLevel, 985 isStage2, ArmFault::LpaeTran); 986 } else { 987 permsFaults++; 988 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d " 989 "priv:%d write:%d\n", ap, is_priv, is_write); 990 return std::make_shared<DataAbort>( 991 vaddr_tainted, te->domain, is_write, 992 ArmFault::PermissionLL + te->lookupLevel, 993 isStage2, ArmFault::LpaeTran); 994 } 995 } 996 997 return NoFault; 998} 999 1000Fault 1001TLB::translateFs(const RequestPtr &req, ThreadContext *tc, Mode mode, 1002 Translation *translation, bool &delay, bool timing, 1003 TLB::ArmTranslationType tranType, bool functional) 1004{ 1005 // No such thing as a functional timing access 1006 assert(!(timing && functional)); 1007 1008 updateMiscReg(tc, tranType); 1009 1010 Addr vaddr_tainted = req->getVaddr(); 1011 Addr vaddr = 0; 1012 if (aarch64) 1013 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr); 1014 else 1015 vaddr = vaddr_tainted; 1016 Request::Flags flags = req->getFlags(); 1017 1018 bool is_fetch = (mode == Execute); 1019 bool is_write = (mode == Write); 1020 bool long_desc_format = aarch64 || longDescFormatInUse(tc); 1021 ArmFault::TranMethod tranMethod = long_desc_format ? ArmFault::LpaeTran 1022 : ArmFault::VmsaTran; 1023 1024 req->setAsid(asid); 1025 1026 DPRINTF(TLBVerbose, "CPSR is priv:%d UserMode:%d secure:%d S1S2NsTran:%d\n", 1027 isPriv, flags & UserMode, isSecure, tranType & S1S2NsTran); 1028 1029 DPRINTF(TLB, "translateFs addr %#x, mode %d, st2 %d, scr %#x sctlr %#x " 1030 "flags %#lx tranType 0x%x\n", vaddr_tainted, mode, isStage2, 1031 scr, sctlr, flags, tranType); 1032 1033 if ((req->isInstFetch() && (!sctlr.i)) || 1034 ((!req->isInstFetch()) && (!sctlr.c))){ 1035 if (!req->isCacheMaintenance()) { 1036 req->setFlags(Request::UNCACHEABLE); 1037 } 1038 req->setFlags(Request::STRICT_ORDER); 1039 } 1040 if (!is_fetch) { 1041 assert(flags & MustBeOne); 1042 if (sctlr.a || !(flags & AllowUnaligned)) { 1043 if (vaddr & mask(flags & AlignmentMask)) { 1044 alignFaults++; 1045 return std::make_shared<DataAbort>( 1046 vaddr_tainted, 1047 TlbEntry::DomainType::NoAccess, is_write, 1048 ArmFault::AlignmentFault, isStage2, 1049 tranMethod); 1050 } 1051 } 1052 } 1053 1054 // If guest MMU is off or hcr.vm=0 go straight to stage2 1055 if ((isStage2 && !hcr.vm) || (!isStage2 && !sctlr.m)) { 1056 1057 req->setPaddr(vaddr); 1058 // When the MMU is off the security attribute corresponds to the 1059 // security state of the processor 1060 if (isSecure) 1061 req->setFlags(Request::SECURE); 1062 1063 // @todo: double check this (ARM ARM issue C B3.2.1) 1064 if (long_desc_format || sctlr.tre == 0 || nmrr.ir0 == 0 || 1065 nmrr.or0 == 0 || prrr.tr0 != 0x2) { 1066 if (!req->isCacheMaintenance()) { 1067 req->setFlags(Request::UNCACHEABLE); 1068 } 1069 req->setFlags(Request::STRICT_ORDER); 1070 } 1071 1072 // Set memory attributes 1073 TlbEntry temp_te; 1074 temp_te.ns = !isSecure; 1075 if (isStage2 || hcr.dc == 0 || isSecure || 1076 (isHyp && !(tranType & S1CTran))) { 1077 1078 temp_te.mtype = is_fetch ? TlbEntry::MemoryType::Normal 1079 : TlbEntry::MemoryType::StronglyOrdered; 1080 temp_te.innerAttrs = 0x0; 1081 temp_te.outerAttrs = 0x0; 1082 temp_te.shareable = true; 1083 temp_te.outerShareable = true; 1084 } else { 1085 temp_te.mtype = TlbEntry::MemoryType::Normal; 1086 temp_te.innerAttrs = 0x3; 1087 temp_te.outerAttrs = 0x3; 1088 temp_te.shareable = false; 1089 temp_te.outerShareable = false; 1090 } 1091 temp_te.setAttributes(long_desc_format); 1092 DPRINTF(TLBVerbose, "(No MMU) setting memory attributes: shareable: " 1093 "%d, innerAttrs: %d, outerAttrs: %d, isStage2: %d\n", 1094 temp_te.shareable, temp_te.innerAttrs, temp_te.outerAttrs, 1095 isStage2); 1096 setAttr(temp_te.attributes); 1097 1098 return testTranslation(req, mode, TlbEntry::DomainType::NoAccess); 1099 } 1100 1101 DPRINTF(TLBVerbose, "Translating %s=%#x context=%d\n", 1102 isStage2 ? "IPA" : "VA", vaddr_tainted, asid); 1103 // Translation enabled 1104 1105 TlbEntry *te = NULL; 1106 TlbEntry mergeTe; 1107 Fault fault = getResultTe(&te, req, tc, mode, translation, timing, 1108 functional, &mergeTe); 1109 // only proceed if we have a valid table entry 1110 if ((te == NULL) && (fault == NoFault)) delay = true; 1111 1112 // If we have the table entry transfer some of the attributes to the 1113 // request that triggered the translation 1114 if (te != NULL) { 1115 // Set memory attributes 1116 DPRINTF(TLBVerbose, 1117 "Setting memory attributes: shareable: %d, innerAttrs: %d, " 1118 "outerAttrs: %d, mtype: %d, isStage2: %d\n", 1119 te->shareable, te->innerAttrs, te->outerAttrs, 1120 static_cast<uint8_t>(te->mtype), isStage2); 1121 setAttr(te->attributes); 1122 1123 if (te->nonCacheable && !req->isCacheMaintenance()) 1124 req->setFlags(Request::UNCACHEABLE); 1125 1126 // Require requests to be ordered if the request goes to 1127 // strongly ordered or device memory (i.e., anything other 1128 // than normal memory requires strict order). 1129 if (te->mtype != TlbEntry::MemoryType::Normal) 1130 req->setFlags(Request::STRICT_ORDER); 1131 1132 Addr pa = te->pAddr(vaddr); 1133 req->setPaddr(pa); 1134 1135 if (isSecure && !te->ns) { 1136 req->setFlags(Request::SECURE); 1137 } 1138 if ((!is_fetch) && (vaddr & mask(flags & AlignmentMask)) && 1139 (te->mtype != TlbEntry::MemoryType::Normal)) { 1140 // Unaligned accesses to Device memory should always cause an 1141 // abort regardless of sctlr.a 1142 alignFaults++; 1143 return std::make_shared<DataAbort>( 1144 vaddr_tainted, 1145 TlbEntry::DomainType::NoAccess, is_write, 1146 ArmFault::AlignmentFault, isStage2, 1147 tranMethod); 1148 } 1149 1150 // Check for a trickbox generated address fault 1151 if (fault == NoFault) 1152 fault = testTranslation(req, mode, te->domain); 1153 } 1154 1155 if (fault == NoFault) { 1156 // Don't try to finalize a physical address unless the 1157 // translation has completed (i.e., there is a table entry). 1158 return te ? finalizePhysical(req, tc, mode) : NoFault; 1159 } else { 1160 return fault; 1161 } 1162} 1163 1164Fault 1165TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode, 1166 TLB::ArmTranslationType tranType) 1167{ 1168 updateMiscReg(tc, tranType); 1169 1170 if (directToStage2) { 1171 assert(stage2Tlb); 1172 return stage2Tlb->translateAtomic(req, tc, mode, tranType); 1173 } 1174 1175 bool delay = false; 1176 Fault fault; 1177 if (FullSystem) 1178 fault = translateFs(req, tc, mode, NULL, delay, false, tranType); 1179 else 1180 fault = translateSe(req, tc, mode, NULL, delay, false); 1181 assert(!delay); 1182 return fault; 1183} 1184 1185Fault 1186TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode, 1187 TLB::ArmTranslationType tranType) 1188{ 1189 updateMiscReg(tc, tranType); 1190 1191 if (directToStage2) { 1192 assert(stage2Tlb); 1193 return stage2Tlb->translateFunctional(req, tc, mode, tranType); 1194 } 1195 1196 bool delay = false; 1197 Fault fault; 1198 if (FullSystem) 1199 fault = translateFs(req, tc, mode, NULL, delay, false, tranType, true); 1200 else 1201 fault = translateSe(req, tc, mode, NULL, delay, false); 1202 assert(!delay); 1203 return fault; 1204} 1205 1206void 1207TLB::translateTiming(const RequestPtr &req, ThreadContext *tc, 1208 Translation *translation, Mode mode, TLB::ArmTranslationType tranType) 1209{ 1210 updateMiscReg(tc, tranType); 1211 1212 if (directToStage2) { 1213 assert(stage2Tlb); 1214 stage2Tlb->translateTiming(req, tc, translation, mode, tranType); 1215 return; 1216 } 1217 1218 assert(translation); 1219 1220 translateComplete(req, tc, translation, mode, tranType, isStage2); 1221} 1222 1223Fault 1224TLB::translateComplete(const RequestPtr &req, ThreadContext *tc, 1225 Translation *translation, Mode mode, TLB::ArmTranslationType tranType, 1226 bool callFromS2) 1227{ 1228 bool delay = false; 1229 Fault fault; 1230 if (FullSystem) 1231 fault = translateFs(req, tc, mode, translation, delay, true, tranType); 1232 else 1233 fault = translateSe(req, tc, mode, translation, delay, true); 1234 DPRINTF(TLBVerbose, "Translation returning delay=%d fault=%d\n", delay, fault != 1235 NoFault); 1236 // If we have a translation, and we're not in the middle of doing a stage 1237 // 2 translation tell the translation that we've either finished or its 1238 // going to take a while. By not doing this when we're in the middle of a 1239 // stage 2 translation we prevent marking the translation as delayed twice, 1240 // one when the translation starts and again when the stage 1 translation 1241 // completes. 1242 if (translation && (callFromS2 || !stage2Req || req->hasPaddr() || fault != NoFault)) { 1243 if (!delay) 1244 translation->finish(fault, req, tc, mode); 1245 else 1246 translation->markDelayed(); 1247 } 1248 return fault; 1249} 1250 1251Port * 1252TLB::getTableWalkerPort() 1253{ 1254 return &stage2Mmu->getDMAPort(); 1255} 1256 1257void 1258TLB::updateMiscReg(ThreadContext *tc, ArmTranslationType tranType) 1259{ 1260 // check if the regs have changed, or the translation mode is different. 1261 // NOTE: the tran type doesn't affect stage 2 TLB's as they only handle 1262 // one type of translation anyway 1263 if (miscRegValid && miscRegContext == tc->contextId() && 1264 ((tranType == curTranType) || isStage2)) { 1265 return; 1266 } 1267 1268 DPRINTF(TLBVerbose, "TLB variables changed!\n"); 1269 cpsr = tc->readMiscReg(MISCREG_CPSR); 1270 1271 // Dependencies: SCR/SCR_EL3, CPSR 1272 isSecure = inSecureState(tc) && 1273 !(tranType & HypMode) && !(tranType & S1S2NsTran); 1274 1275 aarch64EL = tranTypeEL(cpsr, tranType); 1276 aarch64 = isStage2 ? 1277 ELIs64(tc, EL2) : 1278 ELIs64(tc, aarch64EL == EL0 ? EL1 : aarch64EL); 1279 1280 if (aarch64) { // AArch64 1281 // determine EL we need to translate in 1282 switch (aarch64EL) { 1283 case EL0: 1284 case EL1: 1285 { 1286 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1); 1287 ttbcr = tc->readMiscReg(MISCREG_TCR_EL1); 1288 uint64_t ttbr_asid = ttbcr.a1 ? 1289 tc->readMiscReg(MISCREG_TTBR1_EL1) : 1290 tc->readMiscReg(MISCREG_TTBR0_EL1); 1291 asid = bits(ttbr_asid, 1292 (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48); 1293 } 1294 break; 1295 case EL2: 1296 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2); 1297 ttbcr = tc->readMiscReg(MISCREG_TCR_EL2); 1298 asid = -1; 1299 break; 1300 case EL3: 1301 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL3); 1302 ttbcr = tc->readMiscReg(MISCREG_TCR_EL3); 1303 asid = -1; 1304 break; 1305 } 1306 hcr = tc->readMiscReg(MISCREG_HCR_EL2); 1307 scr = tc->readMiscReg(MISCREG_SCR_EL3); 1308 isPriv = aarch64EL != EL0; 1309 if (haveVirtualization) { 1310 vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48); 1311 isHyp = aarch64EL == EL2; 1312 isHyp |= tranType & HypMode; 1313 isHyp &= (tranType & S1S2NsTran) == 0; 1314 isHyp &= (tranType & S1CTran) == 0; 1315 // Work out if we should skip the first stage of translation and go 1316 // directly to stage 2. This value is cached so we don't have to 1317 // compute it for every translation. 1318 stage2Req = isStage2 || 1319 (hcr.vm && !isHyp && !isSecure && 1320 !(tranType & S1CTran) && (aarch64EL < EL2) && 1321 !(tranType & S1E1Tran)); // <--- FIX THIS HACK 1322 stage2DescReq = isStage2 || (hcr.vm && !isHyp && !isSecure && 1323 (aarch64EL < EL2)); 1324 directToStage2 = !isStage2 && stage2Req && !sctlr.m; 1325 } else { 1326 vmid = 0; 1327 isHyp = false; 1328 directToStage2 = false; 1329 stage2Req = false; 1330 stage2DescReq = false; 1331 } 1332 } else { // AArch32 1333 sctlr = tc->readMiscReg(snsBankedIndex(MISCREG_SCTLR, tc, 1334 !isSecure)); 1335 ttbcr = tc->readMiscReg(snsBankedIndex(MISCREG_TTBCR, tc, 1336 !isSecure)); 1337 scr = tc->readMiscReg(MISCREG_SCR); 1338 isPriv = cpsr.mode != MODE_USER; 1339 if (longDescFormatInUse(tc)) { 1340 uint64_t ttbr_asid = tc->readMiscReg( 1341 snsBankedIndex(ttbcr.a1 ? MISCREG_TTBR1 : 1342 MISCREG_TTBR0, 1343 tc, !isSecure)); 1344 asid = bits(ttbr_asid, 55, 48); 1345 } else { // Short-descriptor translation table format in use 1346 CONTEXTIDR context_id = tc->readMiscReg(snsBankedIndex( 1347 MISCREG_CONTEXTIDR, tc,!isSecure)); 1348 asid = context_id.asid; 1349 } 1350 prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, tc, 1351 !isSecure)); 1352 nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, tc, 1353 !isSecure)); 1354 dacr = tc->readMiscReg(snsBankedIndex(MISCREG_DACR, tc, 1355 !isSecure)); 1356 hcr = tc->readMiscReg(MISCREG_HCR); 1357 1358 if (haveVirtualization) { 1359 vmid = bits(tc->readMiscReg(MISCREG_VTTBR), 55, 48); 1360 isHyp = cpsr.mode == MODE_HYP; 1361 isHyp |= tranType & HypMode; 1362 isHyp &= (tranType & S1S2NsTran) == 0; 1363 isHyp &= (tranType & S1CTran) == 0; 1364 if (isHyp) { 1365 sctlr = tc->readMiscReg(MISCREG_HSCTLR); 1366 } 1367 // Work out if we should skip the first stage of translation and go 1368 // directly to stage 2. This value is cached so we don't have to 1369 // compute it for every translation. 1370 stage2Req = hcr.vm && !isStage2 && !isHyp && !isSecure && 1371 !(tranType & S1CTran); 1372 stage2DescReq = hcr.vm && !isStage2 && !isHyp && !isSecure; 1373 directToStage2 = stage2Req && !sctlr.m; 1374 } else { 1375 vmid = 0; 1376 stage2Req = false; 1377 isHyp = false; 1378 directToStage2 = false; 1379 stage2DescReq = false; 1380 } 1381 } 1382 miscRegValid = true; 1383 miscRegContext = tc->contextId(); 1384 curTranType = tranType; 1385} 1386 1387ExceptionLevel 1388TLB::tranTypeEL(CPSR cpsr, ArmTranslationType type) 1389{ 1390 switch (type) { 1391 case S1E0Tran: 1392 case S12E0Tran: 1393 return EL0; 1394 1395 case S1E1Tran: 1396 case S12E1Tran: 1397 return EL1; 1398 1399 case S1E2Tran: 1400 return EL2; 1401 1402 case S1E3Tran: 1403 return EL3; 1404 1405 case NormalTran: 1406 case S1CTran: 1407 case S1S2NsTran: 1408 case HypMode: 1409 return opModeToEL((OperatingMode)(uint8_t)cpsr.mode); 1410 1411 default: 1412 panic("Unknown translation mode!\n"); 1413 } 1414} 1415 1416Fault 1417TLB::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, Mode mode, 1418 Translation *translation, bool timing, bool functional, 1419 bool is_secure, TLB::ArmTranslationType tranType) 1420{ 1421 bool is_fetch = (mode == Execute); 1422 bool is_write = (mode == Write); 1423 1424 Addr vaddr_tainted = req->getVaddr(); 1425 Addr vaddr = 0; 1426 ExceptionLevel target_el = aarch64 ? aarch64EL : EL1; 1427 if (aarch64) { 1428 vaddr = purifyTaggedAddr(vaddr_tainted, tc, target_el, ttbcr); 1429 } else { 1430 vaddr = vaddr_tainted; 1431 } 1432 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el); 1433 if (*te == NULL) { 1434 if (req->isPrefetch()) { 1435 // if the request is a prefetch don't attempt to fill the TLB or go 1436 // any further with the memory access (here we can safely use the 1437 // fault status for the short desc. format in all cases) 1438 prefetchFaults++; 1439 return std::make_shared<PrefetchAbort>( 1440 vaddr_tainted, ArmFault::PrefetchTLBMiss, isStage2); 1441 } 1442 1443 if (is_fetch) 1444 instMisses++; 1445 else if (is_write) 1446 writeMisses++; 1447 else 1448 readMisses++; 1449 1450 // start translation table walk, pass variables rather than 1451 // re-retreaving in table walker for speed 1452 DPRINTF(TLB, "TLB Miss: Starting hardware table walker for %#x(%d:%d)\n", 1453 vaddr_tainted, asid, vmid); 1454 Fault fault; 1455 fault = tableWalker->walk(req, tc, asid, vmid, isHyp, mode, 1456 translation, timing, functional, is_secure, 1457 tranType, stage2DescReq); 1458 // for timing mode, return and wait for table walk, 1459 if (timing || fault != NoFault) { 1460 return fault; 1461 } 1462 1463 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el); 1464 if (!*te) 1465 printTlb(); 1466 assert(*te); 1467 } else { 1468 if (is_fetch) 1469 instHits++; 1470 else if (is_write) 1471 writeHits++; 1472 else 1473 readHits++; 1474 } 1475 return NoFault; 1476} 1477 1478Fault 1479TLB::getResultTe(TlbEntry **te, const RequestPtr &req, 1480 ThreadContext *tc, Mode mode, 1481 Translation *translation, bool timing, bool functional, 1482 TlbEntry *mergeTe) 1483{ 1484 Fault fault; 1485 1486 if (isStage2) { 1487 // We are already in the stage 2 TLB. Grab the table entry for stage 1488 // 2 only. We are here because stage 1 translation is disabled. 1489 TlbEntry *s2Te = NULL; 1490 // Get the stage 2 table entry 1491 fault = getTE(&s2Te, req, tc, mode, translation, timing, functional, 1492 isSecure, curTranType); 1493 // Check permissions of stage 2 1494 if ((s2Te != NULL) && (fault == NoFault)) { 1495 if (aarch64) 1496 fault = checkPermissions64(s2Te, req, mode, tc); 1497 else 1498 fault = checkPermissions(s2Te, req, mode); 1499 } 1500 *te = s2Te; 1501 return fault; 1502 } 1503 1504 TlbEntry *s1Te = NULL; 1505 1506 Addr vaddr_tainted = req->getVaddr(); 1507 1508 // Get the stage 1 table entry 1509 fault = getTE(&s1Te, req, tc, mode, translation, timing, functional, 1510 isSecure, curTranType); 1511 // only proceed if we have a valid table entry 1512 if ((s1Te != NULL) && (fault == NoFault)) { 1513 // Check stage 1 permissions before checking stage 2 1514 if (aarch64) 1515 fault = checkPermissions64(s1Te, req, mode, tc); 1516 else 1517 fault = checkPermissions(s1Te, req, mode); 1518 if (stage2Req & (fault == NoFault)) { 1519 Stage2LookUp *s2Lookup = new Stage2LookUp(this, stage2Tlb, *s1Te, 1520 req, translation, mode, timing, functional, curTranType); 1521 fault = s2Lookup->getTe(tc, mergeTe); 1522 if (s2Lookup->isComplete()) { 1523 *te = mergeTe; 1524 // We've finished with the lookup so delete it 1525 delete s2Lookup; 1526 } else { 1527 // The lookup hasn't completed, so we can't delete it now. We 1528 // get round this by asking the object to self delete when the 1529 // translation is complete. 1530 s2Lookup->setSelfDelete(); 1531 } 1532 } else { 1533 // This case deals with an S1 hit (or bypass), followed by 1534 // an S2 hit-but-perms issue 1535 if (isStage2) { 1536 DPRINTF(TLBVerbose, "s2TLB: reqVa %#x, reqPa %#x, fault %p\n", 1537 vaddr_tainted, req->hasPaddr() ? req->getPaddr() : ~0, fault); 1538 if (fault != NoFault) { 1539 ArmFault *armFault = reinterpret_cast<ArmFault *>(fault.get()); 1540 armFault->annotate(ArmFault::S1PTW, false); 1541 armFault->annotate(ArmFault::OVA, vaddr_tainted); 1542 } 1543 } 1544 *te = s1Te; 1545 } 1546 } 1547 return fault; 1548} 1549 1550void 1551TLB::setTestInterface(SimObject *_ti) 1552{ 1553 if (!_ti) { 1554 test = nullptr; 1555 } else { 1556 TlbTestInterface *ti(dynamic_cast<TlbTestInterface *>(_ti)); 1557 fatal_if(!ti, "%s is not a valid ARM TLB tester\n", _ti->name()); 1558 test = ti; 1559 } 1560} 1561 1562Fault 1563TLB::testTranslation(const RequestPtr &req, Mode mode, 1564 TlbEntry::DomainType domain) 1565{ 1566 if (!test || !req->hasSize() || req->getSize() == 0 || 1567 req->isCacheMaintenance()) { 1568 return NoFault; 1569 } else { 1570 return test->translationCheck(req, isPriv, mode, domain); 1571 } 1572} 1573 1574Fault 1575TLB::testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode, 1576 TlbEntry::DomainType domain, LookupLevel lookup_level) 1577{ 1578 if (!test) { 1579 return NoFault; 1580 } else { 1581 return test->walkCheck(pa, size, va, is_secure, isPriv, mode, 1582 domain, lookup_level); 1583 } 1584} 1585 1586 1587ArmISA::TLB * 1588ArmTLBParams::create() 1589{ 1590 return new ArmISA::TLB(this); 1591} 1592