table_walker.cc revision 11574
1/* 2 * Copyright (c) 2010, 2012-2016 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Ali Saidi 38 * Giacomo Gabrielli 39 */ 40#include "arch/arm/table_walker.hh" 41 42#include <memory> 43 44#include "arch/arm/faults.hh" 45#include "arch/arm/stage2_mmu.hh" 46#include "arch/arm/system.hh" 47#include "arch/arm/tlb.hh" 48#include "cpu/base.hh" 49#include "cpu/thread_context.hh" 50#include "debug/Checkpoint.hh" 51#include "debug/Drain.hh" 52#include "debug/TLB.hh" 53#include "debug/TLBVerbose.hh" 54#include "dev/dma_device.hh" 55#include "sim/system.hh" 56 57using namespace ArmISA; 58 59TableWalker::TableWalker(const Params *p) 60 : MemObject(p), 61 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId), 62 isStage2(p->is_stage2), tlb(NULL), 63 currState(NULL), pending(false), 64 numSquashable(p->num_squash_per_cycle), 65 pendingReqs(0), 66 pendingChangeTick(curTick()), 67 doL1DescEvent(this), doL2DescEvent(this), 68 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this), 69 doL3LongDescEvent(this), 70 doProcessEvent(this) 71{ 72 sctlr = 0; 73 74 // Cache system-level properties 75 if (FullSystem) { 76 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys); 77 assert(armSys); 78 haveSecurity = armSys->haveSecurity(); 79 _haveLPAE = armSys->haveLPAE(); 80 _haveVirtualization = armSys->haveVirtualization(); 81 physAddrRange = armSys->physAddrRange(); 82 _haveLargeAsid64 = armSys->haveLargeAsid64(); 83 } else { 84 haveSecurity = _haveLPAE = _haveVirtualization = false; 85 _haveLargeAsid64 = false; 86 physAddrRange = 32; 87 } 88 89} 90 91TableWalker::~TableWalker() 92{ 93 ; 94} 95 96void 97TableWalker::setMMU(Stage2MMU *m, MasterID master_id) 98{ 99 stage2Mmu = m; 100 port = &m->getPort(); 101 masterId = master_id; 102} 103 104void 105TableWalker::init() 106{ 107 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n"); 108 fatal_if(!port, "Table walker must have a valid port\n"); 109 fatal_if(!tlb, "Table walker must have a valid TLB\n"); 110} 111 112BaseMasterPort& 113TableWalker::getMasterPort(const std::string &if_name, PortID idx) 114{ 115 if (if_name == "port") { 116 if (!isStage2) { 117 return *port; 118 } else { 119 fatal("Cannot access table walker port through stage-two walker\n"); 120 } 121 } 122 return MemObject::getMasterPort(if_name, idx); 123} 124 125TableWalker::WalkerState::WalkerState() : 126 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr), 127 asid(0), vmid(0), isHyp(false), transState(nullptr), 128 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false), 129 secureLookup(false), rwTable(false), userTable(false), xnTable(false), 130 pxnTable(false), stage2Req(false), doingStage2(false), 131 stage2Tran(nullptr), timing(false), functional(false), 132 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), 133 delayed(false), tableWalker(nullptr) 134{ 135} 136 137void 138TableWalker::completeDrain() 139{ 140 if (drainState() == DrainState::Draining && 141 stateQueues[L1].empty() && stateQueues[L2].empty() && 142 pendingQueue.empty()) { 143 144 DPRINTF(Drain, "TableWalker done draining, processing drain event\n"); 145 signalDrainDone(); 146 } 147} 148 149DrainState 150TableWalker::drain() 151{ 152 bool state_queues_not_empty = false; 153 154 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) { 155 if (!stateQueues[i].empty()) { 156 state_queues_not_empty = true; 157 break; 158 } 159 } 160 161 if (state_queues_not_empty || pendingQueue.size()) { 162 DPRINTF(Drain, "TableWalker not drained\n"); 163 return DrainState::Draining; 164 } else { 165 DPRINTF(Drain, "TableWalker free, no need to drain\n"); 166 return DrainState::Drained; 167 } 168} 169 170void 171TableWalker::drainResume() 172{ 173 if (params()->sys->isTimingMode() && currState) { 174 delete currState; 175 currState = NULL; 176 pendingChange(); 177 } 178} 179 180Fault 181TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid, 182 uint8_t _vmid, bool _isHyp, TLB::Mode _mode, 183 TLB::Translation *_trans, bool _timing, bool _functional, 184 bool secure, TLB::ArmTranslationType tranType) 185{ 186 assert(!(_functional && _timing)); 187 ++statWalks; 188 189 WalkerState *savedCurrState = NULL; 190 191 if (!currState && !_functional) { 192 // For atomic mode, a new WalkerState instance should be only created 193 // once per TLB. For timing mode, a new instance is generated for every 194 // TLB miss. 195 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n"); 196 197 currState = new WalkerState(); 198 currState->tableWalker = this; 199 } else if (_functional) { 200 // If we are mixing functional mode with timing (or even 201 // atomic), we need to to be careful and clean up after 202 // ourselves to not risk getting into an inconsistent state. 203 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n"); 204 savedCurrState = currState; 205 currState = new WalkerState(); 206 currState->tableWalker = this; 207 } else if (_timing) { 208 // This is a translation that was completed and then faulted again 209 // because some underlying parameters that affect the translation 210 // changed out from under us (e.g. asid). It will either be a 211 // misprediction, in which case nothing will happen or we'll use 212 // this fault to re-execute the faulting instruction which should clean 213 // up everything. 214 if (currState->vaddr_tainted == _req->getVaddr()) { 215 ++statSquashedBefore; 216 return std::make_shared<ReExec>(); 217 } 218 } 219 pendingChange(); 220 221 currState->startTime = curTick(); 222 currState->tc = _tc; 223 // ARM DDI 0487A.f (ARMv8 ARM) pg J8-5672 224 // aarch32/translation/translation/AArch32.TranslateAddress dictates 225 // even AArch32 EL0 will use AArch64 translation if EL1 is in AArch64. 226 currState->aarch64 = isStage2 || opModeIs64(currOpMode(_tc)) || 227 ((currEL(_tc) == EL0) && ELIs64(_tc, EL1)); 228 currState->el = currEL(_tc); 229 currState->transState = _trans; 230 currState->req = _req; 231 currState->fault = NoFault; 232 currState->asid = _asid; 233 currState->vmid = _vmid; 234 currState->isHyp = _isHyp; 235 currState->timing = _timing; 236 currState->functional = _functional; 237 currState->mode = _mode; 238 currState->tranType = tranType; 239 currState->isSecure = secure; 240 currState->physAddrRange = physAddrRange; 241 242 /** @todo These should be cached or grabbed from cached copies in 243 the TLB, all these miscreg reads are expensive */ 244 currState->vaddr_tainted = currState->req->getVaddr(); 245 if (currState->aarch64) 246 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted, 247 currState->tc, currState->el); 248 else 249 currState->vaddr = currState->vaddr_tainted; 250 251 if (currState->aarch64) { 252 switch (currState->el) { 253 case EL0: 254 case EL1: 255 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); 256 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1); 257 break; 258 case EL2: 259 assert(_haveVirtualization); 260 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2); 261 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2); 262 break; 263 case EL3: 264 assert(haveSecurity); 265 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3); 266 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3); 267 break; 268 default: 269 panic("Invalid exception level"); 270 break; 271 } 272 } else { 273 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 274 MISCREG_SCTLR, currState->tc, !currState->isSecure)); 275 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 276 MISCREG_TTBCR, currState->tc, !currState->isSecure)); 277 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR); 278 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR); 279 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR); 280 } 281 sctlr = currState->sctlr; 282 283 currState->isFetch = (currState->mode == TLB::Execute); 284 currState->isWrite = (currState->mode == TLB::Write); 285 286 statRequestOrigin[REQUESTED][currState->isFetch]++; 287 288 // We only do a second stage of translation if we're not secure, or in 289 // hyp mode, the second stage MMU is enabled, and this table walker 290 // instance is the first stage. 291 currState->doingStage2 = false; 292 // @todo: for now disable this in AArch64 (HCR is not set) 293 currState->stage2Req = !currState->aarch64 && currState->hcr.vm && 294 !isStage2 && !currState->isSecure && !currState->isHyp; 295 296 bool long_desc_format = currState->aarch64 || _isHyp || isStage2 || 297 longDescFormatInUse(currState->tc); 298 299 if (long_desc_format) { 300 // Helper variables used for hierarchical permissions 301 currState->secureLookup = currState->isSecure; 302 currState->rwTable = true; 303 currState->userTable = true; 304 currState->xnTable = false; 305 currState->pxnTable = false; 306 307 ++statWalksLongDescriptor; 308 } else { 309 ++statWalksShortDescriptor; 310 } 311 312 if (!currState->timing) { 313 Fault fault = NoFault; 314 if (currState->aarch64) 315 fault = processWalkAArch64(); 316 else if (long_desc_format) 317 fault = processWalkLPAE(); 318 else 319 fault = processWalk(); 320 321 // If this was a functional non-timing access restore state to 322 // how we found it. 323 if (currState->functional) { 324 delete currState; 325 currState = savedCurrState; 326 } 327 return fault; 328 } 329 330 if (pending || pendingQueue.size()) { 331 pendingQueue.push_back(currState); 332 currState = NULL; 333 pendingChange(); 334 } else { 335 pending = true; 336 pendingChange(); 337 if (currState->aarch64) 338 return processWalkAArch64(); 339 else if (long_desc_format) 340 return processWalkLPAE(); 341 else 342 return processWalk(); 343 } 344 345 return NoFault; 346} 347 348void 349TableWalker::processWalkWrapper() 350{ 351 assert(!currState); 352 assert(pendingQueue.size()); 353 pendingChange(); 354 currState = pendingQueue.front(); 355 356 ExceptionLevel target_el = EL0; 357 if (currState->aarch64) 358 target_el = currEL(currState->tc); 359 else 360 target_el = EL1; 361 362 // Check if a previous walk filled this request already 363 // @TODO Should this always be the TLB or should we look in the stage2 TLB? 364 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid, 365 currState->vmid, currState->isHyp, currState->isSecure, true, false, 366 target_el); 367 368 // Check if we still need to have a walk for this request. If the requesting 369 // instruction has been squashed, or a previous walk has filled the TLB with 370 // a match, we just want to get rid of the walk. The latter could happen 371 // when there are multiple outstanding misses to a single page and a 372 // previous request has been successfully translated. 373 if (!currState->transState->squashed() && !te) { 374 // We've got a valid request, lets process it 375 pending = true; 376 pendingQueue.pop_front(); 377 // Keep currState in case one of the processWalk... calls NULLs it 378 WalkerState *curr_state_copy = currState; 379 Fault f; 380 if (currState->aarch64) 381 f = processWalkAArch64(); 382 else if (longDescFormatInUse(currState->tc) || 383 currState->isHyp || isStage2) 384 f = processWalkLPAE(); 385 else 386 f = processWalk(); 387 388 if (f != NoFault) { 389 curr_state_copy->transState->finish(f, curr_state_copy->req, 390 curr_state_copy->tc, curr_state_copy->mode); 391 392 delete curr_state_copy; 393 } 394 return; 395 } 396 397 398 // If the instruction that we were translating for has been 399 // squashed we shouldn't bother. 400 unsigned num_squashed = 0; 401 ThreadContext *tc = currState->tc; 402 while ((num_squashed < numSquashable) && currState && 403 (currState->transState->squashed() || te)) { 404 pendingQueue.pop_front(); 405 num_squashed++; 406 statSquashedBefore++; 407 408 DPRINTF(TLB, "Squashing table walk for address %#x\n", 409 currState->vaddr_tainted); 410 411 if (currState->transState->squashed()) { 412 // finish the translation which will delete the translation object 413 currState->transState->finish( 414 std::make_shared<UnimpFault>("Squashed Inst"), 415 currState->req, currState->tc, currState->mode); 416 } else { 417 // translate the request now that we know it will work 418 statWalkServiceTime.sample(curTick() - currState->startTime); 419 tlb->translateTiming(currState->req, currState->tc, 420 currState->transState, currState->mode); 421 422 } 423 424 // delete the current request 425 delete currState; 426 427 // peak at the next one 428 if (pendingQueue.size()) { 429 currState = pendingQueue.front(); 430 te = tlb->lookup(currState->vaddr, currState->asid, 431 currState->vmid, currState->isHyp, currState->isSecure, true, 432 false, target_el); 433 } else { 434 // Terminate the loop, nothing more to do 435 currState = NULL; 436 } 437 } 438 pendingChange(); 439 440 // if we still have pending translations, schedule more work 441 nextWalk(tc); 442 currState = NULL; 443} 444 445Fault 446TableWalker::processWalk() 447{ 448 Addr ttbr = 0; 449 450 // If translation isn't enabled, we shouldn't be here 451 assert(currState->sctlr.m || isStage2); 452 453 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n", 454 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31, 455 32 - currState->ttbcr.n)); 456 457 statWalkWaitTime.sample(curTick() - currState->startTime); 458 459 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31, 460 32 - currState->ttbcr.n)) { 461 DPRINTF(TLB, " - Selecting TTBR0\n"); 462 // Check if table walk is allowed when Security Extensions are enabled 463 if (haveSecurity && currState->ttbcr.pd0) { 464 if (currState->isFetch) 465 return std::make_shared<PrefetchAbort>( 466 currState->vaddr_tainted, 467 ArmFault::TranslationLL + L1, 468 isStage2, 469 ArmFault::VmsaTran); 470 else 471 return std::make_shared<DataAbort>( 472 currState->vaddr_tainted, 473 TlbEntry::DomainType::NoAccess, currState->isWrite, 474 ArmFault::TranslationLL + L1, isStage2, 475 ArmFault::VmsaTran); 476 } 477 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 478 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 479 } else { 480 DPRINTF(TLB, " - Selecting TTBR1\n"); 481 // Check if table walk is allowed when Security Extensions are enabled 482 if (haveSecurity && currState->ttbcr.pd1) { 483 if (currState->isFetch) 484 return std::make_shared<PrefetchAbort>( 485 currState->vaddr_tainted, 486 ArmFault::TranslationLL + L1, 487 isStage2, 488 ArmFault::VmsaTran); 489 else 490 return std::make_shared<DataAbort>( 491 currState->vaddr_tainted, 492 TlbEntry::DomainType::NoAccess, currState->isWrite, 493 ArmFault::TranslationLL + L1, isStage2, 494 ArmFault::VmsaTran); 495 } 496 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 497 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 498 currState->ttbcr.n = 0; 499 } 500 501 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) | 502 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2); 503 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr, 504 currState->isSecure ? "s" : "ns"); 505 506 // Trickbox address check 507 Fault f; 508 f = testWalk(l1desc_addr, sizeof(uint32_t), 509 TlbEntry::DomainType::NoAccess, L1); 510 if (f) { 511 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 512 if (currState->timing) { 513 pending = false; 514 nextWalk(currState->tc); 515 currState = NULL; 516 } else { 517 currState->tc = NULL; 518 currState->req = NULL; 519 } 520 return f; 521 } 522 523 Request::Flags flag = Request::PT_WALK; 524 if (currState->sctlr.c == 0) { 525 flag.set(Request::UNCACHEABLE); 526 } 527 528 if (currState->isSecure) { 529 flag.set(Request::SECURE); 530 } 531 532 bool delayed; 533 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data, 534 sizeof(uint32_t), flag, L1, &doL1DescEvent, 535 &TableWalker::doL1Descriptor); 536 if (!delayed) { 537 f = currState->fault; 538 } 539 540 return f; 541} 542 543Fault 544TableWalker::processWalkLPAE() 545{ 546 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr; 547 int tsz, n; 548 LookupLevel start_lookup_level = L1; 549 550 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n", 551 currState->vaddr_tainted, currState->ttbcr); 552 553 statWalkWaitTime.sample(curTick() - currState->startTime); 554 555 Request::Flags flag = Request::PT_WALK; 556 if (currState->isSecure) 557 flag.set(Request::SECURE); 558 559 // work out which base address register to use, if in hyp mode we always 560 // use HTTBR 561 if (isStage2) { 562 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n"); 563 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR); 564 tsz = sext<4>(currState->vtcr.t0sz); 565 start_lookup_level = currState->vtcr.sl0 ? L1 : L2; 566 } else if (currState->isHyp) { 567 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n"); 568 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR); 569 tsz = currState->htcr.t0sz; 570 } else { 571 assert(longDescFormatInUse(currState->tc)); 572 573 // Determine boundaries of TTBR0/1 regions 574 if (currState->ttbcr.t0sz) 575 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1; 576 else if (currState->ttbcr.t1sz) 577 ttbr0_max = (1ULL << 32) - 578 (1ULL << (32 - currState->ttbcr.t1sz)) - 1; 579 else 580 ttbr0_max = (1ULL << 32) - 1; 581 if (currState->ttbcr.t1sz) 582 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)); 583 else 584 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz)); 585 586 // The following code snippet selects the appropriate translation table base 587 // address (TTBR0 or TTBR1) and the appropriate starting lookup level 588 // depending on the address range supported by the translation table (ARM 589 // ARM issue C B3.6.4) 590 if (currState->vaddr <= ttbr0_max) { 591 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n"); 592 // Check if table walk is allowed 593 if (currState->ttbcr.epd0) { 594 if (currState->isFetch) 595 return std::make_shared<PrefetchAbort>( 596 currState->vaddr_tainted, 597 ArmFault::TranslationLL + L1, 598 isStage2, 599 ArmFault::LpaeTran); 600 else 601 return std::make_shared<DataAbort>( 602 currState->vaddr_tainted, 603 TlbEntry::DomainType::NoAccess, 604 currState->isWrite, 605 ArmFault::TranslationLL + L1, 606 isStage2, 607 ArmFault::LpaeTran); 608 } 609 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 610 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 611 tsz = currState->ttbcr.t0sz; 612 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB 613 start_lookup_level = L2; 614 } else if (currState->vaddr >= ttbr1_min) { 615 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n"); 616 // Check if table walk is allowed 617 if (currState->ttbcr.epd1) { 618 if (currState->isFetch) 619 return std::make_shared<PrefetchAbort>( 620 currState->vaddr_tainted, 621 ArmFault::TranslationLL + L1, 622 isStage2, 623 ArmFault::LpaeTran); 624 else 625 return std::make_shared<DataAbort>( 626 currState->vaddr_tainted, 627 TlbEntry::DomainType::NoAccess, 628 currState->isWrite, 629 ArmFault::TranslationLL + L1, 630 isStage2, 631 ArmFault::LpaeTran); 632 } 633 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 634 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 635 tsz = currState->ttbcr.t1sz; 636 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB 637 start_lookup_level = L2; 638 } else { 639 // Out of boundaries -> translation fault 640 if (currState->isFetch) 641 return std::make_shared<PrefetchAbort>( 642 currState->vaddr_tainted, 643 ArmFault::TranslationLL + L1, 644 isStage2, 645 ArmFault::LpaeTran); 646 else 647 return std::make_shared<DataAbort>( 648 currState->vaddr_tainted, 649 TlbEntry::DomainType::NoAccess, 650 currState->isWrite, ArmFault::TranslationLL + L1, 651 isStage2, ArmFault::LpaeTran); 652 } 653 654 } 655 656 // Perform lookup (ARM ARM issue C B3.6.6) 657 if (start_lookup_level == L1) { 658 n = 5 - tsz; 659 desc_addr = mbits(ttbr, 39, n) | 660 (bits(currState->vaddr, n + 26, 30) << 3); 661 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 662 desc_addr, currState->isSecure ? "s" : "ns"); 663 } else { 664 // Skip first-level lookup 665 n = (tsz >= 2 ? 14 - tsz : 12); 666 desc_addr = mbits(ttbr, 39, n) | 667 (bits(currState->vaddr, n + 17, 21) << 3); 668 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 669 desc_addr, currState->isSecure ? "s" : "ns"); 670 } 671 672 // Trickbox address check 673 Fault f = testWalk(desc_addr, sizeof(uint64_t), 674 TlbEntry::DomainType::NoAccess, start_lookup_level); 675 if (f) { 676 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 677 if (currState->timing) { 678 pending = false; 679 nextWalk(currState->tc); 680 currState = NULL; 681 } else { 682 currState->tc = NULL; 683 currState->req = NULL; 684 } 685 return f; 686 } 687 688 if (currState->sctlr.c == 0) { 689 flag.set(Request::UNCACHEABLE); 690 } 691 692 currState->longDesc.lookupLevel = start_lookup_level; 693 currState->longDesc.aarch64 = false; 694 currState->longDesc.grainSize = Grain4KB; 695 696 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent 697 : (Event *) &doL2LongDescEvent; 698 699 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, 700 sizeof(uint64_t), flag, start_lookup_level, 701 event, &TableWalker::doLongDescriptor); 702 if (!delayed) { 703 f = currState->fault; 704 } 705 706 return f; 707} 708 709unsigned 710TableWalker::adjustTableSizeAArch64(unsigned tsz) 711{ 712 if (tsz < 25) 713 return 25; 714 if (tsz > 48) 715 return 48; 716 return tsz; 717} 718 719bool 720TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange) 721{ 722 return (currPhysAddrRange != MaxPhysAddrRange && 723 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange)); 724} 725 726Fault 727TableWalker::processWalkAArch64() 728{ 729 assert(currState->aarch64); 730 731 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n", 732 currState->vaddr_tainted, currState->tcr); 733 734 static const GrainSize GrainMapDefault[] = 735 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain }; 736 static const GrainSize GrainMap_EL1_tg1[] = 737 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB }; 738 739 statWalkWaitTime.sample(curTick() - currState->startTime); 740 741 // Determine TTBR, table size, granule size and phys. address range 742 Addr ttbr = 0; 743 int tsz = 0, ps = 0; 744 GrainSize tg = Grain4KB; // grain size computed from tg* field 745 bool fault = false; 746 switch (currState->el) { 747 case EL0: 748 case EL1: 749 switch (bits(currState->vaddr, 63,48)) { 750 case 0: 751 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 752 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1); 753 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 754 tg = GrainMapDefault[currState->tcr.tg0]; 755 if (bits(currState->vaddr, 63, tsz) != 0x0 || 756 currState->tcr.epd0) 757 fault = true; 758 break; 759 case 0xffff: 760 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); 761 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1); 762 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); 763 tg = GrainMap_EL1_tg1[currState->tcr.tg1]; 764 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || 765 currState->tcr.epd1) 766 fault = true; 767 break; 768 default: 769 // top two bytes must be all 0s or all 1s, else invalid addr 770 fault = true; 771 } 772 ps = currState->tcr.ips; 773 break; 774 case EL2: 775 case EL3: 776 switch(bits(currState->vaddr, 63,48)) { 777 case 0: 778 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 779 if (currState->el == EL2) 780 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2); 781 else 782 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3); 783 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 784 tg = GrainMapDefault[currState->tcr.tg0]; 785 break; 786 default: 787 // invalid addr if top two bytes are not all 0s 788 fault = true; 789 } 790 ps = currState->tcr.ips; 791 break; 792 } 793 794 if (fault) { 795 Fault f; 796 if (currState->isFetch) 797 f = std::make_shared<PrefetchAbort>( 798 currState->vaddr_tainted, 799 ArmFault::TranslationLL + L0, isStage2, 800 ArmFault::LpaeTran); 801 else 802 f = std::make_shared<DataAbort>( 803 currState->vaddr_tainted, 804 TlbEntry::DomainType::NoAccess, 805 currState->isWrite, 806 ArmFault::TranslationLL + L0, 807 isStage2, ArmFault::LpaeTran); 808 809 if (currState->timing) { 810 pending = false; 811 nextWalk(currState->tc); 812 currState = NULL; 813 } else { 814 currState->tc = NULL; 815 currState->req = NULL; 816 } 817 return f; 818 819 } 820 821 if (tg == ReservedGrain) { 822 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION " 823 "DEFINED behavior takes this to mean 4KB granules\n"); 824 tg = Grain4KB; 825 } 826 827 int stride = tg - 3; 828 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS; 829 830 // Determine starting lookup level 831 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library 832 // in ARM DDI 0487A. These table values correspond to the cascading tests 833 // to compute the lookup level and are of the form 834 // (grain_size + N*stride), for N = {1, 2, 3}. 835 // A value of 64 will never succeed and a value of 0 will always succeed. 836 { 837 struct GrainMap { 838 GrainSize grain_size; 839 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS]; 840 }; 841 static const GrainMap GM[] = { 842 { Grain4KB, { 39, 30, 0, 0 } }, 843 { Grain16KB, { 47, 36, 25, 0 } }, 844 { Grain64KB, { 64, 42, 29, 0 } } 845 }; 846 847 const unsigned *lookup = NULL; // points to a lookup_level_cutoff 848 849 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[] 850 if (tg == GM[i].grain_size) { 851 lookup = GM[i].lookup_level_cutoff; 852 break; 853 } 854 } 855 assert(lookup); 856 857 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) { 858 if (tsz > lookup[L]) { 859 start_lookup_level = (LookupLevel) L; 860 break; 861 } 862 } 863 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, 864 "Table walker couldn't find lookup level\n"); 865 } 866 867 // Determine table base address 868 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg; 869 Addr base_addr = mbits(ttbr, 47, base_addr_lo); 870 871 // Determine physical address size and raise an Address Size Fault if 872 // necessary 873 int pa_range = decodePhysAddrRange64(ps); 874 // Clamp to lower limit 875 if (pa_range > physAddrRange) 876 currState->physAddrRange = physAddrRange; 877 else 878 currState->physAddrRange = pa_range; 879 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) { 880 DPRINTF(TLB, "Address size fault before any lookup\n"); 881 Fault f; 882 if (currState->isFetch) 883 f = std::make_shared<PrefetchAbort>( 884 currState->vaddr_tainted, 885 ArmFault::AddressSizeLL + start_lookup_level, 886 isStage2, 887 ArmFault::LpaeTran); 888 else 889 f = std::make_shared<DataAbort>( 890 currState->vaddr_tainted, 891 TlbEntry::DomainType::NoAccess, 892 currState->isWrite, 893 ArmFault::AddressSizeLL + start_lookup_level, 894 isStage2, 895 ArmFault::LpaeTran); 896 897 898 if (currState->timing) { 899 pending = false; 900 nextWalk(currState->tc); 901 currState = NULL; 902 } else { 903 currState->tc = NULL; 904 currState->req = NULL; 905 } 906 return f; 907 908 } 909 910 // Determine descriptor address 911 Addr desc_addr = base_addr | 912 (bits(currState->vaddr, tsz - 1, 913 stride * (3 - start_lookup_level) + tg) << 3); 914 915 // Trickbox address check 916 Fault f = testWalk(desc_addr, sizeof(uint64_t), 917 TlbEntry::DomainType::NoAccess, start_lookup_level); 918 if (f) { 919 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 920 if (currState->timing) { 921 pending = false; 922 nextWalk(currState->tc); 923 currState = NULL; 924 } else { 925 currState->tc = NULL; 926 currState->req = NULL; 927 } 928 return f; 929 } 930 931 Request::Flags flag = Request::PT_WALK; 932 if (currState->sctlr.c == 0) { 933 flag.set(Request::UNCACHEABLE); 934 } 935 936 if (currState->isSecure) { 937 flag.set(Request::SECURE); 938 } 939 940 currState->longDesc.lookupLevel = start_lookup_level; 941 currState->longDesc.aarch64 = true; 942 currState->longDesc.grainSize = tg; 943 944 if (currState->timing) { 945 Event *event; 946 switch (start_lookup_level) { 947 case L0: 948 event = (Event *) &doL0LongDescEvent; 949 break; 950 case L1: 951 event = (Event *) &doL1LongDescEvent; 952 break; 953 case L2: 954 event = (Event *) &doL2LongDescEvent; 955 break; 956 case L3: 957 event = (Event *) &doL3LongDescEvent; 958 break; 959 default: 960 panic("Invalid table lookup level"); 961 break; 962 } 963 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 964 event, (uint8_t*) &currState->longDesc.data, 965 currState->tc->getCpuPtr()->clockPeriod(), flag); 966 DPRINTF(TLBVerbose, 967 "Adding to walker fifo: queue size before adding: %d\n", 968 stateQueues[start_lookup_level].size()); 969 stateQueues[start_lookup_level].push_back(currState); 970 currState = NULL; 971 } else if (!currState->functional) { 972 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 973 NULL, (uint8_t*) &currState->longDesc.data, 974 currState->tc->getCpuPtr()->clockPeriod(), flag); 975 doLongDescriptor(); 976 f = currState->fault; 977 } else { 978 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag, 979 masterId); 980 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 981 pkt->dataStatic((uint8_t*) &currState->longDesc.data); 982 port->sendFunctional(pkt); 983 doLongDescriptor(); 984 delete req; 985 delete pkt; 986 f = currState->fault; 987 } 988 989 return f; 990} 991 992void 993TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, 994 uint8_t texcb, bool s) 995{ 996 // Note: tc and sctlr local variables are hiding tc and sctrl class 997 // variables 998 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s); 999 te.shareable = false; // default value 1000 te.nonCacheable = false; 1001 te.outerShareable = false; 1002 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) { 1003 switch(texcb) { 1004 case 0: // Stongly-ordered 1005 te.nonCacheable = true; 1006 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1007 te.shareable = true; 1008 te.innerAttrs = 1; 1009 te.outerAttrs = 0; 1010 break; 1011 case 1: // Shareable Device 1012 te.nonCacheable = true; 1013 te.mtype = TlbEntry::MemoryType::Device; 1014 te.shareable = true; 1015 te.innerAttrs = 3; 1016 te.outerAttrs = 0; 1017 break; 1018 case 2: // Outer and Inner Write-Through, no Write-Allocate 1019 te.mtype = TlbEntry::MemoryType::Normal; 1020 te.shareable = s; 1021 te.innerAttrs = 6; 1022 te.outerAttrs = bits(texcb, 1, 0); 1023 break; 1024 case 3: // Outer and Inner Write-Back, no Write-Allocate 1025 te.mtype = TlbEntry::MemoryType::Normal; 1026 te.shareable = s; 1027 te.innerAttrs = 7; 1028 te.outerAttrs = bits(texcb, 1, 0); 1029 break; 1030 case 4: // Outer and Inner Non-cacheable 1031 te.nonCacheable = true; 1032 te.mtype = TlbEntry::MemoryType::Normal; 1033 te.shareable = s; 1034 te.innerAttrs = 0; 1035 te.outerAttrs = bits(texcb, 1, 0); 1036 break; 1037 case 5: // Reserved 1038 panic("Reserved texcb value!\n"); 1039 break; 1040 case 6: // Implementation Defined 1041 panic("Implementation-defined texcb value!\n"); 1042 break; 1043 case 7: // Outer and Inner Write-Back, Write-Allocate 1044 te.mtype = TlbEntry::MemoryType::Normal; 1045 te.shareable = s; 1046 te.innerAttrs = 5; 1047 te.outerAttrs = 1; 1048 break; 1049 case 8: // Non-shareable Device 1050 te.nonCacheable = true; 1051 te.mtype = TlbEntry::MemoryType::Device; 1052 te.shareable = false; 1053 te.innerAttrs = 3; 1054 te.outerAttrs = 0; 1055 break; 1056 case 9 ... 15: // Reserved 1057 panic("Reserved texcb value!\n"); 1058 break; 1059 case 16 ... 31: // Cacheable Memory 1060 te.mtype = TlbEntry::MemoryType::Normal; 1061 te.shareable = s; 1062 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0) 1063 te.nonCacheable = true; 1064 te.innerAttrs = bits(texcb, 1, 0); 1065 te.outerAttrs = bits(texcb, 3, 2); 1066 break; 1067 default: 1068 panic("More than 32 states for 5 bits?\n"); 1069 } 1070 } else { 1071 assert(tc); 1072 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR, 1073 currState->tc, !currState->isSecure)); 1074 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR, 1075 currState->tc, !currState->isSecure)); 1076 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr); 1077 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0; 1078 switch(bits(texcb, 2,0)) { 1079 case 0: 1080 curr_tr = prrr.tr0; 1081 curr_ir = nmrr.ir0; 1082 curr_or = nmrr.or0; 1083 te.outerShareable = (prrr.nos0 == 0); 1084 break; 1085 case 1: 1086 curr_tr = prrr.tr1; 1087 curr_ir = nmrr.ir1; 1088 curr_or = nmrr.or1; 1089 te.outerShareable = (prrr.nos1 == 0); 1090 break; 1091 case 2: 1092 curr_tr = prrr.tr2; 1093 curr_ir = nmrr.ir2; 1094 curr_or = nmrr.or2; 1095 te.outerShareable = (prrr.nos2 == 0); 1096 break; 1097 case 3: 1098 curr_tr = prrr.tr3; 1099 curr_ir = nmrr.ir3; 1100 curr_or = nmrr.or3; 1101 te.outerShareable = (prrr.nos3 == 0); 1102 break; 1103 case 4: 1104 curr_tr = prrr.tr4; 1105 curr_ir = nmrr.ir4; 1106 curr_or = nmrr.or4; 1107 te.outerShareable = (prrr.nos4 == 0); 1108 break; 1109 case 5: 1110 curr_tr = prrr.tr5; 1111 curr_ir = nmrr.ir5; 1112 curr_or = nmrr.or5; 1113 te.outerShareable = (prrr.nos5 == 0); 1114 break; 1115 case 6: 1116 panic("Imp defined type\n"); 1117 case 7: 1118 curr_tr = prrr.tr7; 1119 curr_ir = nmrr.ir7; 1120 curr_or = nmrr.or7; 1121 te.outerShareable = (prrr.nos7 == 0); 1122 break; 1123 } 1124 1125 switch(curr_tr) { 1126 case 0: 1127 DPRINTF(TLBVerbose, "StronglyOrdered\n"); 1128 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1129 te.nonCacheable = true; 1130 te.innerAttrs = 1; 1131 te.outerAttrs = 0; 1132 te.shareable = true; 1133 break; 1134 case 1: 1135 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n", 1136 prrr.ds1, prrr.ds0, s); 1137 te.mtype = TlbEntry::MemoryType::Device; 1138 te.nonCacheable = true; 1139 te.innerAttrs = 3; 1140 te.outerAttrs = 0; 1141 if (prrr.ds1 && s) 1142 te.shareable = true; 1143 if (prrr.ds0 && !s) 1144 te.shareable = true; 1145 break; 1146 case 2: 1147 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n", 1148 prrr.ns1, prrr.ns0, s); 1149 te.mtype = TlbEntry::MemoryType::Normal; 1150 if (prrr.ns1 && s) 1151 te.shareable = true; 1152 if (prrr.ns0 && !s) 1153 te.shareable = true; 1154 break; 1155 case 3: 1156 panic("Reserved type"); 1157 } 1158 1159 if (te.mtype == TlbEntry::MemoryType::Normal){ 1160 switch(curr_ir) { 1161 case 0: 1162 te.nonCacheable = true; 1163 te.innerAttrs = 0; 1164 break; 1165 case 1: 1166 te.innerAttrs = 5; 1167 break; 1168 case 2: 1169 te.innerAttrs = 6; 1170 break; 1171 case 3: 1172 te.innerAttrs = 7; 1173 break; 1174 } 1175 1176 switch(curr_or) { 1177 case 0: 1178 te.nonCacheable = true; 1179 te.outerAttrs = 0; 1180 break; 1181 case 1: 1182 te.outerAttrs = 1; 1183 break; 1184 case 2: 1185 te.outerAttrs = 2; 1186 break; 1187 case 3: 1188 te.outerAttrs = 3; 1189 break; 1190 } 1191 } 1192 } 1193 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, " 1194 "outerAttrs: %d\n", 1195 te.shareable, te.innerAttrs, te.outerAttrs); 1196 te.setAttributes(false); 1197} 1198 1199void 1200TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, 1201 LongDescriptor &lDescriptor) 1202{ 1203 assert(_haveLPAE); 1204 1205 uint8_t attr; 1206 uint8_t sh = lDescriptor.sh(); 1207 // Different format and source of attributes if this is a stage 2 1208 // translation 1209 if (isStage2) { 1210 attr = lDescriptor.memAttr(); 1211 uint8_t attr_3_2 = (attr >> 2) & 0x3; 1212 uint8_t attr_1_0 = attr & 0x3; 1213 1214 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh); 1215 1216 if (attr_3_2 == 0) { 1217 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered 1218 : TlbEntry::MemoryType::Device; 1219 te.outerAttrs = 0; 1220 te.innerAttrs = attr_1_0 == 0 ? 1 : 3; 1221 te.nonCacheable = true; 1222 } else { 1223 te.mtype = TlbEntry::MemoryType::Normal; 1224 te.outerAttrs = attr_3_2 == 1 ? 0 : 1225 attr_3_2 == 2 ? 2 : 1; 1226 te.innerAttrs = attr_1_0 == 1 ? 0 : 1227 attr_1_0 == 2 ? 6 : 5; 1228 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1); 1229 } 1230 } else { 1231 uint8_t attrIndx = lDescriptor.attrIndx(); 1232 1233 // LPAE always uses remapping of memory attributes, irrespective of the 1234 // value of SCTLR.TRE 1235 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0; 1236 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc, 1237 !currState->isSecure); 1238 uint32_t mair = currState->tc->readMiscReg(reg_as_int); 1239 attr = (mair >> (8 * (attrIndx % 4))) & 0xff; 1240 uint8_t attr_7_4 = bits(attr, 7, 4); 1241 uint8_t attr_3_0 = bits(attr, 3, 0); 1242 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr); 1243 1244 // Note: the memory subsystem only cares about the 'cacheable' memory 1245 // attribute. The other attributes are only used to fill the PAR register 1246 // accordingly to provide the illusion of full support 1247 te.nonCacheable = false; 1248 1249 switch (attr_7_4) { 1250 case 0x0: 1251 // Strongly-ordered or Device memory 1252 if (attr_3_0 == 0x0) 1253 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1254 else if (attr_3_0 == 0x4) 1255 te.mtype = TlbEntry::MemoryType::Device; 1256 else 1257 panic("Unpredictable behavior\n"); 1258 te.nonCacheable = true; 1259 te.outerAttrs = 0; 1260 break; 1261 case 0x4: 1262 // Normal memory, Outer Non-cacheable 1263 te.mtype = TlbEntry::MemoryType::Normal; 1264 te.outerAttrs = 0; 1265 if (attr_3_0 == 0x4) 1266 // Inner Non-cacheable 1267 te.nonCacheable = true; 1268 else if (attr_3_0 < 0x8) 1269 panic("Unpredictable behavior\n"); 1270 break; 1271 case 0x8: 1272 case 0x9: 1273 case 0xa: 1274 case 0xb: 1275 case 0xc: 1276 case 0xd: 1277 case 0xe: 1278 case 0xf: 1279 if (attr_7_4 & 0x4) { 1280 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3; 1281 } else { 1282 te.outerAttrs = 0x2; 1283 } 1284 // Normal memory, Outer Cacheable 1285 te.mtype = TlbEntry::MemoryType::Normal; 1286 if (attr_3_0 != 0x4 && attr_3_0 < 0x8) 1287 panic("Unpredictable behavior\n"); 1288 break; 1289 default: 1290 panic("Unpredictable behavior\n"); 1291 break; 1292 } 1293 1294 switch (attr_3_0) { 1295 case 0x0: 1296 te.innerAttrs = 0x1; 1297 break; 1298 case 0x4: 1299 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0; 1300 break; 1301 case 0x8: 1302 case 0x9: 1303 case 0xA: 1304 case 0xB: 1305 te.innerAttrs = 6; 1306 break; 1307 case 0xC: 1308 case 0xD: 1309 case 0xE: 1310 case 0xF: 1311 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7; 1312 break; 1313 default: 1314 panic("Unpredictable behavior\n"); 1315 break; 1316 } 1317 } 1318 1319 te.outerShareable = sh == 2; 1320 te.shareable = (sh & 0x2) ? true : false; 1321 te.setAttributes(true); 1322 te.attributes |= (uint64_t) attr << 56; 1323} 1324 1325void 1326TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx, 1327 uint8_t sh) 1328{ 1329 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh); 1330 1331 // Select MAIR 1332 uint64_t mair; 1333 switch (currState->el) { 1334 case EL0: 1335 case EL1: 1336 mair = tc->readMiscReg(MISCREG_MAIR_EL1); 1337 break; 1338 case EL2: 1339 mair = tc->readMiscReg(MISCREG_MAIR_EL2); 1340 break; 1341 case EL3: 1342 mair = tc->readMiscReg(MISCREG_MAIR_EL3); 1343 break; 1344 default: 1345 panic("Invalid exception level"); 1346 break; 1347 } 1348 1349 // Select attributes 1350 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx); 1351 uint8_t attr_lo = bits(attr, 3, 0); 1352 uint8_t attr_hi = bits(attr, 7, 4); 1353 1354 // Memory type 1355 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; 1356 1357 // Cacheability 1358 te.nonCacheable = false; 1359 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory 1360 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable 1361 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable 1362 te.nonCacheable = true; 1363 } 1364 1365 te.shareable = sh == 2; 1366 te.outerShareable = (sh & 0x2) ? true : false; 1367 // Attributes formatted according to the 64-bit PAR 1368 te.attributes = ((uint64_t) attr << 56) | 1369 (1 << 11) | // LPAE bit 1370 (te.ns << 9) | // NS bit 1371 (sh << 7); 1372} 1373 1374void 1375TableWalker::doL1Descriptor() 1376{ 1377 if (currState->fault != NoFault) { 1378 return; 1379 } 1380 1381 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n", 1382 currState->vaddr_tainted, currState->l1Desc.data); 1383 TlbEntry te; 1384 1385 switch (currState->l1Desc.type()) { 1386 case L1Descriptor::Ignore: 1387 case L1Descriptor::Reserved: 1388 if (!currState->timing) { 1389 currState->tc = NULL; 1390 currState->req = NULL; 1391 } 1392 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n"); 1393 if (currState->isFetch) 1394 currState->fault = 1395 std::make_shared<PrefetchAbort>( 1396 currState->vaddr_tainted, 1397 ArmFault::TranslationLL + L1, 1398 isStage2, 1399 ArmFault::VmsaTran); 1400 else 1401 currState->fault = 1402 std::make_shared<DataAbort>( 1403 currState->vaddr_tainted, 1404 TlbEntry::DomainType::NoAccess, 1405 currState->isWrite, 1406 ArmFault::TranslationLL + L1, isStage2, 1407 ArmFault::VmsaTran); 1408 return; 1409 case L1Descriptor::Section: 1410 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) { 1411 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is 1412 * enabled if set, do l1.Desc.setAp0() instead of generating 1413 * AccessFlag0 1414 */ 1415 1416 currState->fault = std::make_shared<DataAbort>( 1417 currState->vaddr_tainted, 1418 currState->l1Desc.domain(), 1419 currState->isWrite, 1420 ArmFault::AccessFlagLL + L1, 1421 isStage2, 1422 ArmFault::VmsaTran); 1423 } 1424 if (currState->l1Desc.supersection()) { 1425 panic("Haven't implemented supersections\n"); 1426 } 1427 insertTableEntry(currState->l1Desc, false); 1428 return; 1429 case L1Descriptor::PageTable: 1430 { 1431 Addr l2desc_addr; 1432 l2desc_addr = currState->l1Desc.l2Addr() | 1433 (bits(currState->vaddr, 19, 12) << 2); 1434 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n", 1435 l2desc_addr, currState->isSecure ? "s" : "ns"); 1436 1437 // Trickbox address check 1438 currState->fault = testWalk(l2desc_addr, sizeof(uint32_t), 1439 currState->l1Desc.domain(), L2); 1440 1441 if (currState->fault) { 1442 if (!currState->timing) { 1443 currState->tc = NULL; 1444 currState->req = NULL; 1445 } 1446 return; 1447 } 1448 1449 Request::Flags flag = Request::PT_WALK; 1450 if (currState->isSecure) 1451 flag.set(Request::SECURE); 1452 1453 bool delayed; 1454 delayed = fetchDescriptor(l2desc_addr, 1455 (uint8_t*)&currState->l2Desc.data, 1456 sizeof(uint32_t), flag, -1, &doL2DescEvent, 1457 &TableWalker::doL2Descriptor); 1458 if (delayed) { 1459 currState->delayed = true; 1460 } 1461 1462 return; 1463 } 1464 default: 1465 panic("A new type in a 2 bit field?\n"); 1466 } 1467} 1468 1469void 1470TableWalker::doLongDescriptor() 1471{ 1472 if (currState->fault != NoFault) { 1473 return; 1474 } 1475 1476 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n", 1477 currState->longDesc.lookupLevel, currState->vaddr_tainted, 1478 currState->longDesc.data, 1479 currState->aarch64 ? "AArch64" : "long-desc."); 1480 1481 if ((currState->longDesc.type() == LongDescriptor::Block) || 1482 (currState->longDesc.type() == LongDescriptor::Page)) { 1483 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, " 1484 "xn: %d, ap: %d, af: %d, type: %d\n", 1485 currState->longDesc.lookupLevel, 1486 currState->longDesc.data, 1487 currState->longDesc.pxn(), 1488 currState->longDesc.xn(), 1489 currState->longDesc.ap(), 1490 currState->longDesc.af(), 1491 currState->longDesc.type()); 1492 } else { 1493 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n", 1494 currState->longDesc.lookupLevel, 1495 currState->longDesc.data, 1496 currState->longDesc.type()); 1497 } 1498 1499 TlbEntry te; 1500 1501 switch (currState->longDesc.type()) { 1502 case LongDescriptor::Invalid: 1503 if (!currState->timing) { 1504 currState->tc = NULL; 1505 currState->req = NULL; 1506 } 1507 1508 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n", 1509 currState->longDesc.lookupLevel, 1510 ArmFault::TranslationLL + currState->longDesc.lookupLevel); 1511 if (currState->isFetch) 1512 currState->fault = std::make_shared<PrefetchAbort>( 1513 currState->vaddr_tainted, 1514 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1515 isStage2, 1516 ArmFault::LpaeTran); 1517 else 1518 currState->fault = std::make_shared<DataAbort>( 1519 currState->vaddr_tainted, 1520 TlbEntry::DomainType::NoAccess, 1521 currState->isWrite, 1522 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1523 isStage2, 1524 ArmFault::LpaeTran); 1525 return; 1526 case LongDescriptor::Block: 1527 case LongDescriptor::Page: 1528 { 1529 bool fault = false; 1530 bool aff = false; 1531 // Check for address size fault 1532 if (checkAddrSizeFaultAArch64( 1533 mbits(currState->longDesc.data, MaxPhysAddrRange - 1, 1534 currState->longDesc.offsetBits()), 1535 currState->physAddrRange)) { 1536 fault = true; 1537 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1538 currState->longDesc.lookupLevel); 1539 // Check for access fault 1540 } else if (currState->longDesc.af() == 0) { 1541 fault = true; 1542 DPRINTF(TLB, "L%d descriptor causing Access Fault\n", 1543 currState->longDesc.lookupLevel); 1544 aff = true; 1545 } 1546 if (fault) { 1547 if (currState->isFetch) 1548 currState->fault = std::make_shared<PrefetchAbort>( 1549 currState->vaddr_tainted, 1550 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1551 currState->longDesc.lookupLevel, 1552 isStage2, 1553 ArmFault::LpaeTran); 1554 else 1555 currState->fault = std::make_shared<DataAbort>( 1556 currState->vaddr_tainted, 1557 TlbEntry::DomainType::NoAccess, currState->isWrite, 1558 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1559 currState->longDesc.lookupLevel, 1560 isStage2, 1561 ArmFault::LpaeTran); 1562 } else { 1563 insertTableEntry(currState->longDesc, true); 1564 } 1565 } 1566 return; 1567 case LongDescriptor::Table: 1568 { 1569 // Set hierarchical permission flags 1570 currState->secureLookup = currState->secureLookup && 1571 currState->longDesc.secureTable(); 1572 currState->rwTable = currState->rwTable && 1573 currState->longDesc.rwTable(); 1574 currState->userTable = currState->userTable && 1575 currState->longDesc.userTable(); 1576 currState->xnTable = currState->xnTable || 1577 currState->longDesc.xnTable(); 1578 currState->pxnTable = currState->pxnTable || 1579 currState->longDesc.pxnTable(); 1580 1581 // Set up next level lookup 1582 Addr next_desc_addr = currState->longDesc.nextDescAddr( 1583 currState->vaddr); 1584 1585 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n", 1586 currState->longDesc.lookupLevel, 1587 currState->longDesc.lookupLevel + 1, 1588 next_desc_addr, 1589 currState->secureLookup ? "s" : "ns"); 1590 1591 // Check for address size fault 1592 if (currState->aarch64 && checkAddrSizeFaultAArch64( 1593 next_desc_addr, currState->physAddrRange)) { 1594 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1595 currState->longDesc.lookupLevel); 1596 if (currState->isFetch) 1597 currState->fault = std::make_shared<PrefetchAbort>( 1598 currState->vaddr_tainted, 1599 ArmFault::AddressSizeLL 1600 + currState->longDesc.lookupLevel, 1601 isStage2, 1602 ArmFault::LpaeTran); 1603 else 1604 currState->fault = std::make_shared<DataAbort>( 1605 currState->vaddr_tainted, 1606 TlbEntry::DomainType::NoAccess, currState->isWrite, 1607 ArmFault::AddressSizeLL 1608 + currState->longDesc.lookupLevel, 1609 isStage2, 1610 ArmFault::LpaeTran); 1611 return; 1612 } 1613 1614 // Trickbox address check 1615 currState->fault = testWalk( 1616 next_desc_addr, sizeof(uint64_t), TlbEntry::DomainType::Client, 1617 toLookupLevel(currState->longDesc.lookupLevel +1)); 1618 1619 if (currState->fault) { 1620 if (!currState->timing) { 1621 currState->tc = NULL; 1622 currState->req = NULL; 1623 } 1624 return; 1625 } 1626 1627 Request::Flags flag = Request::PT_WALK; 1628 if (currState->secureLookup) 1629 flag.set(Request::SECURE); 1630 1631 currState->longDesc.lookupLevel = 1632 (LookupLevel) (currState->longDesc.lookupLevel + 1); 1633 Event *event = NULL; 1634 switch (currState->longDesc.lookupLevel) { 1635 case L1: 1636 assert(currState->aarch64); 1637 event = &doL1LongDescEvent; 1638 break; 1639 case L2: 1640 event = &doL2LongDescEvent; 1641 break; 1642 case L3: 1643 event = &doL3LongDescEvent; 1644 break; 1645 default: 1646 panic("Wrong lookup level in table walk\n"); 1647 break; 1648 } 1649 1650 bool delayed; 1651 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data, 1652 sizeof(uint64_t), flag, -1, event, 1653 &TableWalker::doLongDescriptor); 1654 if (delayed) { 1655 currState->delayed = true; 1656 } 1657 } 1658 return; 1659 default: 1660 panic("A new type in a 2 bit field?\n"); 1661 } 1662} 1663 1664void 1665TableWalker::doL2Descriptor() 1666{ 1667 if (currState->fault != NoFault) { 1668 return; 1669 } 1670 1671 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n", 1672 currState->vaddr_tainted, currState->l2Desc.data); 1673 TlbEntry te; 1674 1675 if (currState->l2Desc.invalid()) { 1676 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n"); 1677 if (!currState->timing) { 1678 currState->tc = NULL; 1679 currState->req = NULL; 1680 } 1681 if (currState->isFetch) 1682 currState->fault = std::make_shared<PrefetchAbort>( 1683 currState->vaddr_tainted, 1684 ArmFault::TranslationLL + L2, 1685 isStage2, 1686 ArmFault::VmsaTran); 1687 else 1688 currState->fault = std::make_shared<DataAbort>( 1689 currState->vaddr_tainted, currState->l1Desc.domain(), 1690 currState->isWrite, ArmFault::TranslationLL + L2, 1691 isStage2, 1692 ArmFault::VmsaTran); 1693 return; 1694 } 1695 1696 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) { 1697 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled 1698 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0 1699 */ 1700 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n", 1701 currState->sctlr.afe, currState->l2Desc.ap()); 1702 1703 currState->fault = std::make_shared<DataAbort>( 1704 currState->vaddr_tainted, 1705 TlbEntry::DomainType::NoAccess, currState->isWrite, 1706 ArmFault::AccessFlagLL + L2, isStage2, 1707 ArmFault::VmsaTran); 1708 } 1709 1710 insertTableEntry(currState->l2Desc, false); 1711} 1712 1713void 1714TableWalker::doL1DescriptorWrapper() 1715{ 1716 currState = stateQueues[L1].front(); 1717 currState->delayed = false; 1718 // if there's a stage2 translation object we don't need it any more 1719 if (currState->stage2Tran) { 1720 delete currState->stage2Tran; 1721 currState->stage2Tran = NULL; 1722 } 1723 1724 1725 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data); 1726 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data); 1727 1728 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted); 1729 doL1Descriptor(); 1730 1731 stateQueues[L1].pop_front(); 1732 // Check if fault was generated 1733 if (currState->fault != NoFault) { 1734 currState->transState->finish(currState->fault, currState->req, 1735 currState->tc, currState->mode); 1736 statWalksShortTerminatedAtLevel[0]++; 1737 1738 pending = false; 1739 nextWalk(currState->tc); 1740 1741 currState->req = NULL; 1742 currState->tc = NULL; 1743 currState->delayed = false; 1744 delete currState; 1745 } 1746 else if (!currState->delayed) { 1747 // delay is not set so there is no L2 to do 1748 // Don't finish the translation if a stage 2 look up is underway 1749 if (!currState->doingStage2) { 1750 statWalkServiceTime.sample(curTick() - currState->startTime); 1751 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1752 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1753 currState->transState, currState->mode); 1754 statWalksShortTerminatedAtLevel[0]++; 1755 } 1756 1757 pending = false; 1758 nextWalk(currState->tc); 1759 1760 currState->req = NULL; 1761 currState->tc = NULL; 1762 currState->delayed = false; 1763 delete currState; 1764 } else { 1765 // need to do L2 descriptor 1766 stateQueues[L2].push_back(currState); 1767 } 1768 currState = NULL; 1769} 1770 1771void 1772TableWalker::doL2DescriptorWrapper() 1773{ 1774 currState = stateQueues[L2].front(); 1775 assert(currState->delayed); 1776 // if there's a stage2 translation object we don't need it any more 1777 if (currState->stage2Tran) { 1778 delete currState->stage2Tran; 1779 currState->stage2Tran = NULL; 1780 } 1781 1782 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n", 1783 currState->vaddr_tainted); 1784 doL2Descriptor(); 1785 1786 // Check if fault was generated 1787 if (currState->fault != NoFault) { 1788 currState->transState->finish(currState->fault, currState->req, 1789 currState->tc, currState->mode); 1790 statWalksShortTerminatedAtLevel[1]++; 1791 } 1792 else { 1793 // Don't finish the translation if a stage 2 look up is underway 1794 if (!currState->doingStage2) { 1795 statWalkServiceTime.sample(curTick() - currState->startTime); 1796 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1797 currState->fault = tlb->translateTiming(currState->req, 1798 currState->tc, currState->transState, currState->mode); 1799 statWalksShortTerminatedAtLevel[1]++; 1800 } 1801 } 1802 1803 1804 stateQueues[L2].pop_front(); 1805 pending = false; 1806 nextWalk(currState->tc); 1807 1808 currState->req = NULL; 1809 currState->tc = NULL; 1810 currState->delayed = false; 1811 1812 delete currState; 1813 currState = NULL; 1814} 1815 1816void 1817TableWalker::doL0LongDescriptorWrapper() 1818{ 1819 doLongDescriptorWrapper(L0); 1820} 1821 1822void 1823TableWalker::doL1LongDescriptorWrapper() 1824{ 1825 doLongDescriptorWrapper(L1); 1826} 1827 1828void 1829TableWalker::doL2LongDescriptorWrapper() 1830{ 1831 doLongDescriptorWrapper(L2); 1832} 1833 1834void 1835TableWalker::doL3LongDescriptorWrapper() 1836{ 1837 doLongDescriptorWrapper(L3); 1838} 1839 1840void 1841TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) 1842{ 1843 currState = stateQueues[curr_lookup_level].front(); 1844 assert(curr_lookup_level == currState->longDesc.lookupLevel); 1845 currState->delayed = false; 1846 1847 // if there's a stage2 translation object we don't need it any more 1848 if (currState->stage2Tran) { 1849 delete currState->stage2Tran; 1850 currState->stage2Tran = NULL; 1851 } 1852 1853 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n", 1854 currState->vaddr_tainted); 1855 doLongDescriptor(); 1856 1857 stateQueues[curr_lookup_level].pop_front(); 1858 1859 if (currState->fault != NoFault) { 1860 // A fault was generated 1861 currState->transState->finish(currState->fault, currState->req, 1862 currState->tc, currState->mode); 1863 1864 pending = false; 1865 nextWalk(currState->tc); 1866 1867 currState->req = NULL; 1868 currState->tc = NULL; 1869 currState->delayed = false; 1870 delete currState; 1871 } else if (!currState->delayed) { 1872 // No additional lookups required 1873 // Don't finish the translation if a stage 2 look up is underway 1874 if (!currState->doingStage2) { 1875 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1876 statWalkServiceTime.sample(curTick() - currState->startTime); 1877 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1878 currState->transState, 1879 currState->mode); 1880 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; 1881 } 1882 1883 pending = false; 1884 nextWalk(currState->tc); 1885 1886 currState->req = NULL; 1887 currState->tc = NULL; 1888 currState->delayed = false; 1889 delete currState; 1890 } else { 1891 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1) 1892 panic("Max. number of lookups already reached in table walk\n"); 1893 // Need to perform additional lookups 1894 stateQueues[currState->longDesc.lookupLevel].push_back(currState); 1895 } 1896 currState = NULL; 1897} 1898 1899 1900void 1901TableWalker::nextWalk(ThreadContext *tc) 1902{ 1903 if (pendingQueue.size()) 1904 schedule(doProcessEvent, clockEdge(Cycles(1))); 1905 else 1906 completeDrain(); 1907} 1908 1909bool 1910TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, 1911 Request::Flags flags, int queueIndex, Event *event, 1912 void (TableWalker::*doDescriptor)()) 1913{ 1914 bool isTiming = currState->timing; 1915 1916 // do the requests for the page table descriptors have to go through the 1917 // second stage MMU 1918 if (currState->stage2Req) { 1919 Fault fault; 1920 flags = flags | TLB::MustBeOne; 1921 1922 if (isTiming) { 1923 Stage2MMU::Stage2Translation *tran = new 1924 Stage2MMU::Stage2Translation(*stage2Mmu, data, event, 1925 currState->vaddr); 1926 currState->stage2Tran = tran; 1927 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes, 1928 flags); 1929 fault = tran->fault; 1930 } else { 1931 fault = stage2Mmu->readDataUntimed(currState->tc, 1932 currState->vaddr, descAddr, data, numBytes, flags, 1933 currState->functional); 1934 } 1935 1936 if (fault != NoFault) { 1937 currState->fault = fault; 1938 } 1939 if (isTiming) { 1940 if (queueIndex >= 0) { 1941 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1942 stateQueues[queueIndex].size()); 1943 stateQueues[queueIndex].push_back(currState); 1944 currState = NULL; 1945 } 1946 } else { 1947 (this->*doDescriptor)(); 1948 } 1949 } else { 1950 if (isTiming) { 1951 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data, 1952 currState->tc->getCpuPtr()->clockPeriod(),flags); 1953 if (queueIndex >= 0) { 1954 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1955 stateQueues[queueIndex].size()); 1956 stateQueues[queueIndex].push_back(currState); 1957 currState = NULL; 1958 } 1959 } else if (!currState->functional) { 1960 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data, 1961 currState->tc->getCpuPtr()->clockPeriod(), flags); 1962 (this->*doDescriptor)(); 1963 } else { 1964 RequestPtr req = new Request(descAddr, numBytes, flags, masterId); 1965 req->taskId(ContextSwitchTaskId::DMA); 1966 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 1967 pkt->dataStatic(data); 1968 port->sendFunctional(pkt); 1969 (this->*doDescriptor)(); 1970 delete req; 1971 delete pkt; 1972 } 1973 } 1974 return (isTiming); 1975} 1976 1977void 1978TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor) 1979{ 1980 TlbEntry te; 1981 1982 // Create and fill a new page table entry 1983 te.valid = true; 1984 te.longDescFormat = longDescriptor; 1985 te.isHyp = currState->isHyp; 1986 te.asid = currState->asid; 1987 te.vmid = currState->vmid; 1988 te.N = descriptor.offsetBits(); 1989 te.vpn = currState->vaddr >> te.N; 1990 te.size = (1<<te.N) - 1; 1991 te.pfn = descriptor.pfn(); 1992 te.domain = descriptor.domain(); 1993 te.lookupLevel = descriptor.lookupLevel; 1994 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2; 1995 te.nstid = !currState->isSecure; 1996 te.xn = descriptor.xn(); 1997 if (currState->aarch64) 1998 te.el = currState->el; 1999 else 2000 te.el = 1; 2001 2002 statPageSizes[pageSizeNtoStatBin(te.N)]++; 2003 statRequestOrigin[COMPLETED][currState->isFetch]++; 2004 2005 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries 2006 // as global 2007 te.global = descriptor.global(currState) || isStage2; 2008 if (longDescriptor) { 2009 LongDescriptor lDescriptor = 2010 dynamic_cast<LongDescriptor &>(descriptor); 2011 2012 te.xn |= currState->xnTable; 2013 te.pxn = currState->pxnTable || lDescriptor.pxn(); 2014 if (isStage2) { 2015 // this is actually the HAP field, but its stored in the same bit 2016 // possitions as the AP field in a stage 1 translation. 2017 te.hap = lDescriptor.ap(); 2018 } else { 2019 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) | 2020 (currState->userTable && (descriptor.ap() & 0x1)); 2021 } 2022 if (currState->aarch64) 2023 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(), 2024 currState->longDesc.sh()); 2025 else 2026 memAttrsLPAE(currState->tc, te, lDescriptor); 2027 } else { 2028 te.ap = descriptor.ap(); 2029 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(), 2030 descriptor.shareable()); 2031 } 2032 2033 // Debug output 2034 DPRINTF(TLB, descriptor.dbgHeader().c_str()); 2035 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n", 2036 te.N, te.pfn, te.size, te.global, te.valid); 2037 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d " 2038 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn, 2039 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp, 2040 te.nonCacheable, te.ns); 2041 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n", 2042 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()), 2043 descriptor.getRawData()); 2044 2045 // Insert the entry into the TLB 2046 tlb->insert(currState->vaddr, te); 2047 if (!currState->timing) { 2048 currState->tc = NULL; 2049 currState->req = NULL; 2050 } 2051} 2052 2053ArmISA::TableWalker * 2054ArmTableWalkerParams::create() 2055{ 2056 return new ArmISA::TableWalker(this); 2057} 2058 2059LookupLevel 2060TableWalker::toLookupLevel(uint8_t lookup_level_as_int) 2061{ 2062 switch (lookup_level_as_int) { 2063 case L1: 2064 return L1; 2065 case L2: 2066 return L2; 2067 case L3: 2068 return L3; 2069 default: 2070 panic("Invalid lookup level conversion"); 2071 } 2072} 2073 2074/* this method keeps track of the table walker queue's residency, so 2075 * needs to be called whenever requests start and complete. */ 2076void 2077TableWalker::pendingChange() 2078{ 2079 unsigned n = pendingQueue.size(); 2080 if ((currState != NULL) && (currState != pendingQueue.front())) { 2081 ++n; 2082 } 2083 2084 if (n != pendingReqs) { 2085 Tick now = curTick(); 2086 statPendingWalks.sample(pendingReqs, now - pendingChangeTick); 2087 pendingReqs = n; 2088 pendingChangeTick = now; 2089 } 2090} 2091 2092Fault 2093TableWalker::testWalk(Addr pa, Addr size, TlbEntry::DomainType domain, 2094 LookupLevel lookup_level) 2095{ 2096 return tlb->testWalk(pa, size, currState->vaddr, currState->isSecure, 2097 currState->mode, domain, lookup_level); 2098} 2099 2100 2101uint8_t 2102TableWalker::pageSizeNtoStatBin(uint8_t N) 2103{ 2104 /* for statPageSizes */ 2105 switch(N) { 2106 case 12: return 0; // 4K 2107 case 14: return 1; // 16K (using 16K granule in v8-64) 2108 case 16: return 2; // 64K 2109 case 20: return 3; // 1M 2110 case 21: return 4; // 2M-LPAE 2111 case 24: return 5; // 16M 2112 case 25: return 6; // 32M (using 16K granule in v8-64) 2113 case 29: return 7; // 512M (using 64K granule in v8-64) 2114 case 30: return 8; // 1G-LPAE 2115 default: 2116 panic("unknown page size"); 2117 return 255; 2118 } 2119} 2120 2121void 2122TableWalker::regStats() 2123{ 2124 ClockedObject::regStats(); 2125 2126 statWalks 2127 .name(name() + ".walks") 2128 .desc("Table walker walks requested") 2129 ; 2130 2131 statWalksShortDescriptor 2132 .name(name() + ".walksShort") 2133 .desc("Table walker walks initiated with short descriptors") 2134 .flags(Stats::nozero) 2135 ; 2136 2137 statWalksLongDescriptor 2138 .name(name() + ".walksLong") 2139 .desc("Table walker walks initiated with long descriptors") 2140 .flags(Stats::nozero) 2141 ; 2142 2143 statWalksShortTerminatedAtLevel 2144 .init(2) 2145 .name(name() + ".walksShortTerminationLevel") 2146 .desc("Level at which table walker walks " 2147 "with short descriptors terminate") 2148 .flags(Stats::nozero) 2149 ; 2150 statWalksShortTerminatedAtLevel.subname(0, "Level1"); 2151 statWalksShortTerminatedAtLevel.subname(1, "Level2"); 2152 2153 statWalksLongTerminatedAtLevel 2154 .init(4) 2155 .name(name() + ".walksLongTerminationLevel") 2156 .desc("Level at which table walker walks " 2157 "with long descriptors terminate") 2158 .flags(Stats::nozero) 2159 ; 2160 statWalksLongTerminatedAtLevel.subname(0, "Level0"); 2161 statWalksLongTerminatedAtLevel.subname(1, "Level1"); 2162 statWalksLongTerminatedAtLevel.subname(2, "Level2"); 2163 statWalksLongTerminatedAtLevel.subname(3, "Level3"); 2164 2165 statSquashedBefore 2166 .name(name() + ".walksSquashedBefore") 2167 .desc("Table walks squashed before starting") 2168 .flags(Stats::nozero) 2169 ; 2170 2171 statSquashedAfter 2172 .name(name() + ".walksSquashedAfter") 2173 .desc("Table walks squashed after completion") 2174 .flags(Stats::nozero) 2175 ; 2176 2177 statWalkWaitTime 2178 .init(16) 2179 .name(name() + ".walkWaitTime") 2180 .desc("Table walker wait (enqueue to first request) latency") 2181 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2182 ; 2183 2184 statWalkServiceTime 2185 .init(16) 2186 .name(name() + ".walkCompletionTime") 2187 .desc("Table walker service (enqueue to completion) latency") 2188 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2189 ; 2190 2191 statPendingWalks 2192 .init(16) 2193 .name(name() + ".walksPending") 2194 .desc("Table walker pending requests distribution") 2195 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan) 2196 ; 2197 2198 statPageSizes // see DDI 0487A D4-1661 2199 .init(9) 2200 .name(name() + ".walkPageSizes") 2201 .desc("Table walker page sizes translated") 2202 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero) 2203 ; 2204 statPageSizes.subname(0, "4K"); 2205 statPageSizes.subname(1, "16K"); 2206 statPageSizes.subname(2, "64K"); 2207 statPageSizes.subname(3, "1M"); 2208 statPageSizes.subname(4, "2M"); 2209 statPageSizes.subname(5, "16M"); 2210 statPageSizes.subname(6, "32M"); 2211 statPageSizes.subname(7, "512M"); 2212 statPageSizes.subname(8, "1G"); 2213 2214 statRequestOrigin 2215 .init(2,2) // Instruction/Data, requests/completed 2216 .name(name() + ".walkRequestOrigin") 2217 .desc("Table walker requests started/completed, data/inst") 2218 .flags(Stats::total) 2219 ; 2220 statRequestOrigin.subname(0,"Requested"); 2221 statRequestOrigin.subname(1,"Completed"); 2222 statRequestOrigin.ysubname(0,"Data"); 2223 statRequestOrigin.ysubname(1,"Inst"); 2224} 2225