table_walker.cc revision 10913
1/* 2 * Copyright (c) 2010, 2012-2015 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Ali Saidi 38 * Giacomo Gabrielli 39 */ 40#include "arch/arm/table_walker.hh" 41 42#include <memory> 43 44#include "arch/arm/faults.hh" 45#include "arch/arm/stage2_mmu.hh" 46#include "arch/arm/system.hh" 47#include "arch/arm/tlb.hh" 48#include "cpu/base.hh" 49#include "cpu/thread_context.hh" 50#include "debug/Checkpoint.hh" 51#include "debug/Drain.hh" 52#include "debug/TLB.hh" 53#include "debug/TLBVerbose.hh" 54#include "dev/dma_device.hh" 55#include "sim/system.hh" 56 57using namespace ArmISA; 58 59TableWalker::TableWalker(const Params *p) 60 : MemObject(p), 61 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId), 62 isStage2(p->is_stage2), tlb(NULL), 63 currState(NULL), pending(false), 64 numSquashable(p->num_squash_per_cycle), 65 pendingReqs(0), 66 pendingChangeTick(curTick()), 67 doL1DescEvent(this), doL2DescEvent(this), 68 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this), 69 doL3LongDescEvent(this), 70 doProcessEvent(this) 71{ 72 sctlr = 0; 73 74 // Cache system-level properties 75 if (FullSystem) { 76 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys); 77 assert(armSys); 78 haveSecurity = armSys->haveSecurity(); 79 _haveLPAE = armSys->haveLPAE(); 80 _haveVirtualization = armSys->haveVirtualization(); 81 physAddrRange = armSys->physAddrRange(); 82 _haveLargeAsid64 = armSys->haveLargeAsid64(); 83 } else { 84 haveSecurity = _haveLPAE = _haveVirtualization = false; 85 _haveLargeAsid64 = false; 86 physAddrRange = 32; 87 } 88 89} 90 91TableWalker::~TableWalker() 92{ 93 ; 94} 95 96void 97TableWalker::setMMU(Stage2MMU *m, MasterID master_id) 98{ 99 stage2Mmu = m; 100 port = &m->getPort(); 101 masterId = master_id; 102} 103 104void 105TableWalker::init() 106{ 107 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n"); 108 fatal_if(!port, "Table walker must have a valid port\n"); 109 fatal_if(!tlb, "Table walker must have a valid TLB\n"); 110} 111 112BaseMasterPort& 113TableWalker::getMasterPort(const std::string &if_name, PortID idx) 114{ 115 if (if_name == "port") { 116 if (!isStage2) { 117 return *port; 118 } else { 119 fatal("Cannot access table walker port through stage-two walker\n"); 120 } 121 } 122 return MemObject::getMasterPort(if_name, idx); 123} 124 125TableWalker::WalkerState::WalkerState() : 126 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr), 127 asid(0), vmid(0), isHyp(false), transState(nullptr), 128 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false), 129 secureLookup(false), rwTable(false), userTable(false), xnTable(false), 130 pxnTable(false), stage2Req(false), doingStage2(false), 131 stage2Tran(nullptr), timing(false), functional(false), 132 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), 133 delayed(false), tableWalker(nullptr) 134{ 135} 136 137void 138TableWalker::completeDrain() 139{ 140 if (drainState() == DrainState::Draining && 141 stateQueues[L1].empty() && stateQueues[L2].empty() && 142 pendingQueue.empty()) { 143 144 DPRINTF(Drain, "TableWalker done draining, processing drain event\n"); 145 signalDrainDone(); 146 } 147} 148 149DrainState 150TableWalker::drain() 151{ 152 bool state_queues_not_empty = false; 153 154 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) { 155 if (!stateQueues[i].empty()) { 156 state_queues_not_empty = true; 157 break; 158 } 159 } 160 161 if (state_queues_not_empty || pendingQueue.size()) { 162 DPRINTF(Drain, "TableWalker not drained\n"); 163 return DrainState::Draining; 164 } else { 165 DPRINTF(Drain, "TableWalker free, no need to drain\n"); 166 return DrainState::Drained; 167 } 168} 169 170void 171TableWalker::drainResume() 172{ 173 if (params()->sys->isTimingMode() && currState) { 174 delete currState; 175 currState = NULL; 176 pendingChange(); 177 } 178} 179 180Fault 181TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid, 182 uint8_t _vmid, bool _isHyp, TLB::Mode _mode, 183 TLB::Translation *_trans, bool _timing, bool _functional, 184 bool secure, TLB::ArmTranslationType tranType) 185{ 186 assert(!(_functional && _timing)); 187 ++statWalks; 188 189 WalkerState *savedCurrState = NULL; 190 191 if (!currState && !_functional) { 192 // For atomic mode, a new WalkerState instance should be only created 193 // once per TLB. For timing mode, a new instance is generated for every 194 // TLB miss. 195 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n"); 196 197 currState = new WalkerState(); 198 currState->tableWalker = this; 199 } else if (_functional) { 200 // If we are mixing functional mode with timing (or even 201 // atomic), we need to to be careful and clean up after 202 // ourselves to not risk getting into an inconsistent state. 203 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n"); 204 savedCurrState = currState; 205 currState = new WalkerState(); 206 currState->tableWalker = this; 207 } else if (_timing) { 208 // This is a translation that was completed and then faulted again 209 // because some underlying parameters that affect the translation 210 // changed out from under us (e.g. asid). It will either be a 211 // misprediction, in which case nothing will happen or we'll use 212 // this fault to re-execute the faulting instruction which should clean 213 // up everything. 214 if (currState->vaddr_tainted == _req->getVaddr()) { 215 ++statSquashedBefore; 216 return std::make_shared<ReExec>(); 217 } 218 } 219 pendingChange(); 220 221 currState->startTime = curTick(); 222 currState->tc = _tc; 223 currState->aarch64 = opModeIs64(currOpMode(_tc)); 224 currState->el = currEL(_tc); 225 currState->transState = _trans; 226 currState->req = _req; 227 currState->fault = NoFault; 228 currState->asid = _asid; 229 currState->vmid = _vmid; 230 currState->isHyp = _isHyp; 231 currState->timing = _timing; 232 currState->functional = _functional; 233 currState->mode = _mode; 234 currState->tranType = tranType; 235 currState->isSecure = secure; 236 currState->physAddrRange = physAddrRange; 237 238 /** @todo These should be cached or grabbed from cached copies in 239 the TLB, all these miscreg reads are expensive */ 240 currState->vaddr_tainted = currState->req->getVaddr(); 241 if (currState->aarch64) 242 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted, 243 currState->tc, currState->el); 244 else 245 currState->vaddr = currState->vaddr_tainted; 246 247 if (currState->aarch64) { 248 switch (currState->el) { 249 case EL0: 250 case EL1: 251 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); 252 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1); 253 break; 254 // @todo: uncomment this to enable Virtualization 255 // case EL2: 256 // assert(haveVirtualization); 257 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2); 258 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2); 259 // break; 260 case EL3: 261 assert(haveSecurity); 262 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3); 263 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3); 264 break; 265 default: 266 panic("Invalid exception level"); 267 break; 268 } 269 } else { 270 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 271 MISCREG_SCTLR, currState->tc, !currState->isSecure)); 272 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 273 MISCREG_TTBCR, currState->tc, !currState->isSecure)); 274 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR); 275 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR); 276 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR); 277 } 278 sctlr = currState->sctlr; 279 280 currState->isFetch = (currState->mode == TLB::Execute); 281 currState->isWrite = (currState->mode == TLB::Write); 282 283 statRequestOrigin[REQUESTED][currState->isFetch]++; 284 285 // We only do a second stage of translation if we're not secure, or in 286 // hyp mode, the second stage MMU is enabled, and this table walker 287 // instance is the first stage. 288 currState->doingStage2 = false; 289 // @todo: for now disable this in AArch64 (HCR is not set) 290 currState->stage2Req = !currState->aarch64 && currState->hcr.vm && 291 !isStage2 && !currState->isSecure && !currState->isHyp; 292 293 bool long_desc_format = currState->aarch64 || 294 (_haveLPAE && currState->ttbcr.eae) || 295 _isHyp || isStage2; 296 297 if (long_desc_format) { 298 // Helper variables used for hierarchical permissions 299 currState->secureLookup = currState->isSecure; 300 currState->rwTable = true; 301 currState->userTable = true; 302 currState->xnTable = false; 303 currState->pxnTable = false; 304 305 ++statWalksLongDescriptor; 306 } else { 307 ++statWalksShortDescriptor; 308 } 309 310 if (!currState->timing) { 311 Fault fault = NoFault; 312 if (currState->aarch64) 313 fault = processWalkAArch64(); 314 else if (long_desc_format) 315 fault = processWalkLPAE(); 316 else 317 fault = processWalk(); 318 319 // If this was a functional non-timing access restore state to 320 // how we found it. 321 if (currState->functional) { 322 delete currState; 323 currState = savedCurrState; 324 } 325 return fault; 326 } 327 328 if (pending || pendingQueue.size()) { 329 pendingQueue.push_back(currState); 330 currState = NULL; 331 pendingChange(); 332 } else { 333 pending = true; 334 pendingChange(); 335 if (currState->aarch64) 336 return processWalkAArch64(); 337 else if (long_desc_format) 338 return processWalkLPAE(); 339 else 340 return processWalk(); 341 } 342 343 return NoFault; 344} 345 346void 347TableWalker::processWalkWrapper() 348{ 349 assert(!currState); 350 assert(pendingQueue.size()); 351 pendingChange(); 352 currState = pendingQueue.front(); 353 354 ExceptionLevel target_el = EL0; 355 if (currState->aarch64) 356 target_el = currEL(currState->tc); 357 else 358 target_el = EL1; 359 360 // Check if a previous walk filled this request already 361 // @TODO Should this always be the TLB or should we look in the stage2 TLB? 362 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid, 363 currState->vmid, currState->isHyp, currState->isSecure, true, false, 364 target_el); 365 366 // Check if we still need to have a walk for this request. If the requesting 367 // instruction has been squashed, or a previous walk has filled the TLB with 368 // a match, we just want to get rid of the walk. The latter could happen 369 // when there are multiple outstanding misses to a single page and a 370 // previous request has been successfully translated. 371 if (!currState->transState->squashed() && !te) { 372 // We've got a valid request, lets process it 373 pending = true; 374 pendingQueue.pop_front(); 375 // Keep currState in case one of the processWalk... calls NULLs it 376 WalkerState *curr_state_copy = currState; 377 Fault f; 378 if (currState->aarch64) 379 f = processWalkAArch64(); 380 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2) 381 f = processWalkLPAE(); 382 else 383 f = processWalk(); 384 385 if (f != NoFault) { 386 curr_state_copy->transState->finish(f, curr_state_copy->req, 387 curr_state_copy->tc, curr_state_copy->mode); 388 389 delete curr_state_copy; 390 } 391 return; 392 } 393 394 395 // If the instruction that we were translating for has been 396 // squashed we shouldn't bother. 397 unsigned num_squashed = 0; 398 ThreadContext *tc = currState->tc; 399 while ((num_squashed < numSquashable) && currState && 400 (currState->transState->squashed() || te)) { 401 pendingQueue.pop_front(); 402 num_squashed++; 403 statSquashedBefore++; 404 405 DPRINTF(TLB, "Squashing table walk for address %#x\n", 406 currState->vaddr_tainted); 407 408 if (currState->transState->squashed()) { 409 // finish the translation which will delete the translation object 410 currState->transState->finish( 411 std::make_shared<UnimpFault>("Squashed Inst"), 412 currState->req, currState->tc, currState->mode); 413 } else { 414 // translate the request now that we know it will work 415 statWalkServiceTime.sample(curTick() - currState->startTime); 416 tlb->translateTiming(currState->req, currState->tc, 417 currState->transState, currState->mode); 418 419 } 420 421 // delete the current request 422 delete currState; 423 424 // peak at the next one 425 if (pendingQueue.size()) { 426 currState = pendingQueue.front(); 427 te = tlb->lookup(currState->vaddr, currState->asid, 428 currState->vmid, currState->isHyp, currState->isSecure, true, 429 false, target_el); 430 } else { 431 // Terminate the loop, nothing more to do 432 currState = NULL; 433 } 434 } 435 pendingChange(); 436 437 // if we still have pending translations, schedule more work 438 nextWalk(tc); 439 currState = NULL; 440} 441 442Fault 443TableWalker::processWalk() 444{ 445 Addr ttbr = 0; 446 447 // If translation isn't enabled, we shouldn't be here 448 assert(currState->sctlr.m || isStage2); 449 450 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n", 451 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31, 452 32 - currState->ttbcr.n)); 453 454 statWalkWaitTime.sample(curTick() - currState->startTime); 455 456 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31, 457 32 - currState->ttbcr.n)) { 458 DPRINTF(TLB, " - Selecting TTBR0\n"); 459 // Check if table walk is allowed when Security Extensions are enabled 460 if (haveSecurity && currState->ttbcr.pd0) { 461 if (currState->isFetch) 462 return std::make_shared<PrefetchAbort>( 463 currState->vaddr_tainted, 464 ArmFault::TranslationLL + L1, 465 isStage2, 466 ArmFault::VmsaTran); 467 else 468 return std::make_shared<DataAbort>( 469 currState->vaddr_tainted, 470 TlbEntry::DomainType::NoAccess, currState->isWrite, 471 ArmFault::TranslationLL + L1, isStage2, 472 ArmFault::VmsaTran); 473 } 474 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 475 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 476 } else { 477 DPRINTF(TLB, " - Selecting TTBR1\n"); 478 // Check if table walk is allowed when Security Extensions are enabled 479 if (haveSecurity && currState->ttbcr.pd1) { 480 if (currState->isFetch) 481 return std::make_shared<PrefetchAbort>( 482 currState->vaddr_tainted, 483 ArmFault::TranslationLL + L1, 484 isStage2, 485 ArmFault::VmsaTran); 486 else 487 return std::make_shared<DataAbort>( 488 currState->vaddr_tainted, 489 TlbEntry::DomainType::NoAccess, currState->isWrite, 490 ArmFault::TranslationLL + L1, isStage2, 491 ArmFault::VmsaTran); 492 } 493 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 494 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 495 currState->ttbcr.n = 0; 496 } 497 498 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) | 499 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2); 500 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr, 501 currState->isSecure ? "s" : "ns"); 502 503 // Trickbox address check 504 Fault f; 505 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure, 506 currState->vaddr, sizeof(uint32_t), currState->isFetch, 507 currState->isWrite, TlbEntry::DomainType::NoAccess, L1); 508 if (f) { 509 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 510 if (currState->timing) { 511 pending = false; 512 nextWalk(currState->tc); 513 currState = NULL; 514 } else { 515 currState->tc = NULL; 516 currState->req = NULL; 517 } 518 return f; 519 } 520 521 Request::Flags flag = Request::PT_WALK; 522 if (currState->sctlr.c == 0) { 523 flag.set(Request::UNCACHEABLE); 524 } 525 526 bool delayed; 527 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data, 528 sizeof(uint32_t), flag, L1, &doL1DescEvent, 529 &TableWalker::doL1Descriptor); 530 if (!delayed) { 531 f = currState->fault; 532 } 533 534 return f; 535} 536 537Fault 538TableWalker::processWalkLPAE() 539{ 540 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr; 541 int tsz, n; 542 LookupLevel start_lookup_level = L1; 543 544 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n", 545 currState->vaddr_tainted, currState->ttbcr); 546 547 statWalkWaitTime.sample(curTick() - currState->startTime); 548 549 Request::Flags flag = Request::PT_WALK; 550 if (currState->isSecure) 551 flag.set(Request::SECURE); 552 553 // work out which base address register to use, if in hyp mode we always 554 // use HTTBR 555 if (isStage2) { 556 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n"); 557 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR); 558 tsz = sext<4>(currState->vtcr.t0sz); 559 start_lookup_level = currState->vtcr.sl0 ? L1 : L2; 560 } else if (currState->isHyp) { 561 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n"); 562 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR); 563 tsz = currState->htcr.t0sz; 564 } else { 565 assert(_haveLPAE && currState->ttbcr.eae); 566 567 // Determine boundaries of TTBR0/1 regions 568 if (currState->ttbcr.t0sz) 569 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1; 570 else if (currState->ttbcr.t1sz) 571 ttbr0_max = (1ULL << 32) - 572 (1ULL << (32 - currState->ttbcr.t1sz)) - 1; 573 else 574 ttbr0_max = (1ULL << 32) - 1; 575 if (currState->ttbcr.t1sz) 576 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)); 577 else 578 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz)); 579 580 // The following code snippet selects the appropriate translation table base 581 // address (TTBR0 or TTBR1) and the appropriate starting lookup level 582 // depending on the address range supported by the translation table (ARM 583 // ARM issue C B3.6.4) 584 if (currState->vaddr <= ttbr0_max) { 585 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n"); 586 // Check if table walk is allowed 587 if (currState->ttbcr.epd0) { 588 if (currState->isFetch) 589 return std::make_shared<PrefetchAbort>( 590 currState->vaddr_tainted, 591 ArmFault::TranslationLL + L1, 592 isStage2, 593 ArmFault::LpaeTran); 594 else 595 return std::make_shared<DataAbort>( 596 currState->vaddr_tainted, 597 TlbEntry::DomainType::NoAccess, 598 currState->isWrite, 599 ArmFault::TranslationLL + L1, 600 isStage2, 601 ArmFault::LpaeTran); 602 } 603 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 604 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 605 tsz = currState->ttbcr.t0sz; 606 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB 607 start_lookup_level = L2; 608 } else if (currState->vaddr >= ttbr1_min) { 609 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n"); 610 // Check if table walk is allowed 611 if (currState->ttbcr.epd1) { 612 if (currState->isFetch) 613 return std::make_shared<PrefetchAbort>( 614 currState->vaddr_tainted, 615 ArmFault::TranslationLL + L1, 616 isStage2, 617 ArmFault::LpaeTran); 618 else 619 return std::make_shared<DataAbort>( 620 currState->vaddr_tainted, 621 TlbEntry::DomainType::NoAccess, 622 currState->isWrite, 623 ArmFault::TranslationLL + L1, 624 isStage2, 625 ArmFault::LpaeTran); 626 } 627 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 628 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 629 tsz = currState->ttbcr.t1sz; 630 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB 631 start_lookup_level = L2; 632 } else { 633 // Out of boundaries -> translation fault 634 if (currState->isFetch) 635 return std::make_shared<PrefetchAbort>( 636 currState->vaddr_tainted, 637 ArmFault::TranslationLL + L1, 638 isStage2, 639 ArmFault::LpaeTran); 640 else 641 return std::make_shared<DataAbort>( 642 currState->vaddr_tainted, 643 TlbEntry::DomainType::NoAccess, 644 currState->isWrite, ArmFault::TranslationLL + L1, 645 isStage2, ArmFault::LpaeTran); 646 } 647 648 } 649 650 // Perform lookup (ARM ARM issue C B3.6.6) 651 if (start_lookup_level == L1) { 652 n = 5 - tsz; 653 desc_addr = mbits(ttbr, 39, n) | 654 (bits(currState->vaddr, n + 26, 30) << 3); 655 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 656 desc_addr, currState->isSecure ? "s" : "ns"); 657 } else { 658 // Skip first-level lookup 659 n = (tsz >= 2 ? 14 - tsz : 12); 660 desc_addr = mbits(ttbr, 39, n) | 661 (bits(currState->vaddr, n + 17, 21) << 3); 662 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 663 desc_addr, currState->isSecure ? "s" : "ns"); 664 } 665 666 // Trickbox address check 667 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure, 668 currState->vaddr, sizeof(uint64_t), currState->isFetch, 669 currState->isWrite, TlbEntry::DomainType::NoAccess, 670 start_lookup_level); 671 if (f) { 672 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 673 if (currState->timing) { 674 pending = false; 675 nextWalk(currState->tc); 676 currState = NULL; 677 } else { 678 currState->tc = NULL; 679 currState->req = NULL; 680 } 681 return f; 682 } 683 684 if (currState->sctlr.c == 0) { 685 flag.set(Request::UNCACHEABLE); 686 } 687 688 if (currState->isSecure) 689 flag.set(Request::SECURE); 690 691 currState->longDesc.lookupLevel = start_lookup_level; 692 currState->longDesc.aarch64 = false; 693 currState->longDesc.grainSize = Grain4KB; 694 695 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent 696 : (Event *) &doL2LongDescEvent; 697 698 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, 699 sizeof(uint64_t), flag, start_lookup_level, 700 event, &TableWalker::doLongDescriptor); 701 if (!delayed) { 702 f = currState->fault; 703 } 704 705 return f; 706} 707 708unsigned 709TableWalker::adjustTableSizeAArch64(unsigned tsz) 710{ 711 if (tsz < 25) 712 return 25; 713 if (tsz > 48) 714 return 48; 715 return tsz; 716} 717 718bool 719TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange) 720{ 721 return (currPhysAddrRange != MaxPhysAddrRange && 722 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange)); 723} 724 725Fault 726TableWalker::processWalkAArch64() 727{ 728 assert(currState->aarch64); 729 730 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n", 731 currState->vaddr_tainted, currState->tcr); 732 733 static const GrainSize GrainMapDefault[] = 734 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain }; 735 static const GrainSize GrainMap_EL1_tg1[] = 736 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB }; 737 738 statWalkWaitTime.sample(curTick() - currState->startTime); 739 740 // Determine TTBR, table size, granule size and phys. address range 741 Addr ttbr = 0; 742 int tsz = 0, ps = 0; 743 GrainSize tg = Grain4KB; // grain size computed from tg* field 744 bool fault = false; 745 switch (currState->el) { 746 case EL0: 747 case EL1: 748 switch (bits(currState->vaddr, 63,48)) { 749 case 0: 750 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 751 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1); 752 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 753 tg = GrainMapDefault[currState->tcr.tg0]; 754 if (bits(currState->vaddr, 63, tsz) != 0x0 || 755 currState->tcr.epd0) 756 fault = true; 757 break; 758 case 0xffff: 759 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); 760 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1); 761 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); 762 tg = GrainMap_EL1_tg1[currState->tcr.tg1]; 763 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || 764 currState->tcr.epd1) 765 fault = true; 766 break; 767 default: 768 // top two bytes must be all 0s or all 1s, else invalid addr 769 fault = true; 770 } 771 ps = currState->tcr.ips; 772 break; 773 case EL2: 774 case EL3: 775 switch(bits(currState->vaddr, 63,48)) { 776 case 0: 777 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 778 if (currState->el == EL2) 779 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2); 780 else 781 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3); 782 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 783 tg = GrainMapDefault[currState->tcr.tg0]; 784 break; 785 default: 786 // invalid addr if top two bytes are not all 0s 787 fault = true; 788 } 789 ps = currState->tcr.ips; 790 break; 791 } 792 793 if (fault) { 794 Fault f; 795 if (currState->isFetch) 796 f = std::make_shared<PrefetchAbort>( 797 currState->vaddr_tainted, 798 ArmFault::TranslationLL + L0, isStage2, 799 ArmFault::LpaeTran); 800 else 801 f = std::make_shared<DataAbort>( 802 currState->vaddr_tainted, 803 TlbEntry::DomainType::NoAccess, 804 currState->isWrite, 805 ArmFault::TranslationLL + L0, 806 isStage2, ArmFault::LpaeTran); 807 808 if (currState->timing) { 809 pending = false; 810 nextWalk(currState->tc); 811 currState = NULL; 812 } else { 813 currState->tc = NULL; 814 currState->req = NULL; 815 } 816 return f; 817 818 } 819 820 if (tg == ReservedGrain) { 821 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION " 822 "DEFINED behavior takes this to mean 4KB granules\n"); 823 tg = Grain4KB; 824 } 825 826 int stride = tg - 3; 827 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS; 828 829 // Determine starting lookup level 830 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library 831 // in ARM DDI 0487A. These table values correspond to the cascading tests 832 // to compute the lookup level and are of the form 833 // (grain_size + N*stride), for N = {1, 2, 3}. 834 // A value of 64 will never succeed and a value of 0 will always succeed. 835 { 836 struct GrainMap { 837 GrainSize grain_size; 838 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS]; 839 }; 840 static const GrainMap GM[] = { 841 { Grain4KB, { 39, 30, 0, 0 } }, 842 { Grain16KB, { 47, 36, 25, 0 } }, 843 { Grain64KB, { 64, 42, 29, 0 } } 844 }; 845 846 const unsigned *lookup = NULL; // points to a lookup_level_cutoff 847 848 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[] 849 if (tg == GM[i].grain_size) { 850 lookup = GM[i].lookup_level_cutoff; 851 break; 852 } 853 } 854 assert(lookup); 855 856 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) { 857 if (tsz > lookup[L]) { 858 start_lookup_level = (LookupLevel) L; 859 break; 860 } 861 } 862 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, 863 "Table walker couldn't find lookup level\n"); 864 } 865 866 // Determine table base address 867 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg; 868 Addr base_addr = mbits(ttbr, 47, base_addr_lo); 869 870 // Determine physical address size and raise an Address Size Fault if 871 // necessary 872 int pa_range = decodePhysAddrRange64(ps); 873 // Clamp to lower limit 874 if (pa_range > physAddrRange) 875 currState->physAddrRange = physAddrRange; 876 else 877 currState->physAddrRange = pa_range; 878 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) { 879 DPRINTF(TLB, "Address size fault before any lookup\n"); 880 Fault f; 881 if (currState->isFetch) 882 f = std::make_shared<PrefetchAbort>( 883 currState->vaddr_tainted, 884 ArmFault::AddressSizeLL + start_lookup_level, 885 isStage2, 886 ArmFault::LpaeTran); 887 else 888 f = std::make_shared<DataAbort>( 889 currState->vaddr_tainted, 890 TlbEntry::DomainType::NoAccess, 891 currState->isWrite, 892 ArmFault::AddressSizeLL + start_lookup_level, 893 isStage2, 894 ArmFault::LpaeTran); 895 896 897 if (currState->timing) { 898 pending = false; 899 nextWalk(currState->tc); 900 currState = NULL; 901 } else { 902 currState->tc = NULL; 903 currState->req = NULL; 904 } 905 return f; 906 907 } 908 909 // Determine descriptor address 910 Addr desc_addr = base_addr | 911 (bits(currState->vaddr, tsz - 1, 912 stride * (3 - start_lookup_level) + tg) << 3); 913 914 // Trickbox address check 915 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure, 916 currState->vaddr, sizeof(uint64_t), currState->isFetch, 917 currState->isWrite, TlbEntry::DomainType::NoAccess, 918 start_lookup_level); 919 if (f) { 920 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 921 if (currState->timing) { 922 pending = false; 923 nextWalk(currState->tc); 924 currState = NULL; 925 } else { 926 currState->tc = NULL; 927 currState->req = NULL; 928 } 929 return f; 930 } 931 932 Request::Flags flag = Request::PT_WALK; 933 if (currState->sctlr.c == 0) { 934 flag.set(Request::UNCACHEABLE); 935 } 936 937 currState->longDesc.lookupLevel = start_lookup_level; 938 currState->longDesc.aarch64 = true; 939 currState->longDesc.grainSize = tg; 940 941 if (currState->timing) { 942 Event *event; 943 switch (start_lookup_level) { 944 case L0: 945 event = (Event *) &doL0LongDescEvent; 946 break; 947 case L1: 948 event = (Event *) &doL1LongDescEvent; 949 break; 950 case L2: 951 event = (Event *) &doL2LongDescEvent; 952 break; 953 case L3: 954 event = (Event *) &doL3LongDescEvent; 955 break; 956 default: 957 panic("Invalid table lookup level"); 958 break; 959 } 960 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 961 event, (uint8_t*) &currState->longDesc.data, 962 currState->tc->getCpuPtr()->clockPeriod(), flag); 963 DPRINTF(TLBVerbose, 964 "Adding to walker fifo: queue size before adding: %d\n", 965 stateQueues[start_lookup_level].size()); 966 stateQueues[start_lookup_level].push_back(currState); 967 currState = NULL; 968 } else if (!currState->functional) { 969 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 970 NULL, (uint8_t*) &currState->longDesc.data, 971 currState->tc->getCpuPtr()->clockPeriod(), flag); 972 doLongDescriptor(); 973 f = currState->fault; 974 } else { 975 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag, 976 masterId); 977 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 978 pkt->dataStatic((uint8_t*) &currState->longDesc.data); 979 port->sendFunctional(pkt); 980 doLongDescriptor(); 981 delete req; 982 delete pkt; 983 f = currState->fault; 984 } 985 986 return f; 987} 988 989void 990TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, 991 uint8_t texcb, bool s) 992{ 993 // Note: tc and sctlr local variables are hiding tc and sctrl class 994 // variables 995 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s); 996 te.shareable = false; // default value 997 te.nonCacheable = false; 998 te.outerShareable = false; 999 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) { 1000 switch(texcb) { 1001 case 0: // Stongly-ordered 1002 te.nonCacheable = true; 1003 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1004 te.shareable = true; 1005 te.innerAttrs = 1; 1006 te.outerAttrs = 0; 1007 break; 1008 case 1: // Shareable Device 1009 te.nonCacheable = true; 1010 te.mtype = TlbEntry::MemoryType::Device; 1011 te.shareable = true; 1012 te.innerAttrs = 3; 1013 te.outerAttrs = 0; 1014 break; 1015 case 2: // Outer and Inner Write-Through, no Write-Allocate 1016 te.mtype = TlbEntry::MemoryType::Normal; 1017 te.shareable = s; 1018 te.innerAttrs = 6; 1019 te.outerAttrs = bits(texcb, 1, 0); 1020 break; 1021 case 3: // Outer and Inner Write-Back, no Write-Allocate 1022 te.mtype = TlbEntry::MemoryType::Normal; 1023 te.shareable = s; 1024 te.innerAttrs = 7; 1025 te.outerAttrs = bits(texcb, 1, 0); 1026 break; 1027 case 4: // Outer and Inner Non-cacheable 1028 te.nonCacheable = true; 1029 te.mtype = TlbEntry::MemoryType::Normal; 1030 te.shareable = s; 1031 te.innerAttrs = 0; 1032 te.outerAttrs = bits(texcb, 1, 0); 1033 break; 1034 case 5: // Reserved 1035 panic("Reserved texcb value!\n"); 1036 break; 1037 case 6: // Implementation Defined 1038 panic("Implementation-defined texcb value!\n"); 1039 break; 1040 case 7: // Outer and Inner Write-Back, Write-Allocate 1041 te.mtype = TlbEntry::MemoryType::Normal; 1042 te.shareable = s; 1043 te.innerAttrs = 5; 1044 te.outerAttrs = 1; 1045 break; 1046 case 8: // Non-shareable Device 1047 te.nonCacheable = true; 1048 te.mtype = TlbEntry::MemoryType::Device; 1049 te.shareable = false; 1050 te.innerAttrs = 3; 1051 te.outerAttrs = 0; 1052 break; 1053 case 9 ... 15: // Reserved 1054 panic("Reserved texcb value!\n"); 1055 break; 1056 case 16 ... 31: // Cacheable Memory 1057 te.mtype = TlbEntry::MemoryType::Normal; 1058 te.shareable = s; 1059 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0) 1060 te.nonCacheable = true; 1061 te.innerAttrs = bits(texcb, 1, 0); 1062 te.outerAttrs = bits(texcb, 3, 2); 1063 break; 1064 default: 1065 panic("More than 32 states for 5 bits?\n"); 1066 } 1067 } else { 1068 assert(tc); 1069 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR, 1070 currState->tc, !currState->isSecure)); 1071 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR, 1072 currState->tc, !currState->isSecure)); 1073 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr); 1074 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0; 1075 switch(bits(texcb, 2,0)) { 1076 case 0: 1077 curr_tr = prrr.tr0; 1078 curr_ir = nmrr.ir0; 1079 curr_or = nmrr.or0; 1080 te.outerShareable = (prrr.nos0 == 0); 1081 break; 1082 case 1: 1083 curr_tr = prrr.tr1; 1084 curr_ir = nmrr.ir1; 1085 curr_or = nmrr.or1; 1086 te.outerShareable = (prrr.nos1 == 0); 1087 break; 1088 case 2: 1089 curr_tr = prrr.tr2; 1090 curr_ir = nmrr.ir2; 1091 curr_or = nmrr.or2; 1092 te.outerShareable = (prrr.nos2 == 0); 1093 break; 1094 case 3: 1095 curr_tr = prrr.tr3; 1096 curr_ir = nmrr.ir3; 1097 curr_or = nmrr.or3; 1098 te.outerShareable = (prrr.nos3 == 0); 1099 break; 1100 case 4: 1101 curr_tr = prrr.tr4; 1102 curr_ir = nmrr.ir4; 1103 curr_or = nmrr.or4; 1104 te.outerShareable = (prrr.nos4 == 0); 1105 break; 1106 case 5: 1107 curr_tr = prrr.tr5; 1108 curr_ir = nmrr.ir5; 1109 curr_or = nmrr.or5; 1110 te.outerShareable = (prrr.nos5 == 0); 1111 break; 1112 case 6: 1113 panic("Imp defined type\n"); 1114 case 7: 1115 curr_tr = prrr.tr7; 1116 curr_ir = nmrr.ir7; 1117 curr_or = nmrr.or7; 1118 te.outerShareable = (prrr.nos7 == 0); 1119 break; 1120 } 1121 1122 switch(curr_tr) { 1123 case 0: 1124 DPRINTF(TLBVerbose, "StronglyOrdered\n"); 1125 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1126 te.nonCacheable = true; 1127 te.innerAttrs = 1; 1128 te.outerAttrs = 0; 1129 te.shareable = true; 1130 break; 1131 case 1: 1132 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n", 1133 prrr.ds1, prrr.ds0, s); 1134 te.mtype = TlbEntry::MemoryType::Device; 1135 te.nonCacheable = true; 1136 te.innerAttrs = 3; 1137 te.outerAttrs = 0; 1138 if (prrr.ds1 && s) 1139 te.shareable = true; 1140 if (prrr.ds0 && !s) 1141 te.shareable = true; 1142 break; 1143 case 2: 1144 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n", 1145 prrr.ns1, prrr.ns0, s); 1146 te.mtype = TlbEntry::MemoryType::Normal; 1147 if (prrr.ns1 && s) 1148 te.shareable = true; 1149 if (prrr.ns0 && !s) 1150 te.shareable = true; 1151 break; 1152 case 3: 1153 panic("Reserved type"); 1154 } 1155 1156 if (te.mtype == TlbEntry::MemoryType::Normal){ 1157 switch(curr_ir) { 1158 case 0: 1159 te.nonCacheable = true; 1160 te.innerAttrs = 0; 1161 break; 1162 case 1: 1163 te.innerAttrs = 5; 1164 break; 1165 case 2: 1166 te.innerAttrs = 6; 1167 break; 1168 case 3: 1169 te.innerAttrs = 7; 1170 break; 1171 } 1172 1173 switch(curr_or) { 1174 case 0: 1175 te.nonCacheable = true; 1176 te.outerAttrs = 0; 1177 break; 1178 case 1: 1179 te.outerAttrs = 1; 1180 break; 1181 case 2: 1182 te.outerAttrs = 2; 1183 break; 1184 case 3: 1185 te.outerAttrs = 3; 1186 break; 1187 } 1188 } 1189 } 1190 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, " 1191 "outerAttrs: %d\n", 1192 te.shareable, te.innerAttrs, te.outerAttrs); 1193 te.setAttributes(false); 1194} 1195 1196void 1197TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, 1198 LongDescriptor &lDescriptor) 1199{ 1200 assert(_haveLPAE); 1201 1202 uint8_t attr; 1203 uint8_t sh = lDescriptor.sh(); 1204 // Different format and source of attributes if this is a stage 2 1205 // translation 1206 if (isStage2) { 1207 attr = lDescriptor.memAttr(); 1208 uint8_t attr_3_2 = (attr >> 2) & 0x3; 1209 uint8_t attr_1_0 = attr & 0x3; 1210 1211 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh); 1212 1213 if (attr_3_2 == 0) { 1214 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered 1215 : TlbEntry::MemoryType::Device; 1216 te.outerAttrs = 0; 1217 te.innerAttrs = attr_1_0 == 0 ? 1 : 3; 1218 te.nonCacheable = true; 1219 } else { 1220 te.mtype = TlbEntry::MemoryType::Normal; 1221 te.outerAttrs = attr_3_2 == 1 ? 0 : 1222 attr_3_2 == 2 ? 2 : 1; 1223 te.innerAttrs = attr_1_0 == 1 ? 0 : 1224 attr_1_0 == 2 ? 6 : 5; 1225 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1); 1226 } 1227 } else { 1228 uint8_t attrIndx = lDescriptor.attrIndx(); 1229 1230 // LPAE always uses remapping of memory attributes, irrespective of the 1231 // value of SCTLR.TRE 1232 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0; 1233 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc, 1234 !currState->isSecure); 1235 uint32_t mair = currState->tc->readMiscReg(reg_as_int); 1236 attr = (mair >> (8 * (attrIndx % 4))) & 0xff; 1237 uint8_t attr_7_4 = bits(attr, 7, 4); 1238 uint8_t attr_3_0 = bits(attr, 3, 0); 1239 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr); 1240 1241 // Note: the memory subsystem only cares about the 'cacheable' memory 1242 // attribute. The other attributes are only used to fill the PAR register 1243 // accordingly to provide the illusion of full support 1244 te.nonCacheable = false; 1245 1246 switch (attr_7_4) { 1247 case 0x0: 1248 // Strongly-ordered or Device memory 1249 if (attr_3_0 == 0x0) 1250 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1251 else if (attr_3_0 == 0x4) 1252 te.mtype = TlbEntry::MemoryType::Device; 1253 else 1254 panic("Unpredictable behavior\n"); 1255 te.nonCacheable = true; 1256 te.outerAttrs = 0; 1257 break; 1258 case 0x4: 1259 // Normal memory, Outer Non-cacheable 1260 te.mtype = TlbEntry::MemoryType::Normal; 1261 te.outerAttrs = 0; 1262 if (attr_3_0 == 0x4) 1263 // Inner Non-cacheable 1264 te.nonCacheable = true; 1265 else if (attr_3_0 < 0x8) 1266 panic("Unpredictable behavior\n"); 1267 break; 1268 case 0x8: 1269 case 0x9: 1270 case 0xa: 1271 case 0xb: 1272 case 0xc: 1273 case 0xd: 1274 case 0xe: 1275 case 0xf: 1276 if (attr_7_4 & 0x4) { 1277 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3; 1278 } else { 1279 te.outerAttrs = 0x2; 1280 } 1281 // Normal memory, Outer Cacheable 1282 te.mtype = TlbEntry::MemoryType::Normal; 1283 if (attr_3_0 != 0x4 && attr_3_0 < 0x8) 1284 panic("Unpredictable behavior\n"); 1285 break; 1286 default: 1287 panic("Unpredictable behavior\n"); 1288 break; 1289 } 1290 1291 switch (attr_3_0) { 1292 case 0x0: 1293 te.innerAttrs = 0x1; 1294 break; 1295 case 0x4: 1296 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0; 1297 break; 1298 case 0x8: 1299 case 0x9: 1300 case 0xA: 1301 case 0xB: 1302 te.innerAttrs = 6; 1303 break; 1304 case 0xC: 1305 case 0xD: 1306 case 0xE: 1307 case 0xF: 1308 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7; 1309 break; 1310 default: 1311 panic("Unpredictable behavior\n"); 1312 break; 1313 } 1314 } 1315 1316 te.outerShareable = sh == 2; 1317 te.shareable = (sh & 0x2) ? true : false; 1318 te.setAttributes(true); 1319 te.attributes |= (uint64_t) attr << 56; 1320} 1321 1322void 1323TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx, 1324 uint8_t sh) 1325{ 1326 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh); 1327 1328 // Select MAIR 1329 uint64_t mair; 1330 switch (currState->el) { 1331 case EL0: 1332 case EL1: 1333 mair = tc->readMiscReg(MISCREG_MAIR_EL1); 1334 break; 1335 case EL2: 1336 mair = tc->readMiscReg(MISCREG_MAIR_EL2); 1337 break; 1338 case EL3: 1339 mair = tc->readMiscReg(MISCREG_MAIR_EL3); 1340 break; 1341 default: 1342 panic("Invalid exception level"); 1343 break; 1344 } 1345 1346 // Select attributes 1347 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx); 1348 uint8_t attr_lo = bits(attr, 3, 0); 1349 uint8_t attr_hi = bits(attr, 7, 4); 1350 1351 // Memory type 1352 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; 1353 1354 // Cacheability 1355 te.nonCacheable = false; 1356 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory 1357 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable 1358 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable 1359 te.nonCacheable = true; 1360 } 1361 1362 te.shareable = sh == 2; 1363 te.outerShareable = (sh & 0x2) ? true : false; 1364 // Attributes formatted according to the 64-bit PAR 1365 te.attributes = ((uint64_t) attr << 56) | 1366 (1 << 11) | // LPAE bit 1367 (te.ns << 9) | // NS bit 1368 (sh << 7); 1369} 1370 1371void 1372TableWalker::doL1Descriptor() 1373{ 1374 if (currState->fault != NoFault) { 1375 return; 1376 } 1377 1378 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n", 1379 currState->vaddr_tainted, currState->l1Desc.data); 1380 TlbEntry te; 1381 1382 switch (currState->l1Desc.type()) { 1383 case L1Descriptor::Ignore: 1384 case L1Descriptor::Reserved: 1385 if (!currState->timing) { 1386 currState->tc = NULL; 1387 currState->req = NULL; 1388 } 1389 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n"); 1390 if (currState->isFetch) 1391 currState->fault = 1392 std::make_shared<PrefetchAbort>( 1393 currState->vaddr_tainted, 1394 ArmFault::TranslationLL + L1, 1395 isStage2, 1396 ArmFault::VmsaTran); 1397 else 1398 currState->fault = 1399 std::make_shared<DataAbort>( 1400 currState->vaddr_tainted, 1401 TlbEntry::DomainType::NoAccess, 1402 currState->isWrite, 1403 ArmFault::TranslationLL + L1, isStage2, 1404 ArmFault::VmsaTran); 1405 return; 1406 case L1Descriptor::Section: 1407 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) { 1408 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is 1409 * enabled if set, do l1.Desc.setAp0() instead of generating 1410 * AccessFlag0 1411 */ 1412 1413 currState->fault = std::make_shared<DataAbort>( 1414 currState->vaddr_tainted, 1415 currState->l1Desc.domain(), 1416 currState->isWrite, 1417 ArmFault::AccessFlagLL + L1, 1418 isStage2, 1419 ArmFault::VmsaTran); 1420 } 1421 if (currState->l1Desc.supersection()) { 1422 panic("Haven't implemented supersections\n"); 1423 } 1424 insertTableEntry(currState->l1Desc, false); 1425 return; 1426 case L1Descriptor::PageTable: 1427 { 1428 Addr l2desc_addr; 1429 l2desc_addr = currState->l1Desc.l2Addr() | 1430 (bits(currState->vaddr, 19, 12) << 2); 1431 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n", 1432 l2desc_addr, currState->isSecure ? "s" : "ns"); 1433 1434 // Trickbox address check 1435 currState->fault = tlb->walkTrickBoxCheck( 1436 l2desc_addr, currState->isSecure, currState->vaddr, 1437 sizeof(uint32_t), currState->isFetch, currState->isWrite, 1438 currState->l1Desc.domain(), L2); 1439 1440 if (currState->fault) { 1441 if (!currState->timing) { 1442 currState->tc = NULL; 1443 currState->req = NULL; 1444 } 1445 return; 1446 } 1447 1448 Request::Flags flag = Request::PT_WALK; 1449 if (currState->isSecure) 1450 flag.set(Request::SECURE); 1451 1452 bool delayed; 1453 delayed = fetchDescriptor(l2desc_addr, 1454 (uint8_t*)&currState->l2Desc.data, 1455 sizeof(uint32_t), flag, -1, &doL2DescEvent, 1456 &TableWalker::doL2Descriptor); 1457 if (delayed) { 1458 currState->delayed = true; 1459 } 1460 1461 return; 1462 } 1463 default: 1464 panic("A new type in a 2 bit field?\n"); 1465 } 1466} 1467 1468void 1469TableWalker::doLongDescriptor() 1470{ 1471 if (currState->fault != NoFault) { 1472 return; 1473 } 1474 1475 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n", 1476 currState->longDesc.lookupLevel, currState->vaddr_tainted, 1477 currState->longDesc.data, 1478 currState->aarch64 ? "AArch64" : "long-desc."); 1479 1480 if ((currState->longDesc.type() == LongDescriptor::Block) || 1481 (currState->longDesc.type() == LongDescriptor::Page)) { 1482 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, " 1483 "xn: %d, ap: %d, af: %d, type: %d\n", 1484 currState->longDesc.lookupLevel, 1485 currState->longDesc.data, 1486 currState->longDesc.pxn(), 1487 currState->longDesc.xn(), 1488 currState->longDesc.ap(), 1489 currState->longDesc.af(), 1490 currState->longDesc.type()); 1491 } else { 1492 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n", 1493 currState->longDesc.lookupLevel, 1494 currState->longDesc.data, 1495 currState->longDesc.type()); 1496 } 1497 1498 TlbEntry te; 1499 1500 switch (currState->longDesc.type()) { 1501 case LongDescriptor::Invalid: 1502 if (!currState->timing) { 1503 currState->tc = NULL; 1504 currState->req = NULL; 1505 } 1506 1507 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n", 1508 currState->longDesc.lookupLevel, 1509 ArmFault::TranslationLL + currState->longDesc.lookupLevel); 1510 if (currState->isFetch) 1511 currState->fault = std::make_shared<PrefetchAbort>( 1512 currState->vaddr_tainted, 1513 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1514 isStage2, 1515 ArmFault::LpaeTran); 1516 else 1517 currState->fault = std::make_shared<DataAbort>( 1518 currState->vaddr_tainted, 1519 TlbEntry::DomainType::NoAccess, 1520 currState->isWrite, 1521 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1522 isStage2, 1523 ArmFault::LpaeTran); 1524 return; 1525 case LongDescriptor::Block: 1526 case LongDescriptor::Page: 1527 { 1528 bool fault = false; 1529 bool aff = false; 1530 // Check for address size fault 1531 if (checkAddrSizeFaultAArch64( 1532 mbits(currState->longDesc.data, MaxPhysAddrRange - 1, 1533 currState->longDesc.offsetBits()), 1534 currState->physAddrRange)) { 1535 fault = true; 1536 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1537 currState->longDesc.lookupLevel); 1538 // Check for access fault 1539 } else if (currState->longDesc.af() == 0) { 1540 fault = true; 1541 DPRINTF(TLB, "L%d descriptor causing Access Fault\n", 1542 currState->longDesc.lookupLevel); 1543 aff = true; 1544 } 1545 if (fault) { 1546 if (currState->isFetch) 1547 currState->fault = std::make_shared<PrefetchAbort>( 1548 currState->vaddr_tainted, 1549 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1550 currState->longDesc.lookupLevel, 1551 isStage2, 1552 ArmFault::LpaeTran); 1553 else 1554 currState->fault = std::make_shared<DataAbort>( 1555 currState->vaddr_tainted, 1556 TlbEntry::DomainType::NoAccess, currState->isWrite, 1557 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1558 currState->longDesc.lookupLevel, 1559 isStage2, 1560 ArmFault::LpaeTran); 1561 } else { 1562 insertTableEntry(currState->longDesc, true); 1563 } 1564 } 1565 return; 1566 case LongDescriptor::Table: 1567 { 1568 // Set hierarchical permission flags 1569 currState->secureLookup = currState->secureLookup && 1570 currState->longDesc.secureTable(); 1571 currState->rwTable = currState->rwTable && 1572 currState->longDesc.rwTable(); 1573 currState->userTable = currState->userTable && 1574 currState->longDesc.userTable(); 1575 currState->xnTable = currState->xnTable || 1576 currState->longDesc.xnTable(); 1577 currState->pxnTable = currState->pxnTable || 1578 currState->longDesc.pxnTable(); 1579 1580 // Set up next level lookup 1581 Addr next_desc_addr = currState->longDesc.nextDescAddr( 1582 currState->vaddr); 1583 1584 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n", 1585 currState->longDesc.lookupLevel, 1586 currState->longDesc.lookupLevel + 1, 1587 next_desc_addr, 1588 currState->secureLookup ? "s" : "ns"); 1589 1590 // Check for address size fault 1591 if (currState->aarch64 && checkAddrSizeFaultAArch64( 1592 next_desc_addr, currState->physAddrRange)) { 1593 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1594 currState->longDesc.lookupLevel); 1595 if (currState->isFetch) 1596 currState->fault = std::make_shared<PrefetchAbort>( 1597 currState->vaddr_tainted, 1598 ArmFault::AddressSizeLL 1599 + currState->longDesc.lookupLevel, 1600 isStage2, 1601 ArmFault::LpaeTran); 1602 else 1603 currState->fault = std::make_shared<DataAbort>( 1604 currState->vaddr_tainted, 1605 TlbEntry::DomainType::NoAccess, currState->isWrite, 1606 ArmFault::AddressSizeLL 1607 + currState->longDesc.lookupLevel, 1608 isStage2, 1609 ArmFault::LpaeTran); 1610 return; 1611 } 1612 1613 // Trickbox address check 1614 currState->fault = tlb->walkTrickBoxCheck( 1615 next_desc_addr, currState->vaddr, 1616 currState->vaddr, sizeof(uint64_t), 1617 currState->isFetch, currState->isWrite, 1618 TlbEntry::DomainType::Client, 1619 toLookupLevel(currState->longDesc.lookupLevel +1)); 1620 1621 if (currState->fault) { 1622 if (!currState->timing) { 1623 currState->tc = NULL; 1624 currState->req = NULL; 1625 } 1626 return; 1627 } 1628 1629 Request::Flags flag = Request::PT_WALK; 1630 if (currState->secureLookup) 1631 flag.set(Request::SECURE); 1632 1633 currState->longDesc.lookupLevel = 1634 (LookupLevel) (currState->longDesc.lookupLevel + 1); 1635 Event *event = NULL; 1636 switch (currState->longDesc.lookupLevel) { 1637 case L1: 1638 assert(currState->aarch64); 1639 event = &doL1LongDescEvent; 1640 break; 1641 case L2: 1642 event = &doL2LongDescEvent; 1643 break; 1644 case L3: 1645 event = &doL3LongDescEvent; 1646 break; 1647 default: 1648 panic("Wrong lookup level in table walk\n"); 1649 break; 1650 } 1651 1652 bool delayed; 1653 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data, 1654 sizeof(uint64_t), flag, -1, event, 1655 &TableWalker::doLongDescriptor); 1656 if (delayed) { 1657 currState->delayed = true; 1658 } 1659 } 1660 return; 1661 default: 1662 panic("A new type in a 2 bit field?\n"); 1663 } 1664} 1665 1666void 1667TableWalker::doL2Descriptor() 1668{ 1669 if (currState->fault != NoFault) { 1670 return; 1671 } 1672 1673 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n", 1674 currState->vaddr_tainted, currState->l2Desc.data); 1675 TlbEntry te; 1676 1677 if (currState->l2Desc.invalid()) { 1678 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n"); 1679 if (!currState->timing) { 1680 currState->tc = NULL; 1681 currState->req = NULL; 1682 } 1683 if (currState->isFetch) 1684 currState->fault = std::make_shared<PrefetchAbort>( 1685 currState->vaddr_tainted, 1686 ArmFault::TranslationLL + L2, 1687 isStage2, 1688 ArmFault::VmsaTran); 1689 else 1690 currState->fault = std::make_shared<DataAbort>( 1691 currState->vaddr_tainted, currState->l1Desc.domain(), 1692 currState->isWrite, ArmFault::TranslationLL + L2, 1693 isStage2, 1694 ArmFault::VmsaTran); 1695 return; 1696 } 1697 1698 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) { 1699 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled 1700 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0 1701 */ 1702 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n", 1703 currState->sctlr.afe, currState->l2Desc.ap()); 1704 1705 currState->fault = std::make_shared<DataAbort>( 1706 currState->vaddr_tainted, 1707 TlbEntry::DomainType::NoAccess, currState->isWrite, 1708 ArmFault::AccessFlagLL + L2, isStage2, 1709 ArmFault::VmsaTran); 1710 } 1711 1712 insertTableEntry(currState->l2Desc, false); 1713} 1714 1715void 1716TableWalker::doL1DescriptorWrapper() 1717{ 1718 currState = stateQueues[L1].front(); 1719 currState->delayed = false; 1720 // if there's a stage2 translation object we don't need it any more 1721 if (currState->stage2Tran) { 1722 delete currState->stage2Tran; 1723 currState->stage2Tran = NULL; 1724 } 1725 1726 1727 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data); 1728 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data); 1729 1730 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted); 1731 doL1Descriptor(); 1732 1733 stateQueues[L1].pop_front(); 1734 // Check if fault was generated 1735 if (currState->fault != NoFault) { 1736 currState->transState->finish(currState->fault, currState->req, 1737 currState->tc, currState->mode); 1738 statWalksShortTerminatedAtLevel[0]++; 1739 1740 pending = false; 1741 nextWalk(currState->tc); 1742 1743 currState->req = NULL; 1744 currState->tc = NULL; 1745 currState->delayed = false; 1746 delete currState; 1747 } 1748 else if (!currState->delayed) { 1749 // delay is not set so there is no L2 to do 1750 // Don't finish the translation if a stage 2 look up is underway 1751 if (!currState->doingStage2) { 1752 statWalkServiceTime.sample(curTick() - currState->startTime); 1753 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1754 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1755 currState->transState, currState->mode); 1756 statWalksShortTerminatedAtLevel[0]++; 1757 } 1758 1759 pending = false; 1760 nextWalk(currState->tc); 1761 1762 currState->req = NULL; 1763 currState->tc = NULL; 1764 currState->delayed = false; 1765 delete currState; 1766 } else { 1767 // need to do L2 descriptor 1768 stateQueues[L2].push_back(currState); 1769 } 1770 currState = NULL; 1771} 1772 1773void 1774TableWalker::doL2DescriptorWrapper() 1775{ 1776 currState = stateQueues[L2].front(); 1777 assert(currState->delayed); 1778 // if there's a stage2 translation object we don't need it any more 1779 if (currState->stage2Tran) { 1780 delete currState->stage2Tran; 1781 currState->stage2Tran = NULL; 1782 } 1783 1784 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n", 1785 currState->vaddr_tainted); 1786 doL2Descriptor(); 1787 1788 // Check if fault was generated 1789 if (currState->fault != NoFault) { 1790 currState->transState->finish(currState->fault, currState->req, 1791 currState->tc, currState->mode); 1792 statWalksShortTerminatedAtLevel[1]++; 1793 } 1794 else { 1795 // Don't finish the translation if a stage 2 look up is underway 1796 if (!currState->doingStage2) { 1797 statWalkServiceTime.sample(curTick() - currState->startTime); 1798 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1799 currState->fault = tlb->translateTiming(currState->req, 1800 currState->tc, currState->transState, currState->mode); 1801 statWalksShortTerminatedAtLevel[1]++; 1802 } 1803 } 1804 1805 1806 stateQueues[L2].pop_front(); 1807 pending = false; 1808 nextWalk(currState->tc); 1809 1810 currState->req = NULL; 1811 currState->tc = NULL; 1812 currState->delayed = false; 1813 1814 delete currState; 1815 currState = NULL; 1816} 1817 1818void 1819TableWalker::doL0LongDescriptorWrapper() 1820{ 1821 doLongDescriptorWrapper(L0); 1822} 1823 1824void 1825TableWalker::doL1LongDescriptorWrapper() 1826{ 1827 doLongDescriptorWrapper(L1); 1828} 1829 1830void 1831TableWalker::doL2LongDescriptorWrapper() 1832{ 1833 doLongDescriptorWrapper(L2); 1834} 1835 1836void 1837TableWalker::doL3LongDescriptorWrapper() 1838{ 1839 doLongDescriptorWrapper(L3); 1840} 1841 1842void 1843TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) 1844{ 1845 currState = stateQueues[curr_lookup_level].front(); 1846 assert(curr_lookup_level == currState->longDesc.lookupLevel); 1847 currState->delayed = false; 1848 1849 // if there's a stage2 translation object we don't need it any more 1850 if (currState->stage2Tran) { 1851 delete currState->stage2Tran; 1852 currState->stage2Tran = NULL; 1853 } 1854 1855 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n", 1856 currState->vaddr_tainted); 1857 doLongDescriptor(); 1858 1859 stateQueues[curr_lookup_level].pop_front(); 1860 1861 if (currState->fault != NoFault) { 1862 // A fault was generated 1863 currState->transState->finish(currState->fault, currState->req, 1864 currState->tc, currState->mode); 1865 1866 pending = false; 1867 nextWalk(currState->tc); 1868 1869 currState->req = NULL; 1870 currState->tc = NULL; 1871 currState->delayed = false; 1872 delete currState; 1873 } else if (!currState->delayed) { 1874 // No additional lookups required 1875 // Don't finish the translation if a stage 2 look up is underway 1876 if (!currState->doingStage2) { 1877 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1878 statWalkServiceTime.sample(curTick() - currState->startTime); 1879 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1880 currState->transState, 1881 currState->mode); 1882 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; 1883 } 1884 1885 pending = false; 1886 nextWalk(currState->tc); 1887 1888 currState->req = NULL; 1889 currState->tc = NULL; 1890 currState->delayed = false; 1891 delete currState; 1892 } else { 1893 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1) 1894 panic("Max. number of lookups already reached in table walk\n"); 1895 // Need to perform additional lookups 1896 stateQueues[currState->longDesc.lookupLevel].push_back(currState); 1897 } 1898 currState = NULL; 1899} 1900 1901 1902void 1903TableWalker::nextWalk(ThreadContext *tc) 1904{ 1905 if (pendingQueue.size()) 1906 schedule(doProcessEvent, clockEdge(Cycles(1))); 1907 else 1908 completeDrain(); 1909} 1910 1911bool 1912TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, 1913 Request::Flags flags, int queueIndex, Event *event, 1914 void (TableWalker::*doDescriptor)()) 1915{ 1916 bool isTiming = currState->timing; 1917 1918 // do the requests for the page table descriptors have to go through the 1919 // second stage MMU 1920 if (currState->stage2Req) { 1921 Fault fault; 1922 flags = flags | TLB::MustBeOne; 1923 1924 if (isTiming) { 1925 Stage2MMU::Stage2Translation *tran = new 1926 Stage2MMU::Stage2Translation(*stage2Mmu, data, event, 1927 currState->vaddr); 1928 currState->stage2Tran = tran; 1929 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes, 1930 flags); 1931 fault = tran->fault; 1932 } else { 1933 fault = stage2Mmu->readDataUntimed(currState->tc, 1934 currState->vaddr, descAddr, data, numBytes, flags, 1935 currState->functional); 1936 } 1937 1938 if (fault != NoFault) { 1939 currState->fault = fault; 1940 } 1941 if (isTiming) { 1942 if (queueIndex >= 0) { 1943 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1944 stateQueues[queueIndex].size()); 1945 stateQueues[queueIndex].push_back(currState); 1946 currState = NULL; 1947 } 1948 } else { 1949 (this->*doDescriptor)(); 1950 } 1951 } else { 1952 if (isTiming) { 1953 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data, 1954 currState->tc->getCpuPtr()->clockPeriod(),flags); 1955 if (queueIndex >= 0) { 1956 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1957 stateQueues[queueIndex].size()); 1958 stateQueues[queueIndex].push_back(currState); 1959 currState = NULL; 1960 } 1961 } else if (!currState->functional) { 1962 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data, 1963 currState->tc->getCpuPtr()->clockPeriod(), flags); 1964 (this->*doDescriptor)(); 1965 } else { 1966 RequestPtr req = new Request(descAddr, numBytes, flags, masterId); 1967 req->taskId(ContextSwitchTaskId::DMA); 1968 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 1969 pkt->dataStatic(data); 1970 port->sendFunctional(pkt); 1971 (this->*doDescriptor)(); 1972 delete req; 1973 delete pkt; 1974 } 1975 } 1976 return (isTiming); 1977} 1978 1979void 1980TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor) 1981{ 1982 TlbEntry te; 1983 1984 // Create and fill a new page table entry 1985 te.valid = true; 1986 te.longDescFormat = longDescriptor; 1987 te.isHyp = currState->isHyp; 1988 te.asid = currState->asid; 1989 te.vmid = currState->vmid; 1990 te.N = descriptor.offsetBits(); 1991 te.vpn = currState->vaddr >> te.N; 1992 te.size = (1<<te.N) - 1; 1993 te.pfn = descriptor.pfn(); 1994 te.domain = descriptor.domain(); 1995 te.lookupLevel = descriptor.lookupLevel; 1996 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2; 1997 te.nstid = !currState->isSecure; 1998 te.xn = descriptor.xn(); 1999 if (currState->aarch64) 2000 te.el = currState->el; 2001 else 2002 te.el = 1; 2003 2004 statPageSizes[pageSizeNtoStatBin(te.N)]++; 2005 statRequestOrigin[COMPLETED][currState->isFetch]++; 2006 2007 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries 2008 // as global 2009 te.global = descriptor.global(currState) || isStage2; 2010 if (longDescriptor) { 2011 LongDescriptor lDescriptor = 2012 dynamic_cast<LongDescriptor &>(descriptor); 2013 2014 te.xn |= currState->xnTable; 2015 te.pxn = currState->pxnTable || lDescriptor.pxn(); 2016 if (isStage2) { 2017 // this is actually the HAP field, but its stored in the same bit 2018 // possitions as the AP field in a stage 1 translation. 2019 te.hap = lDescriptor.ap(); 2020 } else { 2021 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) | 2022 (currState->userTable && (descriptor.ap() & 0x1)); 2023 } 2024 if (currState->aarch64) 2025 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(), 2026 currState->longDesc.sh()); 2027 else 2028 memAttrsLPAE(currState->tc, te, lDescriptor); 2029 } else { 2030 te.ap = descriptor.ap(); 2031 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(), 2032 descriptor.shareable()); 2033 } 2034 2035 // Debug output 2036 DPRINTF(TLB, descriptor.dbgHeader().c_str()); 2037 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n", 2038 te.N, te.pfn, te.size, te.global, te.valid); 2039 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d " 2040 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn, 2041 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp, 2042 te.nonCacheable, te.ns); 2043 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n", 2044 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()), 2045 descriptor.getRawData()); 2046 2047 // Insert the entry into the TLB 2048 tlb->insert(currState->vaddr, te); 2049 if (!currState->timing) { 2050 currState->tc = NULL; 2051 currState->req = NULL; 2052 } 2053} 2054 2055ArmISA::TableWalker * 2056ArmTableWalkerParams::create() 2057{ 2058 return new ArmISA::TableWalker(this); 2059} 2060 2061LookupLevel 2062TableWalker::toLookupLevel(uint8_t lookup_level_as_int) 2063{ 2064 switch (lookup_level_as_int) { 2065 case L1: 2066 return L1; 2067 case L2: 2068 return L2; 2069 case L3: 2070 return L3; 2071 default: 2072 panic("Invalid lookup level conversion"); 2073 } 2074} 2075 2076/* this method keeps track of the table walker queue's residency, so 2077 * needs to be called whenever requests start and complete. */ 2078void 2079TableWalker::pendingChange() 2080{ 2081 unsigned n = pendingQueue.size(); 2082 if ((currState != NULL) && (currState != pendingQueue.front())) { 2083 ++n; 2084 } 2085 2086 if (n != pendingReqs) { 2087 Tick now = curTick(); 2088 statPendingWalks.sample(pendingReqs, now - pendingChangeTick); 2089 pendingReqs = n; 2090 pendingChangeTick = now; 2091 } 2092} 2093 2094uint8_t 2095TableWalker::pageSizeNtoStatBin(uint8_t N) 2096{ 2097 /* for statPageSizes */ 2098 switch(N) { 2099 case 12: return 0; // 4K 2100 case 14: return 1; // 16K (using 16K granule in v8-64) 2101 case 16: return 2; // 64K 2102 case 20: return 3; // 1M 2103 case 21: return 4; // 2M-LPAE 2104 case 24: return 5; // 16M 2105 case 25: return 6; // 32M (using 16K granule in v8-64) 2106 case 29: return 7; // 512M (using 64K granule in v8-64) 2107 case 30: return 8; // 1G-LPAE 2108 default: 2109 panic("unknown page size"); 2110 return 255; 2111 } 2112} 2113 2114void 2115TableWalker::regStats() 2116{ 2117 statWalks 2118 .name(name() + ".walks") 2119 .desc("Table walker walks requested") 2120 ; 2121 2122 statWalksShortDescriptor 2123 .name(name() + ".walksShort") 2124 .desc("Table walker walks initiated with short descriptors") 2125 .flags(Stats::nozero) 2126 ; 2127 2128 statWalksLongDescriptor 2129 .name(name() + ".walksLong") 2130 .desc("Table walker walks initiated with long descriptors") 2131 .flags(Stats::nozero) 2132 ; 2133 2134 statWalksShortTerminatedAtLevel 2135 .init(2) 2136 .name(name() + ".walksShortTerminationLevel") 2137 .desc("Level at which table walker walks " 2138 "with short descriptors terminate") 2139 .flags(Stats::nozero) 2140 ; 2141 statWalksShortTerminatedAtLevel.subname(0, "Level1"); 2142 statWalksShortTerminatedAtLevel.subname(1, "Level2"); 2143 2144 statWalksLongTerminatedAtLevel 2145 .init(4) 2146 .name(name() + ".walksLongTerminationLevel") 2147 .desc("Level at which table walker walks " 2148 "with long descriptors terminate") 2149 .flags(Stats::nozero) 2150 ; 2151 statWalksLongTerminatedAtLevel.subname(0, "Level0"); 2152 statWalksLongTerminatedAtLevel.subname(1, "Level1"); 2153 statWalksLongTerminatedAtLevel.subname(2, "Level2"); 2154 statWalksLongTerminatedAtLevel.subname(3, "Level3"); 2155 2156 statSquashedBefore 2157 .name(name() + ".walksSquashedBefore") 2158 .desc("Table walks squashed before starting") 2159 .flags(Stats::nozero) 2160 ; 2161 2162 statSquashedAfter 2163 .name(name() + ".walksSquashedAfter") 2164 .desc("Table walks squashed after completion") 2165 .flags(Stats::nozero) 2166 ; 2167 2168 statWalkWaitTime 2169 .init(16) 2170 .name(name() + ".walkWaitTime") 2171 .desc("Table walker wait (enqueue to first request) latency") 2172 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2173 ; 2174 2175 statWalkServiceTime 2176 .init(16) 2177 .name(name() + ".walkCompletionTime") 2178 .desc("Table walker service (enqueue to completion) latency") 2179 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2180 ; 2181 2182 statPendingWalks 2183 .init(16) 2184 .name(name() + ".walksPending") 2185 .desc("Table walker pending requests distribution") 2186 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan) 2187 ; 2188 2189 statPageSizes // see DDI 0487A D4-1661 2190 .init(9) 2191 .name(name() + ".walkPageSizes") 2192 .desc("Table walker page sizes translated") 2193 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero) 2194 ; 2195 statPageSizes.subname(0, "4K"); 2196 statPageSizes.subname(1, "16K"); 2197 statPageSizes.subname(2, "64K"); 2198 statPageSizes.subname(3, "1M"); 2199 statPageSizes.subname(4, "2M"); 2200 statPageSizes.subname(5, "16M"); 2201 statPageSizes.subname(6, "32M"); 2202 statPageSizes.subname(7, "512M"); 2203 statPageSizes.subname(8, "1G"); 2204 2205 statRequestOrigin 2206 .init(2,2) // Instruction/Data, requests/completed 2207 .name(name() + ".walkRequestOrigin") 2208 .desc("Table walker requests started/completed, data/inst") 2209 .flags(Stats::total) 2210 ; 2211 statRequestOrigin.subname(0,"Requested"); 2212 statRequestOrigin.subname(1,"Completed"); 2213 statRequestOrigin.ysubname(0,"Data"); 2214 statRequestOrigin.ysubname(1,"Inst"); 2215} 2216