1/* 2 * Copyright (c) 2010, 2012-2015 ARM Limited 3 * All rights reserved 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions are 16 * met: redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer; 18 * redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution; 21 * neither the name of the copyright holders nor the names of its 22 * contributors may be used to endorse or promote products derived from 23 * this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 * 37 * Authors: Ali Saidi 38 * Giacomo Gabrielli 39 */ 40#include "arch/arm/table_walker.hh" 41 42#include <memory> 43 44#include "arch/arm/faults.hh" 45#include "arch/arm/stage2_mmu.hh" 46#include "arch/arm/system.hh" 47#include "arch/arm/tlb.hh" 48#include "cpu/base.hh" 49#include "cpu/thread_context.hh" 50#include "debug/Checkpoint.hh" 51#include "debug/Drain.hh" 52#include "debug/TLB.hh" 53#include "debug/TLBVerbose.hh" 54#include "dev/dma_device.hh" 55#include "sim/system.hh" 56 57using namespace ArmISA; 58 59TableWalker::TableWalker(const Params *p) 60 : MemObject(p), 61 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId), 62 isStage2(p->is_stage2), tlb(NULL), 63 currState(NULL), pending(false), 64 numSquashable(p->num_squash_per_cycle), 65 pendingReqs(0), 66 pendingChangeTick(curTick()), 67 doL1DescEvent(this), doL2DescEvent(this), 68 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this), 69 doL3LongDescEvent(this), 70 doProcessEvent(this) 71{ 72 sctlr = 0; 73 74 // Cache system-level properties 75 if (FullSystem) { 76 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys); 77 assert(armSys); 78 haveSecurity = armSys->haveSecurity(); 79 _haveLPAE = armSys->haveLPAE(); 80 _haveVirtualization = armSys->haveVirtualization(); 81 physAddrRange = armSys->physAddrRange(); 82 _haveLargeAsid64 = armSys->haveLargeAsid64(); 83 } else { 84 haveSecurity = _haveLPAE = _haveVirtualization = false; 85 _haveLargeAsid64 = false; 86 physAddrRange = 32; 87 } 88 89} 90 91TableWalker::~TableWalker() 92{ 93 ; 94} 95 96void 97TableWalker::setMMU(Stage2MMU *m, MasterID master_id) 98{ 99 stage2Mmu = m; 100 port = &m->getPort(); 101 masterId = master_id; 102} 103 104void 105TableWalker::init() 106{ 107 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n"); 108 fatal_if(!port, "Table walker must have a valid port\n"); 109 fatal_if(!tlb, "Table walker must have a valid TLB\n"); 110} 111 112BaseMasterPort& 113TableWalker::getMasterPort(const std::string &if_name, PortID idx) 114{ 115 if (if_name == "port") { 116 if (!isStage2) { 117 return *port; 118 } else { 119 fatal("Cannot access table walker port through stage-two walker\n"); 120 } 121 } 122 return MemObject::getMasterPort(if_name, idx); 123} 124 125TableWalker::WalkerState::WalkerState() : 126 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr), 127 asid(0), vmid(0), isHyp(false), transState(nullptr), 128 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false), 129 secureLookup(false), rwTable(false), userTable(false), xnTable(false), 130 pxnTable(false), stage2Req(false), doingStage2(false), 131 stage2Tran(nullptr), timing(false), functional(false), 132 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), 133 delayed(false), tableWalker(nullptr) 134{ 135} 136 137void 138TableWalker::completeDrain() 139{ 140 if (drainState() == DrainState::Draining && 141 stateQueues[L1].empty() && stateQueues[L2].empty() && 142 pendingQueue.empty()) { 143 144 DPRINTF(Drain, "TableWalker done draining, processing drain event\n"); 145 signalDrainDone(); 146 } 147} 148 149DrainState 150TableWalker::drain() 151{ 152 bool state_queues_not_empty = false; 153 154 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) { 155 if (!stateQueues[i].empty()) { 156 state_queues_not_empty = true; 157 break; 158 } 159 } 160 161 if (state_queues_not_empty || pendingQueue.size()) { 162 DPRINTF(Drain, "TableWalker not drained\n"); 163 return DrainState::Draining; 164 } else { 165 DPRINTF(Drain, "TableWalker free, no need to drain\n"); 166 return DrainState::Drained; 167 } 168} 169 170void 171TableWalker::drainResume() 172{ 173 if (params()->sys->isTimingMode() && currState) { 174 delete currState; 175 currState = NULL; 176 pendingChange(); 177 } 178} 179 180Fault 181TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid, 182 uint8_t _vmid, bool _isHyp, TLB::Mode _mode, 183 TLB::Translation *_trans, bool _timing, bool _functional, 184 bool secure, TLB::ArmTranslationType tranType) 185{ 186 assert(!(_functional && _timing)); 187 ++statWalks; 188 189 WalkerState *savedCurrState = NULL; 190 191 if (!currState && !_functional) { 192 // For atomic mode, a new WalkerState instance should be only created 193 // once per TLB. For timing mode, a new instance is generated for every 194 // TLB miss. 195 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n"); 196 197 currState = new WalkerState(); 198 currState->tableWalker = this; 199 } else if (_functional) { 200 // If we are mixing functional mode with timing (or even 201 // atomic), we need to to be careful and clean up after 202 // ourselves to not risk getting into an inconsistent state. 203 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n"); 204 savedCurrState = currState; 205 currState = new WalkerState(); 206 currState->tableWalker = this; 207 } else if (_timing) { 208 // This is a translation that was completed and then faulted again 209 // because some underlying parameters that affect the translation 210 // changed out from under us (e.g. asid). It will either be a 211 // misprediction, in which case nothing will happen or we'll use 212 // this fault to re-execute the faulting instruction which should clean 213 // up everything. 214 if (currState->vaddr_tainted == _req->getVaddr()) { 215 ++statSquashedBefore; 216 return std::make_shared<ReExec>(); 217 } 218 } 219 pendingChange(); 220 221 currState->startTime = curTick(); 222 currState->tc = _tc; 223 currState->aarch64 = opModeIs64(currOpMode(_tc)); 224 currState->el = currEL(_tc); 225 currState->transState = _trans; 226 currState->req = _req; 227 currState->fault = NoFault; 228 currState->asid = _asid; 229 currState->vmid = _vmid; 230 currState->isHyp = _isHyp; 231 currState->timing = _timing; 232 currState->functional = _functional; 233 currState->mode = _mode; 234 currState->tranType = tranType; 235 currState->isSecure = secure; 236 currState->physAddrRange = physAddrRange; 237 238 /** @todo These should be cached or grabbed from cached copies in 239 the TLB, all these miscreg reads are expensive */ 240 currState->vaddr_tainted = currState->req->getVaddr(); 241 if (currState->aarch64) 242 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted, 243 currState->tc, currState->el); 244 else 245 currState->vaddr = currState->vaddr_tainted; 246 247 if (currState->aarch64) { 248 switch (currState->el) { 249 case EL0: 250 case EL1: 251 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); 252 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1); 253 break; 254 // @todo: uncomment this to enable Virtualization 255 // case EL2: 256 // assert(haveVirtualization); 257 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2); 258 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2); 259 // break; 260 case EL3: 261 assert(haveSecurity); 262 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3); 263 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3); 264 break; 265 default: 266 panic("Invalid exception level"); 267 break; 268 } 269 } else { 270 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 271 MISCREG_SCTLR, currState->tc, !currState->isSecure)); 272 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 273 MISCREG_TTBCR, currState->tc, !currState->isSecure)); 274 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR); 275 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR); 276 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR); 277 } 278 sctlr = currState->sctlr; 279 280 currState->isFetch = (currState->mode == TLB::Execute); 281 currState->isWrite = (currState->mode == TLB::Write); 282 283 statRequestOrigin[REQUESTED][currState->isFetch]++; 284 285 // We only do a second stage of translation if we're not secure, or in 286 // hyp mode, the second stage MMU is enabled, and this table walker 287 // instance is the first stage. 288 currState->doingStage2 = false; 289 // @todo: for now disable this in AArch64 (HCR is not set) 290 currState->stage2Req = !currState->aarch64 && currState->hcr.vm && 291 !isStage2 && !currState->isSecure && !currState->isHyp; 292 293 bool long_desc_format = currState->aarch64 || 294 (_haveLPAE && currState->ttbcr.eae) || 295 _isHyp || isStage2; 296 297 if (long_desc_format) { 298 // Helper variables used for hierarchical permissions 299 currState->secureLookup = currState->isSecure; 300 currState->rwTable = true; 301 currState->userTable = true; 302 currState->xnTable = false; 303 currState->pxnTable = false; 304 305 ++statWalksLongDescriptor; 306 } else { 307 ++statWalksShortDescriptor; 308 } 309 310 if (!currState->timing) { 311 Fault fault = NoFault; 312 if (currState->aarch64) 313 fault = processWalkAArch64(); 314 else if (long_desc_format) 315 fault = processWalkLPAE(); 316 else 317 fault = processWalk(); 318 319 // If this was a functional non-timing access restore state to 320 // how we found it. 321 if (currState->functional) { 322 delete currState; 323 currState = savedCurrState; 324 } 325 return fault; 326 } 327 328 if (pending || pendingQueue.size()) { 329 pendingQueue.push_back(currState); 330 currState = NULL; 331 pendingChange(); 332 } else { 333 pending = true; 334 pendingChange(); 335 if (currState->aarch64) 336 return processWalkAArch64(); 337 else if (long_desc_format) 338 return processWalkLPAE(); 339 else 340 return processWalk(); 341 } 342 343 return NoFault; 344} 345 346void 347TableWalker::processWalkWrapper() 348{ 349 assert(!currState); 350 assert(pendingQueue.size()); 351 pendingChange(); 352 currState = pendingQueue.front(); 353 354 ExceptionLevel target_el = EL0; 355 if (currState->aarch64) 356 target_el = currEL(currState->tc); 357 else 358 target_el = EL1; 359 360 // Check if a previous walk filled this request already 361 // @TODO Should this always be the TLB or should we look in the stage2 TLB? 362 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid, 363 currState->vmid, currState->isHyp, currState->isSecure, true, false, 364 target_el); 365 366 // Check if we still need to have a walk for this request. If the requesting 367 // instruction has been squashed, or a previous walk has filled the TLB with 368 // a match, we just want to get rid of the walk. The latter could happen 369 // when there are multiple outstanding misses to a single page and a 370 // previous request has been successfully translated. 371 if (!currState->transState->squashed() && !te) { 372 // We've got a valid request, lets process it 373 pending = true; 374 pendingQueue.pop_front(); 375 // Keep currState in case one of the processWalk... calls NULLs it 376 WalkerState *curr_state_copy = currState; 377 Fault f; 378 if (currState->aarch64) 379 f = processWalkAArch64(); 380 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2) 381 f = processWalkLPAE(); 382 else 383 f = processWalk(); 384 385 if (f != NoFault) { 386 curr_state_copy->transState->finish(f, curr_state_copy->req, 387 curr_state_copy->tc, curr_state_copy->mode); 388 389 delete curr_state_copy; 390 } 391 return; 392 } 393 394 395 // If the instruction that we were translating for has been 396 // squashed we shouldn't bother. 397 unsigned num_squashed = 0; 398 ThreadContext *tc = currState->tc; 399 while ((num_squashed < numSquashable) && currState && 400 (currState->transState->squashed() || te)) { 401 pendingQueue.pop_front(); 402 num_squashed++; 403 statSquashedBefore++; 404 405 DPRINTF(TLB, "Squashing table walk for address %#x\n", 406 currState->vaddr_tainted); 407 408 if (currState->transState->squashed()) { 409 // finish the translation which will delete the translation object 410 currState->transState->finish( 411 std::make_shared<UnimpFault>("Squashed Inst"), 412 currState->req, currState->tc, currState->mode); 413 } else { 414 // translate the request now that we know it will work 415 statWalkServiceTime.sample(curTick() - currState->startTime); 416 tlb->translateTiming(currState->req, currState->tc, 417 currState->transState, currState->mode); 418 419 } 420 421 // delete the current request 422 delete currState; 423 424 // peak at the next one 425 if (pendingQueue.size()) { 426 currState = pendingQueue.front(); 427 te = tlb->lookup(currState->vaddr, currState->asid, 428 currState->vmid, currState->isHyp, currState->isSecure, true, 429 false, target_el); 430 } else { 431 // Terminate the loop, nothing more to do 432 currState = NULL; 433 } 434 } 435 pendingChange(); 436 437 // if we still have pending translations, schedule more work 438 nextWalk(tc); 439 currState = NULL; 440} 441 442Fault 443TableWalker::processWalk() 444{ 445 Addr ttbr = 0; 446 447 // If translation isn't enabled, we shouldn't be here 448 assert(currState->sctlr.m || isStage2); 449 450 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n", 451 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31, 452 32 - currState->ttbcr.n)); 453 454 statWalkWaitTime.sample(curTick() - currState->startTime); 455 456 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31, 457 32 - currState->ttbcr.n)) { 458 DPRINTF(TLB, " - Selecting TTBR0\n"); 459 // Check if table walk is allowed when Security Extensions are enabled 460 if (haveSecurity && currState->ttbcr.pd0) { 461 if (currState->isFetch) 462 return std::make_shared<PrefetchAbort>( 463 currState->vaddr_tainted, 464 ArmFault::TranslationLL + L1, 465 isStage2, 466 ArmFault::VmsaTran); 467 else 468 return std::make_shared<DataAbort>( 469 currState->vaddr_tainted, 470 TlbEntry::DomainType::NoAccess, currState->isWrite, 471 ArmFault::TranslationLL + L1, isStage2, 472 ArmFault::VmsaTran); 473 } 474 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 475 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 476 } else { 477 DPRINTF(TLB, " - Selecting TTBR1\n"); 478 // Check if table walk is allowed when Security Extensions are enabled 479 if (haveSecurity && currState->ttbcr.pd1) { 480 if (currState->isFetch) 481 return std::make_shared<PrefetchAbort>( 482 currState->vaddr_tainted, 483 ArmFault::TranslationLL + L1, 484 isStage2, 485 ArmFault::VmsaTran); 486 else 487 return std::make_shared<DataAbort>( 488 currState->vaddr_tainted, 489 TlbEntry::DomainType::NoAccess, currState->isWrite, 490 ArmFault::TranslationLL + L1, isStage2, 491 ArmFault::VmsaTran); 492 } 493 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 494 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 495 currState->ttbcr.n = 0; 496 } 497 498 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) | 499 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2); 500 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr, 501 currState->isSecure ? "s" : "ns"); 502 503 // Trickbox address check 504 Fault f;
|
505 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
506 currState->vaddr, sizeof(uint32_t), currState->isFetch,
507 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
|
505 f = testWalk(l1desc_addr, sizeof(uint32_t), 506 TlbEntry::DomainType::NoAccess, L1); |
507 if (f) { 508 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 509 if (currState->timing) { 510 pending = false; 511 nextWalk(currState->tc); 512 currState = NULL; 513 } else { 514 currState->tc = NULL; 515 currState->req = NULL; 516 } 517 return f; 518 } 519 520 Request::Flags flag = Request::PT_WALK; 521 if (currState->sctlr.c == 0) { 522 flag.set(Request::UNCACHEABLE); 523 } 524 525 if (currState->isSecure) { 526 flag.set(Request::SECURE); 527 } 528 529 bool delayed; 530 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data, 531 sizeof(uint32_t), flag, L1, &doL1DescEvent, 532 &TableWalker::doL1Descriptor); 533 if (!delayed) { 534 f = currState->fault; 535 } 536 537 return f; 538} 539 540Fault 541TableWalker::processWalkLPAE() 542{ 543 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr; 544 int tsz, n; 545 LookupLevel start_lookup_level = L1; 546 547 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n", 548 currState->vaddr_tainted, currState->ttbcr); 549 550 statWalkWaitTime.sample(curTick() - currState->startTime); 551 552 Request::Flags flag = Request::PT_WALK; 553 if (currState->isSecure) 554 flag.set(Request::SECURE); 555 556 // work out which base address register to use, if in hyp mode we always 557 // use HTTBR 558 if (isStage2) { 559 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n"); 560 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR); 561 tsz = sext<4>(currState->vtcr.t0sz); 562 start_lookup_level = currState->vtcr.sl0 ? L1 : L2; 563 } else if (currState->isHyp) { 564 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n"); 565 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR); 566 tsz = currState->htcr.t0sz; 567 } else { 568 assert(_haveLPAE && currState->ttbcr.eae); 569 570 // Determine boundaries of TTBR0/1 regions 571 if (currState->ttbcr.t0sz) 572 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1; 573 else if (currState->ttbcr.t1sz) 574 ttbr0_max = (1ULL << 32) - 575 (1ULL << (32 - currState->ttbcr.t1sz)) - 1; 576 else 577 ttbr0_max = (1ULL << 32) - 1; 578 if (currState->ttbcr.t1sz) 579 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)); 580 else 581 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz)); 582 583 // The following code snippet selects the appropriate translation table base 584 // address (TTBR0 or TTBR1) and the appropriate starting lookup level 585 // depending on the address range supported by the translation table (ARM 586 // ARM issue C B3.6.4) 587 if (currState->vaddr <= ttbr0_max) { 588 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n"); 589 // Check if table walk is allowed 590 if (currState->ttbcr.epd0) { 591 if (currState->isFetch) 592 return std::make_shared<PrefetchAbort>( 593 currState->vaddr_tainted, 594 ArmFault::TranslationLL + L1, 595 isStage2, 596 ArmFault::LpaeTran); 597 else 598 return std::make_shared<DataAbort>( 599 currState->vaddr_tainted, 600 TlbEntry::DomainType::NoAccess, 601 currState->isWrite, 602 ArmFault::TranslationLL + L1, 603 isStage2, 604 ArmFault::LpaeTran); 605 } 606 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 607 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 608 tsz = currState->ttbcr.t0sz; 609 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB 610 start_lookup_level = L2; 611 } else if (currState->vaddr >= ttbr1_min) { 612 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n"); 613 // Check if table walk is allowed 614 if (currState->ttbcr.epd1) { 615 if (currState->isFetch) 616 return std::make_shared<PrefetchAbort>( 617 currState->vaddr_tainted, 618 ArmFault::TranslationLL + L1, 619 isStage2, 620 ArmFault::LpaeTran); 621 else 622 return std::make_shared<DataAbort>( 623 currState->vaddr_tainted, 624 TlbEntry::DomainType::NoAccess, 625 currState->isWrite, 626 ArmFault::TranslationLL + L1, 627 isStage2, 628 ArmFault::LpaeTran); 629 } 630 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 631 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 632 tsz = currState->ttbcr.t1sz; 633 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB 634 start_lookup_level = L2; 635 } else { 636 // Out of boundaries -> translation fault 637 if (currState->isFetch) 638 return std::make_shared<PrefetchAbort>( 639 currState->vaddr_tainted, 640 ArmFault::TranslationLL + L1, 641 isStage2, 642 ArmFault::LpaeTran); 643 else 644 return std::make_shared<DataAbort>( 645 currState->vaddr_tainted, 646 TlbEntry::DomainType::NoAccess, 647 currState->isWrite, ArmFault::TranslationLL + L1, 648 isStage2, ArmFault::LpaeTran); 649 } 650 651 } 652 653 // Perform lookup (ARM ARM issue C B3.6.6) 654 if (start_lookup_level == L1) { 655 n = 5 - tsz; 656 desc_addr = mbits(ttbr, 39, n) | 657 (bits(currState->vaddr, n + 26, 30) << 3); 658 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 659 desc_addr, currState->isSecure ? "s" : "ns"); 660 } else { 661 // Skip first-level lookup 662 n = (tsz >= 2 ? 14 - tsz : 12); 663 desc_addr = mbits(ttbr, 39, n) | 664 (bits(currState->vaddr, n + 17, 21) << 3); 665 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 666 desc_addr, currState->isSecure ? "s" : "ns"); 667 } 668 669 // Trickbox address check
|
671 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
672 currState->vaddr, sizeof(uint64_t), currState->isFetch,
673 currState->isWrite, TlbEntry::DomainType::NoAccess,
674 start_lookup_level);
|
670 Fault f = testWalk(desc_addr, sizeof(uint64_t), 671 TlbEntry::DomainType::NoAccess, start_lookup_level); |
672 if (f) { 673 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 674 if (currState->timing) { 675 pending = false; 676 nextWalk(currState->tc); 677 currState = NULL; 678 } else { 679 currState->tc = NULL; 680 currState->req = NULL; 681 } 682 return f; 683 } 684 685 if (currState->sctlr.c == 0) { 686 flag.set(Request::UNCACHEABLE); 687 } 688 689 currState->longDesc.lookupLevel = start_lookup_level; 690 currState->longDesc.aarch64 = false; 691 currState->longDesc.grainSize = Grain4KB; 692 693 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent 694 : (Event *) &doL2LongDescEvent; 695 696 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, 697 sizeof(uint64_t), flag, start_lookup_level, 698 event, &TableWalker::doLongDescriptor); 699 if (!delayed) { 700 f = currState->fault; 701 } 702 703 return f; 704} 705 706unsigned 707TableWalker::adjustTableSizeAArch64(unsigned tsz) 708{ 709 if (tsz < 25) 710 return 25; 711 if (tsz > 48) 712 return 48; 713 return tsz; 714} 715 716bool 717TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange) 718{ 719 return (currPhysAddrRange != MaxPhysAddrRange && 720 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange)); 721} 722 723Fault 724TableWalker::processWalkAArch64() 725{ 726 assert(currState->aarch64); 727 728 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n", 729 currState->vaddr_tainted, currState->tcr); 730 731 static const GrainSize GrainMapDefault[] = 732 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain }; 733 static const GrainSize GrainMap_EL1_tg1[] = 734 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB }; 735 736 statWalkWaitTime.sample(curTick() - currState->startTime); 737 738 // Determine TTBR, table size, granule size and phys. address range 739 Addr ttbr = 0; 740 int tsz = 0, ps = 0; 741 GrainSize tg = Grain4KB; // grain size computed from tg* field 742 bool fault = false; 743 switch (currState->el) { 744 case EL0: 745 case EL1: 746 switch (bits(currState->vaddr, 63,48)) { 747 case 0: 748 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 749 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1); 750 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 751 tg = GrainMapDefault[currState->tcr.tg0]; 752 if (bits(currState->vaddr, 63, tsz) != 0x0 || 753 currState->tcr.epd0) 754 fault = true; 755 break; 756 case 0xffff: 757 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); 758 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1); 759 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); 760 tg = GrainMap_EL1_tg1[currState->tcr.tg1]; 761 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || 762 currState->tcr.epd1) 763 fault = true; 764 break; 765 default: 766 // top two bytes must be all 0s or all 1s, else invalid addr 767 fault = true; 768 } 769 ps = currState->tcr.ips; 770 break; 771 case EL2: 772 case EL3: 773 switch(bits(currState->vaddr, 63,48)) { 774 case 0: 775 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 776 if (currState->el == EL2) 777 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2); 778 else 779 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3); 780 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 781 tg = GrainMapDefault[currState->tcr.tg0]; 782 break; 783 default: 784 // invalid addr if top two bytes are not all 0s 785 fault = true; 786 } 787 ps = currState->tcr.ips; 788 break; 789 } 790 791 if (fault) { 792 Fault f; 793 if (currState->isFetch) 794 f = std::make_shared<PrefetchAbort>( 795 currState->vaddr_tainted, 796 ArmFault::TranslationLL + L0, isStage2, 797 ArmFault::LpaeTran); 798 else 799 f = std::make_shared<DataAbort>( 800 currState->vaddr_tainted, 801 TlbEntry::DomainType::NoAccess, 802 currState->isWrite, 803 ArmFault::TranslationLL + L0, 804 isStage2, ArmFault::LpaeTran); 805 806 if (currState->timing) { 807 pending = false; 808 nextWalk(currState->tc); 809 currState = NULL; 810 } else { 811 currState->tc = NULL; 812 currState->req = NULL; 813 } 814 return f; 815 816 } 817 818 if (tg == ReservedGrain) { 819 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION " 820 "DEFINED behavior takes this to mean 4KB granules\n"); 821 tg = Grain4KB; 822 } 823 824 int stride = tg - 3; 825 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS; 826 827 // Determine starting lookup level 828 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library 829 // in ARM DDI 0487A. These table values correspond to the cascading tests 830 // to compute the lookup level and are of the form 831 // (grain_size + N*stride), for N = {1, 2, 3}. 832 // A value of 64 will never succeed and a value of 0 will always succeed. 833 { 834 struct GrainMap { 835 GrainSize grain_size; 836 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS]; 837 }; 838 static const GrainMap GM[] = { 839 { Grain4KB, { 39, 30, 0, 0 } }, 840 { Grain16KB, { 47, 36, 25, 0 } }, 841 { Grain64KB, { 64, 42, 29, 0 } } 842 }; 843 844 const unsigned *lookup = NULL; // points to a lookup_level_cutoff 845 846 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[] 847 if (tg == GM[i].grain_size) { 848 lookup = GM[i].lookup_level_cutoff; 849 break; 850 } 851 } 852 assert(lookup); 853 854 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) { 855 if (tsz > lookup[L]) { 856 start_lookup_level = (LookupLevel) L; 857 break; 858 } 859 } 860 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, 861 "Table walker couldn't find lookup level\n"); 862 } 863 864 // Determine table base address 865 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg; 866 Addr base_addr = mbits(ttbr, 47, base_addr_lo); 867 868 // Determine physical address size and raise an Address Size Fault if 869 // necessary 870 int pa_range = decodePhysAddrRange64(ps); 871 // Clamp to lower limit 872 if (pa_range > physAddrRange) 873 currState->physAddrRange = physAddrRange; 874 else 875 currState->physAddrRange = pa_range; 876 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) { 877 DPRINTF(TLB, "Address size fault before any lookup\n"); 878 Fault f; 879 if (currState->isFetch) 880 f = std::make_shared<PrefetchAbort>( 881 currState->vaddr_tainted, 882 ArmFault::AddressSizeLL + start_lookup_level, 883 isStage2, 884 ArmFault::LpaeTran); 885 else 886 f = std::make_shared<DataAbort>( 887 currState->vaddr_tainted, 888 TlbEntry::DomainType::NoAccess, 889 currState->isWrite, 890 ArmFault::AddressSizeLL + start_lookup_level, 891 isStage2, 892 ArmFault::LpaeTran); 893 894 895 if (currState->timing) { 896 pending = false; 897 nextWalk(currState->tc); 898 currState = NULL; 899 } else { 900 currState->tc = NULL; 901 currState->req = NULL; 902 } 903 return f; 904 905 } 906 907 // Determine descriptor address 908 Addr desc_addr = base_addr | 909 (bits(currState->vaddr, tsz - 1, 910 stride * (3 - start_lookup_level) + tg) << 3); 911 912 // Trickbox address check
|
916 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
917 currState->vaddr, sizeof(uint64_t), currState->isFetch,
918 currState->isWrite, TlbEntry::DomainType::NoAccess,
919 start_lookup_level);
|
913 Fault f = testWalk(desc_addr, sizeof(uint64_t), 914 TlbEntry::DomainType::NoAccess, start_lookup_level); |
915 if (f) { 916 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 917 if (currState->timing) { 918 pending = false; 919 nextWalk(currState->tc); 920 currState = NULL; 921 } else { 922 currState->tc = NULL; 923 currState->req = NULL; 924 } 925 return f; 926 } 927 928 Request::Flags flag = Request::PT_WALK; 929 if (currState->sctlr.c == 0) { 930 flag.set(Request::UNCACHEABLE); 931 } 932 933 if (currState->isSecure) { 934 flag.set(Request::SECURE); 935 } 936 937 currState->longDesc.lookupLevel = start_lookup_level; 938 currState->longDesc.aarch64 = true; 939 currState->longDesc.grainSize = tg; 940 941 if (currState->timing) { 942 Event *event; 943 switch (start_lookup_level) { 944 case L0: 945 event = (Event *) &doL0LongDescEvent; 946 break; 947 case L1: 948 event = (Event *) &doL1LongDescEvent; 949 break; 950 case L2: 951 event = (Event *) &doL2LongDescEvent; 952 break; 953 case L3: 954 event = (Event *) &doL3LongDescEvent; 955 break; 956 default: 957 panic("Invalid table lookup level"); 958 break; 959 } 960 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 961 event, (uint8_t*) &currState->longDesc.data, 962 currState->tc->getCpuPtr()->clockPeriod(), flag); 963 DPRINTF(TLBVerbose, 964 "Adding to walker fifo: queue size before adding: %d\n", 965 stateQueues[start_lookup_level].size()); 966 stateQueues[start_lookup_level].push_back(currState); 967 currState = NULL; 968 } else if (!currState->functional) { 969 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 970 NULL, (uint8_t*) &currState->longDesc.data, 971 currState->tc->getCpuPtr()->clockPeriod(), flag); 972 doLongDescriptor(); 973 f = currState->fault; 974 } else { 975 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag, 976 masterId); 977 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 978 pkt->dataStatic((uint8_t*) &currState->longDesc.data); 979 port->sendFunctional(pkt); 980 doLongDescriptor(); 981 delete req; 982 delete pkt; 983 f = currState->fault; 984 } 985 986 return f; 987} 988 989void 990TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, 991 uint8_t texcb, bool s) 992{ 993 // Note: tc and sctlr local variables are hiding tc and sctrl class 994 // variables 995 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s); 996 te.shareable = false; // default value 997 te.nonCacheable = false; 998 te.outerShareable = false; 999 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) { 1000 switch(texcb) { 1001 case 0: // Stongly-ordered 1002 te.nonCacheable = true; 1003 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1004 te.shareable = true; 1005 te.innerAttrs = 1; 1006 te.outerAttrs = 0; 1007 break; 1008 case 1: // Shareable Device 1009 te.nonCacheable = true; 1010 te.mtype = TlbEntry::MemoryType::Device; 1011 te.shareable = true; 1012 te.innerAttrs = 3; 1013 te.outerAttrs = 0; 1014 break; 1015 case 2: // Outer and Inner Write-Through, no Write-Allocate 1016 te.mtype = TlbEntry::MemoryType::Normal; 1017 te.shareable = s; 1018 te.innerAttrs = 6; 1019 te.outerAttrs = bits(texcb, 1, 0); 1020 break; 1021 case 3: // Outer and Inner Write-Back, no Write-Allocate 1022 te.mtype = TlbEntry::MemoryType::Normal; 1023 te.shareable = s; 1024 te.innerAttrs = 7; 1025 te.outerAttrs = bits(texcb, 1, 0); 1026 break; 1027 case 4: // Outer and Inner Non-cacheable 1028 te.nonCacheable = true; 1029 te.mtype = TlbEntry::MemoryType::Normal; 1030 te.shareable = s; 1031 te.innerAttrs = 0; 1032 te.outerAttrs = bits(texcb, 1, 0); 1033 break; 1034 case 5: // Reserved 1035 panic("Reserved texcb value!\n"); 1036 break; 1037 case 6: // Implementation Defined 1038 panic("Implementation-defined texcb value!\n"); 1039 break; 1040 case 7: // Outer and Inner Write-Back, Write-Allocate 1041 te.mtype = TlbEntry::MemoryType::Normal; 1042 te.shareable = s; 1043 te.innerAttrs = 5; 1044 te.outerAttrs = 1; 1045 break; 1046 case 8: // Non-shareable Device 1047 te.nonCacheable = true; 1048 te.mtype = TlbEntry::MemoryType::Device; 1049 te.shareable = false; 1050 te.innerAttrs = 3; 1051 te.outerAttrs = 0; 1052 break; 1053 case 9 ... 15: // Reserved 1054 panic("Reserved texcb value!\n"); 1055 break; 1056 case 16 ... 31: // Cacheable Memory 1057 te.mtype = TlbEntry::MemoryType::Normal; 1058 te.shareable = s; 1059 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0) 1060 te.nonCacheable = true; 1061 te.innerAttrs = bits(texcb, 1, 0); 1062 te.outerAttrs = bits(texcb, 3, 2); 1063 break; 1064 default: 1065 panic("More than 32 states for 5 bits?\n"); 1066 } 1067 } else { 1068 assert(tc); 1069 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR, 1070 currState->tc, !currState->isSecure)); 1071 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR, 1072 currState->tc, !currState->isSecure)); 1073 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr); 1074 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0; 1075 switch(bits(texcb, 2,0)) { 1076 case 0: 1077 curr_tr = prrr.tr0; 1078 curr_ir = nmrr.ir0; 1079 curr_or = nmrr.or0; 1080 te.outerShareable = (prrr.nos0 == 0); 1081 break; 1082 case 1: 1083 curr_tr = prrr.tr1; 1084 curr_ir = nmrr.ir1; 1085 curr_or = nmrr.or1; 1086 te.outerShareable = (prrr.nos1 == 0); 1087 break; 1088 case 2: 1089 curr_tr = prrr.tr2; 1090 curr_ir = nmrr.ir2; 1091 curr_or = nmrr.or2; 1092 te.outerShareable = (prrr.nos2 == 0); 1093 break; 1094 case 3: 1095 curr_tr = prrr.tr3; 1096 curr_ir = nmrr.ir3; 1097 curr_or = nmrr.or3; 1098 te.outerShareable = (prrr.nos3 == 0); 1099 break; 1100 case 4: 1101 curr_tr = prrr.tr4; 1102 curr_ir = nmrr.ir4; 1103 curr_or = nmrr.or4; 1104 te.outerShareable = (prrr.nos4 == 0); 1105 break; 1106 case 5: 1107 curr_tr = prrr.tr5; 1108 curr_ir = nmrr.ir5; 1109 curr_or = nmrr.or5; 1110 te.outerShareable = (prrr.nos5 == 0); 1111 break; 1112 case 6: 1113 panic("Imp defined type\n"); 1114 case 7: 1115 curr_tr = prrr.tr7; 1116 curr_ir = nmrr.ir7; 1117 curr_or = nmrr.or7; 1118 te.outerShareable = (prrr.nos7 == 0); 1119 break; 1120 } 1121 1122 switch(curr_tr) { 1123 case 0: 1124 DPRINTF(TLBVerbose, "StronglyOrdered\n"); 1125 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1126 te.nonCacheable = true; 1127 te.innerAttrs = 1; 1128 te.outerAttrs = 0; 1129 te.shareable = true; 1130 break; 1131 case 1: 1132 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n", 1133 prrr.ds1, prrr.ds0, s); 1134 te.mtype = TlbEntry::MemoryType::Device; 1135 te.nonCacheable = true; 1136 te.innerAttrs = 3; 1137 te.outerAttrs = 0; 1138 if (prrr.ds1 && s) 1139 te.shareable = true; 1140 if (prrr.ds0 && !s) 1141 te.shareable = true; 1142 break; 1143 case 2: 1144 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n", 1145 prrr.ns1, prrr.ns0, s); 1146 te.mtype = TlbEntry::MemoryType::Normal; 1147 if (prrr.ns1 && s) 1148 te.shareable = true; 1149 if (prrr.ns0 && !s) 1150 te.shareable = true; 1151 break; 1152 case 3: 1153 panic("Reserved type"); 1154 } 1155 1156 if (te.mtype == TlbEntry::MemoryType::Normal){ 1157 switch(curr_ir) { 1158 case 0: 1159 te.nonCacheable = true; 1160 te.innerAttrs = 0; 1161 break; 1162 case 1: 1163 te.innerAttrs = 5; 1164 break; 1165 case 2: 1166 te.innerAttrs = 6; 1167 break; 1168 case 3: 1169 te.innerAttrs = 7; 1170 break; 1171 } 1172 1173 switch(curr_or) { 1174 case 0: 1175 te.nonCacheable = true; 1176 te.outerAttrs = 0; 1177 break; 1178 case 1: 1179 te.outerAttrs = 1; 1180 break; 1181 case 2: 1182 te.outerAttrs = 2; 1183 break; 1184 case 3: 1185 te.outerAttrs = 3; 1186 break; 1187 } 1188 } 1189 } 1190 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, " 1191 "outerAttrs: %d\n", 1192 te.shareable, te.innerAttrs, te.outerAttrs); 1193 te.setAttributes(false); 1194} 1195 1196void 1197TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, 1198 LongDescriptor &lDescriptor) 1199{ 1200 assert(_haveLPAE); 1201 1202 uint8_t attr; 1203 uint8_t sh = lDescriptor.sh(); 1204 // Different format and source of attributes if this is a stage 2 1205 // translation 1206 if (isStage2) { 1207 attr = lDescriptor.memAttr(); 1208 uint8_t attr_3_2 = (attr >> 2) & 0x3; 1209 uint8_t attr_1_0 = attr & 0x3; 1210 1211 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh); 1212 1213 if (attr_3_2 == 0) { 1214 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered 1215 : TlbEntry::MemoryType::Device; 1216 te.outerAttrs = 0; 1217 te.innerAttrs = attr_1_0 == 0 ? 1 : 3; 1218 te.nonCacheable = true; 1219 } else { 1220 te.mtype = TlbEntry::MemoryType::Normal; 1221 te.outerAttrs = attr_3_2 == 1 ? 0 : 1222 attr_3_2 == 2 ? 2 : 1; 1223 te.innerAttrs = attr_1_0 == 1 ? 0 : 1224 attr_1_0 == 2 ? 6 : 5; 1225 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1); 1226 } 1227 } else { 1228 uint8_t attrIndx = lDescriptor.attrIndx(); 1229 1230 // LPAE always uses remapping of memory attributes, irrespective of the 1231 // value of SCTLR.TRE 1232 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0; 1233 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc, 1234 !currState->isSecure); 1235 uint32_t mair = currState->tc->readMiscReg(reg_as_int); 1236 attr = (mair >> (8 * (attrIndx % 4))) & 0xff; 1237 uint8_t attr_7_4 = bits(attr, 7, 4); 1238 uint8_t attr_3_0 = bits(attr, 3, 0); 1239 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr); 1240 1241 // Note: the memory subsystem only cares about the 'cacheable' memory 1242 // attribute. The other attributes are only used to fill the PAR register 1243 // accordingly to provide the illusion of full support 1244 te.nonCacheable = false; 1245 1246 switch (attr_7_4) { 1247 case 0x0: 1248 // Strongly-ordered or Device memory 1249 if (attr_3_0 == 0x0) 1250 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1251 else if (attr_3_0 == 0x4) 1252 te.mtype = TlbEntry::MemoryType::Device; 1253 else 1254 panic("Unpredictable behavior\n"); 1255 te.nonCacheable = true; 1256 te.outerAttrs = 0; 1257 break; 1258 case 0x4: 1259 // Normal memory, Outer Non-cacheable 1260 te.mtype = TlbEntry::MemoryType::Normal; 1261 te.outerAttrs = 0; 1262 if (attr_3_0 == 0x4) 1263 // Inner Non-cacheable 1264 te.nonCacheable = true; 1265 else if (attr_3_0 < 0x8) 1266 panic("Unpredictable behavior\n"); 1267 break; 1268 case 0x8: 1269 case 0x9: 1270 case 0xa: 1271 case 0xb: 1272 case 0xc: 1273 case 0xd: 1274 case 0xe: 1275 case 0xf: 1276 if (attr_7_4 & 0x4) { 1277 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3; 1278 } else { 1279 te.outerAttrs = 0x2; 1280 } 1281 // Normal memory, Outer Cacheable 1282 te.mtype = TlbEntry::MemoryType::Normal; 1283 if (attr_3_0 != 0x4 && attr_3_0 < 0x8) 1284 panic("Unpredictable behavior\n"); 1285 break; 1286 default: 1287 panic("Unpredictable behavior\n"); 1288 break; 1289 } 1290 1291 switch (attr_3_0) { 1292 case 0x0: 1293 te.innerAttrs = 0x1; 1294 break; 1295 case 0x4: 1296 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0; 1297 break; 1298 case 0x8: 1299 case 0x9: 1300 case 0xA: 1301 case 0xB: 1302 te.innerAttrs = 6; 1303 break; 1304 case 0xC: 1305 case 0xD: 1306 case 0xE: 1307 case 0xF: 1308 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7; 1309 break; 1310 default: 1311 panic("Unpredictable behavior\n"); 1312 break; 1313 } 1314 } 1315 1316 te.outerShareable = sh == 2; 1317 te.shareable = (sh & 0x2) ? true : false; 1318 te.setAttributes(true); 1319 te.attributes |= (uint64_t) attr << 56; 1320} 1321 1322void 1323TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx, 1324 uint8_t sh) 1325{ 1326 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh); 1327 1328 // Select MAIR 1329 uint64_t mair; 1330 switch (currState->el) { 1331 case EL0: 1332 case EL1: 1333 mair = tc->readMiscReg(MISCREG_MAIR_EL1); 1334 break; 1335 case EL2: 1336 mair = tc->readMiscReg(MISCREG_MAIR_EL2); 1337 break; 1338 case EL3: 1339 mair = tc->readMiscReg(MISCREG_MAIR_EL3); 1340 break; 1341 default: 1342 panic("Invalid exception level"); 1343 break; 1344 } 1345 1346 // Select attributes 1347 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx); 1348 uint8_t attr_lo = bits(attr, 3, 0); 1349 uint8_t attr_hi = bits(attr, 7, 4); 1350 1351 // Memory type 1352 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; 1353 1354 // Cacheability 1355 te.nonCacheable = false; 1356 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory 1357 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable 1358 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable 1359 te.nonCacheable = true; 1360 } 1361 1362 te.shareable = sh == 2; 1363 te.outerShareable = (sh & 0x2) ? true : false; 1364 // Attributes formatted according to the 64-bit PAR 1365 te.attributes = ((uint64_t) attr << 56) | 1366 (1 << 11) | // LPAE bit 1367 (te.ns << 9) | // NS bit 1368 (sh << 7); 1369} 1370 1371void 1372TableWalker::doL1Descriptor() 1373{ 1374 if (currState->fault != NoFault) { 1375 return; 1376 } 1377 1378 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n", 1379 currState->vaddr_tainted, currState->l1Desc.data); 1380 TlbEntry te; 1381 1382 switch (currState->l1Desc.type()) { 1383 case L1Descriptor::Ignore: 1384 case L1Descriptor::Reserved: 1385 if (!currState->timing) { 1386 currState->tc = NULL; 1387 currState->req = NULL; 1388 } 1389 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n"); 1390 if (currState->isFetch) 1391 currState->fault = 1392 std::make_shared<PrefetchAbort>( 1393 currState->vaddr_tainted, 1394 ArmFault::TranslationLL + L1, 1395 isStage2, 1396 ArmFault::VmsaTran); 1397 else 1398 currState->fault = 1399 std::make_shared<DataAbort>( 1400 currState->vaddr_tainted, 1401 TlbEntry::DomainType::NoAccess, 1402 currState->isWrite, 1403 ArmFault::TranslationLL + L1, isStage2, 1404 ArmFault::VmsaTran); 1405 return; 1406 case L1Descriptor::Section: 1407 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) { 1408 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is 1409 * enabled if set, do l1.Desc.setAp0() instead of generating 1410 * AccessFlag0 1411 */ 1412 1413 currState->fault = std::make_shared<DataAbort>( 1414 currState->vaddr_tainted, 1415 currState->l1Desc.domain(), 1416 currState->isWrite, 1417 ArmFault::AccessFlagLL + L1, 1418 isStage2, 1419 ArmFault::VmsaTran); 1420 } 1421 if (currState->l1Desc.supersection()) { 1422 panic("Haven't implemented supersections\n"); 1423 } 1424 insertTableEntry(currState->l1Desc, false); 1425 return; 1426 case L1Descriptor::PageTable: 1427 { 1428 Addr l2desc_addr; 1429 l2desc_addr = currState->l1Desc.l2Addr() | 1430 (bits(currState->vaddr, 19, 12) << 2); 1431 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n", 1432 l2desc_addr, currState->isSecure ? "s" : "ns"); 1433 1434 // Trickbox address check
|
1440 currState->fault = tlb->walkTrickBoxCheck(
1441 l2desc_addr, currState->isSecure, currState->vaddr,
1442 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1443 currState->l1Desc.domain(), L2);
|
1435 currState->fault = testWalk(l2desc_addr, sizeof(uint32_t), 1436 currState->l1Desc.domain(), L2); |
1437 1438 if (currState->fault) { 1439 if (!currState->timing) { 1440 currState->tc = NULL; 1441 currState->req = NULL; 1442 } 1443 return; 1444 } 1445 1446 Request::Flags flag = Request::PT_WALK; 1447 if (currState->isSecure) 1448 flag.set(Request::SECURE); 1449 1450 bool delayed; 1451 delayed = fetchDescriptor(l2desc_addr, 1452 (uint8_t*)&currState->l2Desc.data, 1453 sizeof(uint32_t), flag, -1, &doL2DescEvent, 1454 &TableWalker::doL2Descriptor); 1455 if (delayed) { 1456 currState->delayed = true; 1457 } 1458 1459 return; 1460 } 1461 default: 1462 panic("A new type in a 2 bit field?\n"); 1463 } 1464} 1465 1466void 1467TableWalker::doLongDescriptor() 1468{ 1469 if (currState->fault != NoFault) { 1470 return; 1471 } 1472 1473 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n", 1474 currState->longDesc.lookupLevel, currState->vaddr_tainted, 1475 currState->longDesc.data, 1476 currState->aarch64 ? "AArch64" : "long-desc."); 1477 1478 if ((currState->longDesc.type() == LongDescriptor::Block) || 1479 (currState->longDesc.type() == LongDescriptor::Page)) { 1480 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, " 1481 "xn: %d, ap: %d, af: %d, type: %d\n", 1482 currState->longDesc.lookupLevel, 1483 currState->longDesc.data, 1484 currState->longDesc.pxn(), 1485 currState->longDesc.xn(), 1486 currState->longDesc.ap(), 1487 currState->longDesc.af(), 1488 currState->longDesc.type()); 1489 } else { 1490 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n", 1491 currState->longDesc.lookupLevel, 1492 currState->longDesc.data, 1493 currState->longDesc.type()); 1494 } 1495 1496 TlbEntry te; 1497 1498 switch (currState->longDesc.type()) { 1499 case LongDescriptor::Invalid: 1500 if (!currState->timing) { 1501 currState->tc = NULL; 1502 currState->req = NULL; 1503 } 1504 1505 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n", 1506 currState->longDesc.lookupLevel, 1507 ArmFault::TranslationLL + currState->longDesc.lookupLevel); 1508 if (currState->isFetch) 1509 currState->fault = std::make_shared<PrefetchAbort>( 1510 currState->vaddr_tainted, 1511 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1512 isStage2, 1513 ArmFault::LpaeTran); 1514 else 1515 currState->fault = std::make_shared<DataAbort>( 1516 currState->vaddr_tainted, 1517 TlbEntry::DomainType::NoAccess, 1518 currState->isWrite, 1519 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1520 isStage2, 1521 ArmFault::LpaeTran); 1522 return; 1523 case LongDescriptor::Block: 1524 case LongDescriptor::Page: 1525 { 1526 bool fault = false; 1527 bool aff = false; 1528 // Check for address size fault 1529 if (checkAddrSizeFaultAArch64( 1530 mbits(currState->longDesc.data, MaxPhysAddrRange - 1, 1531 currState->longDesc.offsetBits()), 1532 currState->physAddrRange)) { 1533 fault = true; 1534 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1535 currState->longDesc.lookupLevel); 1536 // Check for access fault 1537 } else if (currState->longDesc.af() == 0) { 1538 fault = true; 1539 DPRINTF(TLB, "L%d descriptor causing Access Fault\n", 1540 currState->longDesc.lookupLevel); 1541 aff = true; 1542 } 1543 if (fault) { 1544 if (currState->isFetch) 1545 currState->fault = std::make_shared<PrefetchAbort>( 1546 currState->vaddr_tainted, 1547 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1548 currState->longDesc.lookupLevel, 1549 isStage2, 1550 ArmFault::LpaeTran); 1551 else 1552 currState->fault = std::make_shared<DataAbort>( 1553 currState->vaddr_tainted, 1554 TlbEntry::DomainType::NoAccess, currState->isWrite, 1555 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1556 currState->longDesc.lookupLevel, 1557 isStage2, 1558 ArmFault::LpaeTran); 1559 } else { 1560 insertTableEntry(currState->longDesc, true); 1561 } 1562 } 1563 return; 1564 case LongDescriptor::Table: 1565 { 1566 // Set hierarchical permission flags 1567 currState->secureLookup = currState->secureLookup && 1568 currState->longDesc.secureTable(); 1569 currState->rwTable = currState->rwTable && 1570 currState->longDesc.rwTable(); 1571 currState->userTable = currState->userTable && 1572 currState->longDesc.userTable(); 1573 currState->xnTable = currState->xnTable || 1574 currState->longDesc.xnTable(); 1575 currState->pxnTable = currState->pxnTable || 1576 currState->longDesc.pxnTable(); 1577 1578 // Set up next level lookup 1579 Addr next_desc_addr = currState->longDesc.nextDescAddr( 1580 currState->vaddr); 1581 1582 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n", 1583 currState->longDesc.lookupLevel, 1584 currState->longDesc.lookupLevel + 1, 1585 next_desc_addr, 1586 currState->secureLookup ? "s" : "ns"); 1587 1588 // Check for address size fault 1589 if (currState->aarch64 && checkAddrSizeFaultAArch64( 1590 next_desc_addr, currState->physAddrRange)) { 1591 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1592 currState->longDesc.lookupLevel); 1593 if (currState->isFetch) 1594 currState->fault = std::make_shared<PrefetchAbort>( 1595 currState->vaddr_tainted, 1596 ArmFault::AddressSizeLL 1597 + currState->longDesc.lookupLevel, 1598 isStage2, 1599 ArmFault::LpaeTran); 1600 else 1601 currState->fault = std::make_shared<DataAbort>( 1602 currState->vaddr_tainted, 1603 TlbEntry::DomainType::NoAccess, currState->isWrite, 1604 ArmFault::AddressSizeLL 1605 + currState->longDesc.lookupLevel, 1606 isStage2, 1607 ArmFault::LpaeTran); 1608 return; 1609 } 1610 1611 // Trickbox address check
|
1619 currState->fault = tlb->walkTrickBoxCheck(
1620 next_desc_addr, currState->vaddr,
1621 currState->vaddr, sizeof(uint64_t),
1622 currState->isFetch, currState->isWrite,
1623 TlbEntry::DomainType::Client,
1624 toLookupLevel(currState->longDesc.lookupLevel +1));
|
1612 currState->fault = testWalk( 1613 next_desc_addr, sizeof(uint64_t), TlbEntry::DomainType::Client, 1614 toLookupLevel(currState->longDesc.lookupLevel +1)); |
1615 1616 if (currState->fault) { 1617 if (!currState->timing) { 1618 currState->tc = NULL; 1619 currState->req = NULL; 1620 } 1621 return; 1622 } 1623 1624 Request::Flags flag = Request::PT_WALK; 1625 if (currState->secureLookup) 1626 flag.set(Request::SECURE); 1627 1628 currState->longDesc.lookupLevel = 1629 (LookupLevel) (currState->longDesc.lookupLevel + 1); 1630 Event *event = NULL; 1631 switch (currState->longDesc.lookupLevel) { 1632 case L1: 1633 assert(currState->aarch64); 1634 event = &doL1LongDescEvent; 1635 break; 1636 case L2: 1637 event = &doL2LongDescEvent; 1638 break; 1639 case L3: 1640 event = &doL3LongDescEvent; 1641 break; 1642 default: 1643 panic("Wrong lookup level in table walk\n"); 1644 break; 1645 } 1646 1647 bool delayed; 1648 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data, 1649 sizeof(uint64_t), flag, -1, event, 1650 &TableWalker::doLongDescriptor); 1651 if (delayed) { 1652 currState->delayed = true; 1653 } 1654 } 1655 return; 1656 default: 1657 panic("A new type in a 2 bit field?\n"); 1658 } 1659} 1660 1661void 1662TableWalker::doL2Descriptor() 1663{ 1664 if (currState->fault != NoFault) { 1665 return; 1666 } 1667 1668 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n", 1669 currState->vaddr_tainted, currState->l2Desc.data); 1670 TlbEntry te; 1671 1672 if (currState->l2Desc.invalid()) { 1673 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n"); 1674 if (!currState->timing) { 1675 currState->tc = NULL; 1676 currState->req = NULL; 1677 } 1678 if (currState->isFetch) 1679 currState->fault = std::make_shared<PrefetchAbort>( 1680 currState->vaddr_tainted, 1681 ArmFault::TranslationLL + L2, 1682 isStage2, 1683 ArmFault::VmsaTran); 1684 else 1685 currState->fault = std::make_shared<DataAbort>( 1686 currState->vaddr_tainted, currState->l1Desc.domain(), 1687 currState->isWrite, ArmFault::TranslationLL + L2, 1688 isStage2, 1689 ArmFault::VmsaTran); 1690 return; 1691 } 1692 1693 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) { 1694 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled 1695 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0 1696 */ 1697 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n", 1698 currState->sctlr.afe, currState->l2Desc.ap()); 1699 1700 currState->fault = std::make_shared<DataAbort>( 1701 currState->vaddr_tainted, 1702 TlbEntry::DomainType::NoAccess, currState->isWrite, 1703 ArmFault::AccessFlagLL + L2, isStage2, 1704 ArmFault::VmsaTran); 1705 } 1706 1707 insertTableEntry(currState->l2Desc, false); 1708} 1709 1710void 1711TableWalker::doL1DescriptorWrapper() 1712{ 1713 currState = stateQueues[L1].front(); 1714 currState->delayed = false; 1715 // if there's a stage2 translation object we don't need it any more 1716 if (currState->stage2Tran) { 1717 delete currState->stage2Tran; 1718 currState->stage2Tran = NULL; 1719 } 1720 1721 1722 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data); 1723 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data); 1724 1725 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted); 1726 doL1Descriptor(); 1727 1728 stateQueues[L1].pop_front(); 1729 // Check if fault was generated 1730 if (currState->fault != NoFault) { 1731 currState->transState->finish(currState->fault, currState->req, 1732 currState->tc, currState->mode); 1733 statWalksShortTerminatedAtLevel[0]++; 1734 1735 pending = false; 1736 nextWalk(currState->tc); 1737 1738 currState->req = NULL; 1739 currState->tc = NULL; 1740 currState->delayed = false; 1741 delete currState; 1742 } 1743 else if (!currState->delayed) { 1744 // delay is not set so there is no L2 to do 1745 // Don't finish the translation if a stage 2 look up is underway 1746 if (!currState->doingStage2) { 1747 statWalkServiceTime.sample(curTick() - currState->startTime); 1748 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1749 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1750 currState->transState, currState->mode); 1751 statWalksShortTerminatedAtLevel[0]++; 1752 } 1753 1754 pending = false; 1755 nextWalk(currState->tc); 1756 1757 currState->req = NULL; 1758 currState->tc = NULL; 1759 currState->delayed = false; 1760 delete currState; 1761 } else { 1762 // need to do L2 descriptor 1763 stateQueues[L2].push_back(currState); 1764 } 1765 currState = NULL; 1766} 1767 1768void 1769TableWalker::doL2DescriptorWrapper() 1770{ 1771 currState = stateQueues[L2].front(); 1772 assert(currState->delayed); 1773 // if there's a stage2 translation object we don't need it any more 1774 if (currState->stage2Tran) { 1775 delete currState->stage2Tran; 1776 currState->stage2Tran = NULL; 1777 } 1778 1779 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n", 1780 currState->vaddr_tainted); 1781 doL2Descriptor(); 1782 1783 // Check if fault was generated 1784 if (currState->fault != NoFault) { 1785 currState->transState->finish(currState->fault, currState->req, 1786 currState->tc, currState->mode); 1787 statWalksShortTerminatedAtLevel[1]++; 1788 } 1789 else { 1790 // Don't finish the translation if a stage 2 look up is underway 1791 if (!currState->doingStage2) { 1792 statWalkServiceTime.sample(curTick() - currState->startTime); 1793 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1794 currState->fault = tlb->translateTiming(currState->req, 1795 currState->tc, currState->transState, currState->mode); 1796 statWalksShortTerminatedAtLevel[1]++; 1797 } 1798 } 1799 1800 1801 stateQueues[L2].pop_front(); 1802 pending = false; 1803 nextWalk(currState->tc); 1804 1805 currState->req = NULL; 1806 currState->tc = NULL; 1807 currState->delayed = false; 1808 1809 delete currState; 1810 currState = NULL; 1811} 1812 1813void 1814TableWalker::doL0LongDescriptorWrapper() 1815{ 1816 doLongDescriptorWrapper(L0); 1817} 1818 1819void 1820TableWalker::doL1LongDescriptorWrapper() 1821{ 1822 doLongDescriptorWrapper(L1); 1823} 1824 1825void 1826TableWalker::doL2LongDescriptorWrapper() 1827{ 1828 doLongDescriptorWrapper(L2); 1829} 1830 1831void 1832TableWalker::doL3LongDescriptorWrapper() 1833{ 1834 doLongDescriptorWrapper(L3); 1835} 1836 1837void 1838TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) 1839{ 1840 currState = stateQueues[curr_lookup_level].front(); 1841 assert(curr_lookup_level == currState->longDesc.lookupLevel); 1842 currState->delayed = false; 1843 1844 // if there's a stage2 translation object we don't need it any more 1845 if (currState->stage2Tran) { 1846 delete currState->stage2Tran; 1847 currState->stage2Tran = NULL; 1848 } 1849 1850 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n", 1851 currState->vaddr_tainted); 1852 doLongDescriptor(); 1853 1854 stateQueues[curr_lookup_level].pop_front(); 1855 1856 if (currState->fault != NoFault) { 1857 // A fault was generated 1858 currState->transState->finish(currState->fault, currState->req, 1859 currState->tc, currState->mode); 1860 1861 pending = false; 1862 nextWalk(currState->tc); 1863 1864 currState->req = NULL; 1865 currState->tc = NULL; 1866 currState->delayed = false; 1867 delete currState; 1868 } else if (!currState->delayed) { 1869 // No additional lookups required 1870 // Don't finish the translation if a stage 2 look up is underway 1871 if (!currState->doingStage2) { 1872 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1873 statWalkServiceTime.sample(curTick() - currState->startTime); 1874 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1875 currState->transState, 1876 currState->mode); 1877 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; 1878 } 1879 1880 pending = false; 1881 nextWalk(currState->tc); 1882 1883 currState->req = NULL; 1884 currState->tc = NULL; 1885 currState->delayed = false; 1886 delete currState; 1887 } else { 1888 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1) 1889 panic("Max. number of lookups already reached in table walk\n"); 1890 // Need to perform additional lookups 1891 stateQueues[currState->longDesc.lookupLevel].push_back(currState); 1892 } 1893 currState = NULL; 1894} 1895 1896 1897void 1898TableWalker::nextWalk(ThreadContext *tc) 1899{ 1900 if (pendingQueue.size()) 1901 schedule(doProcessEvent, clockEdge(Cycles(1))); 1902 else 1903 completeDrain(); 1904} 1905 1906bool 1907TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, 1908 Request::Flags flags, int queueIndex, Event *event, 1909 void (TableWalker::*doDescriptor)()) 1910{ 1911 bool isTiming = currState->timing; 1912 1913 // do the requests for the page table descriptors have to go through the 1914 // second stage MMU 1915 if (currState->stage2Req) { 1916 Fault fault; 1917 flags = flags | TLB::MustBeOne; 1918 1919 if (isTiming) { 1920 Stage2MMU::Stage2Translation *tran = new 1921 Stage2MMU::Stage2Translation(*stage2Mmu, data, event, 1922 currState->vaddr); 1923 currState->stage2Tran = tran; 1924 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes, 1925 flags); 1926 fault = tran->fault; 1927 } else { 1928 fault = stage2Mmu->readDataUntimed(currState->tc, 1929 currState->vaddr, descAddr, data, numBytes, flags, 1930 currState->functional); 1931 } 1932 1933 if (fault != NoFault) { 1934 currState->fault = fault; 1935 } 1936 if (isTiming) { 1937 if (queueIndex >= 0) { 1938 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1939 stateQueues[queueIndex].size()); 1940 stateQueues[queueIndex].push_back(currState); 1941 currState = NULL; 1942 } 1943 } else { 1944 (this->*doDescriptor)(); 1945 } 1946 } else { 1947 if (isTiming) { 1948 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data, 1949 currState->tc->getCpuPtr()->clockPeriod(),flags); 1950 if (queueIndex >= 0) { 1951 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1952 stateQueues[queueIndex].size()); 1953 stateQueues[queueIndex].push_back(currState); 1954 currState = NULL; 1955 } 1956 } else if (!currState->functional) { 1957 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data, 1958 currState->tc->getCpuPtr()->clockPeriod(), flags); 1959 (this->*doDescriptor)(); 1960 } else { 1961 RequestPtr req = new Request(descAddr, numBytes, flags, masterId); 1962 req->taskId(ContextSwitchTaskId::DMA); 1963 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 1964 pkt->dataStatic(data); 1965 port->sendFunctional(pkt); 1966 (this->*doDescriptor)(); 1967 delete req; 1968 delete pkt; 1969 } 1970 } 1971 return (isTiming); 1972} 1973 1974void 1975TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor) 1976{ 1977 TlbEntry te; 1978 1979 // Create and fill a new page table entry 1980 te.valid = true; 1981 te.longDescFormat = longDescriptor; 1982 te.isHyp = currState->isHyp; 1983 te.asid = currState->asid; 1984 te.vmid = currState->vmid; 1985 te.N = descriptor.offsetBits(); 1986 te.vpn = currState->vaddr >> te.N; 1987 te.size = (1<<te.N) - 1; 1988 te.pfn = descriptor.pfn(); 1989 te.domain = descriptor.domain(); 1990 te.lookupLevel = descriptor.lookupLevel; 1991 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2; 1992 te.nstid = !currState->isSecure; 1993 te.xn = descriptor.xn(); 1994 if (currState->aarch64) 1995 te.el = currState->el; 1996 else 1997 te.el = 1; 1998 1999 statPageSizes[pageSizeNtoStatBin(te.N)]++; 2000 statRequestOrigin[COMPLETED][currState->isFetch]++; 2001 2002 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries 2003 // as global 2004 te.global = descriptor.global(currState) || isStage2; 2005 if (longDescriptor) { 2006 LongDescriptor lDescriptor = 2007 dynamic_cast<LongDescriptor &>(descriptor); 2008 2009 te.xn |= currState->xnTable; 2010 te.pxn = currState->pxnTable || lDescriptor.pxn(); 2011 if (isStage2) { 2012 // this is actually the HAP field, but its stored in the same bit 2013 // possitions as the AP field in a stage 1 translation. 2014 te.hap = lDescriptor.ap(); 2015 } else { 2016 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) | 2017 (currState->userTable && (descriptor.ap() & 0x1)); 2018 } 2019 if (currState->aarch64) 2020 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(), 2021 currState->longDesc.sh()); 2022 else 2023 memAttrsLPAE(currState->tc, te, lDescriptor); 2024 } else { 2025 te.ap = descriptor.ap(); 2026 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(), 2027 descriptor.shareable()); 2028 } 2029 2030 // Debug output 2031 DPRINTF(TLB, descriptor.dbgHeader().c_str()); 2032 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n", 2033 te.N, te.pfn, te.size, te.global, te.valid); 2034 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d " 2035 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn, 2036 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp, 2037 te.nonCacheable, te.ns); 2038 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n", 2039 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()), 2040 descriptor.getRawData()); 2041 2042 // Insert the entry into the TLB 2043 tlb->insert(currState->vaddr, te); 2044 if (!currState->timing) { 2045 currState->tc = NULL; 2046 currState->req = NULL; 2047 } 2048} 2049 2050ArmISA::TableWalker * 2051ArmTableWalkerParams::create() 2052{ 2053 return new ArmISA::TableWalker(this); 2054} 2055 2056LookupLevel 2057TableWalker::toLookupLevel(uint8_t lookup_level_as_int) 2058{ 2059 switch (lookup_level_as_int) { 2060 case L1: 2061 return L1; 2062 case L2: 2063 return L2; 2064 case L3: 2065 return L3; 2066 default: 2067 panic("Invalid lookup level conversion"); 2068 } 2069} 2070 2071/* this method keeps track of the table walker queue's residency, so 2072 * needs to be called whenever requests start and complete. */ 2073void 2074TableWalker::pendingChange() 2075{ 2076 unsigned n = pendingQueue.size(); 2077 if ((currState != NULL) && (currState != pendingQueue.front())) { 2078 ++n; 2079 } 2080 2081 if (n != pendingReqs) { 2082 Tick now = curTick(); 2083 statPendingWalks.sample(pendingReqs, now - pendingChangeTick); 2084 pendingReqs = n; 2085 pendingChangeTick = now; 2086 } 2087} 2088
|
2089Fault 2090TableWalker::testWalk(Addr pa, Addr size, TlbEntry::DomainType domain, 2091 LookupLevel lookup_level) 2092{ 2093 return tlb->testWalk(pa, size, currState->vaddr, currState->isSecure, 2094 currState->mode, domain, lookup_level); 2095} 2096 2097 |
2098uint8_t 2099TableWalker::pageSizeNtoStatBin(uint8_t N) 2100{ 2101 /* for statPageSizes */ 2102 switch(N) { 2103 case 12: return 0; // 4K 2104 case 14: return 1; // 16K (using 16K granule in v8-64) 2105 case 16: return 2; // 64K 2106 case 20: return 3; // 1M 2107 case 21: return 4; // 2M-LPAE 2108 case 24: return 5; // 16M 2109 case 25: return 6; // 32M (using 16K granule in v8-64) 2110 case 29: return 7; // 512M (using 64K granule in v8-64) 2111 case 30: return 8; // 1G-LPAE 2112 default: 2113 panic("unknown page size"); 2114 return 255; 2115 } 2116} 2117 2118void 2119TableWalker::regStats() 2120{ 2121 statWalks 2122 .name(name() + ".walks") 2123 .desc("Table walker walks requested") 2124 ; 2125 2126 statWalksShortDescriptor 2127 .name(name() + ".walksShort") 2128 .desc("Table walker walks initiated with short descriptors") 2129 .flags(Stats::nozero) 2130 ; 2131 2132 statWalksLongDescriptor 2133 .name(name() + ".walksLong") 2134 .desc("Table walker walks initiated with long descriptors") 2135 .flags(Stats::nozero) 2136 ; 2137 2138 statWalksShortTerminatedAtLevel 2139 .init(2) 2140 .name(name() + ".walksShortTerminationLevel") 2141 .desc("Level at which table walker walks " 2142 "with short descriptors terminate") 2143 .flags(Stats::nozero) 2144 ; 2145 statWalksShortTerminatedAtLevel.subname(0, "Level1"); 2146 statWalksShortTerminatedAtLevel.subname(1, "Level2"); 2147 2148 statWalksLongTerminatedAtLevel 2149 .init(4) 2150 .name(name() + ".walksLongTerminationLevel") 2151 .desc("Level at which table walker walks " 2152 "with long descriptors terminate") 2153 .flags(Stats::nozero) 2154 ; 2155 statWalksLongTerminatedAtLevel.subname(0, "Level0"); 2156 statWalksLongTerminatedAtLevel.subname(1, "Level1"); 2157 statWalksLongTerminatedAtLevel.subname(2, "Level2"); 2158 statWalksLongTerminatedAtLevel.subname(3, "Level3"); 2159 2160 statSquashedBefore 2161 .name(name() + ".walksSquashedBefore") 2162 .desc("Table walks squashed before starting") 2163 .flags(Stats::nozero) 2164 ; 2165 2166 statSquashedAfter 2167 .name(name() + ".walksSquashedAfter") 2168 .desc("Table walks squashed after completion") 2169 .flags(Stats::nozero) 2170 ; 2171 2172 statWalkWaitTime 2173 .init(16) 2174 .name(name() + ".walkWaitTime") 2175 .desc("Table walker wait (enqueue to first request) latency") 2176 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2177 ; 2178 2179 statWalkServiceTime 2180 .init(16) 2181 .name(name() + ".walkCompletionTime") 2182 .desc("Table walker service (enqueue to completion) latency") 2183 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2184 ; 2185 2186 statPendingWalks 2187 .init(16) 2188 .name(name() + ".walksPending") 2189 .desc("Table walker pending requests distribution") 2190 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan) 2191 ; 2192 2193 statPageSizes // see DDI 0487A D4-1661 2194 .init(9) 2195 .name(name() + ".walkPageSizes") 2196 .desc("Table walker page sizes translated") 2197 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero) 2198 ; 2199 statPageSizes.subname(0, "4K"); 2200 statPageSizes.subname(1, "16K"); 2201 statPageSizes.subname(2, "64K"); 2202 statPageSizes.subname(3, "1M"); 2203 statPageSizes.subname(4, "2M"); 2204 statPageSizes.subname(5, "16M"); 2205 statPageSizes.subname(6, "32M"); 2206 statPageSizes.subname(7, "512M"); 2207 statPageSizes.subname(8, "1G"); 2208 2209 statRequestOrigin 2210 .init(2,2) // Instruction/Data, requests/completed 2211 .name(name() + ".walkRequestOrigin") 2212 .desc("Table walker requests started/completed, data/inst") 2213 .flags(Stats::total) 2214 ; 2215 statRequestOrigin.subname(0,"Requested"); 2216 statRequestOrigin.subname(1,"Completed"); 2217 statRequestOrigin.ysubname(0,"Data"); 2218 statRequestOrigin.ysubname(1,"Inst"); 2219}
|