table_walker.cc revision 11181:4daf60db14d7
11722SN/A/* 21722SN/A * Copyright (c) 2010, 2012-2015 ARM Limited 31722SN/A * All rights reserved 41722SN/A * 58332SN/A * The license below extends only to copyright in the software and shall 68332SN/A * not be construed as granting a license to any other intellectual 71722SN/A * property including but not limited to intellectual property relating 81722SN/A * to a hardware implementation of the functionality of the software 91722SN/A * licensed hereunder. You may use the software subject to the license 101722SN/A * terms below provided that you ensure that this notice is replicated 111722SN/A * unmodified and in its entirety in all distributions of the software, 121722SN/A * modified or unmodified, in source code or in binary form. 131722SN/A * 141722SN/A * Redistribution and use in source and binary forms, with or without 151722SN/A * modification, are permitted provided that the following conditions are 161722SN/A * met: redistributions of source code must retain the above copyright 171722SN/A * notice, this list of conditions and the following disclaimer; 181722SN/A * redistributions in binary form must reproduce the above copyright 191722SN/A * notice, this list of conditions and the following disclaimer in the 201722SN/A * documentation and/or other materials provided with the distribution; 211722SN/A * neither the name of the copyright holders nor the names of its 221722SN/A * contributors may be used to endorse or promote products derived from 231722SN/A * this software without specific prior written permission. 241722SN/A * 251722SN/A * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 261722SN/A * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 271722SN/A * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 281722SN/A * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 291722SN/A * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 301722SN/A * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 311722SN/A * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 321722SN/A * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 331722SN/A * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 341722SN/A * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 351722SN/A * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 361722SN/A * 371722SN/A * Authors: Ali Saidi 381722SN/A * Giacomo Gabrielli 391722SN/A */ 401722SN/A#include "arch/arm/table_walker.hh" 411722SN/A 421722SN/A#include <memory> 431722SN/A 441722SN/A#include "arch/arm/faults.hh" 451722SN/A#include "arch/arm/stage2_mmu.hh" 461722SN/A#include "arch/arm/system.hh" 471722SN/A#include "arch/arm/tlb.hh" 481722SN/A#include "cpu/base.hh" 491722SN/A#include "cpu/thread_context.hh" 501722SN/A#include "debug/Checkpoint.hh" 511722SN/A#include "debug/Drain.hh" 521722SN/A#include "debug/TLB.hh" 531722SN/A#include "debug/TLBVerbose.hh" 541722SN/A#include "dev/dma_device.hh" 551722SN/A#include "sim/system.hh" 561722SN/A 571722SN/Ausing namespace ArmISA; 581722SN/A 591722SN/ATableWalker::TableWalker(const Params *p) 601722SN/A : MemObject(p), 611722SN/A stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId), 621722SN/A isStage2(p->is_stage2), tlb(NULL), 631722SN/A currState(NULL), pending(false), 641722SN/A numSquashable(p->num_squash_per_cycle), 651722SN/A pendingReqs(0), 661722SN/A pendingChangeTick(curTick()), 671722SN/A doL1DescEvent(this), doL2DescEvent(this), 681722SN/A doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this), 691722SN/A doL3LongDescEvent(this), 701722SN/A doProcessEvent(this) 711722SN/A{ 721722SN/A sctlr = 0; 731722SN/A 741722SN/A // Cache system-level properties 751722SN/A if (FullSystem) { 761722SN/A ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys); 771722SN/A assert(armSys); 781722SN/A haveSecurity = armSys->haveSecurity(); 791722SN/A _haveLPAE = armSys->haveLPAE(); 801722SN/A _haveVirtualization = armSys->haveVirtualization(); 811722SN/A physAddrRange = armSys->physAddrRange(); 821722SN/A _haveLargeAsid64 = armSys->haveLargeAsid64(); 831722SN/A } else { 841722SN/A haveSecurity = _haveLPAE = _haveVirtualization = false; 851722SN/A _haveLargeAsid64 = false; 861722SN/A physAddrRange = 32; 871722SN/A } 881722SN/A 891722SN/A} 901722SN/A 911722SN/ATableWalker::~TableWalker() 921722SN/A{ 931722SN/A ; 941722SN/A} 951722SN/A 961722SN/Avoid 971722SN/ATableWalker::setMMU(Stage2MMU *m, MasterID master_id) 981722SN/A{ 991722SN/A stage2Mmu = m; 1001722SN/A port = &m->getPort(); 1011722SN/A masterId = master_id; 1021722SN/A} 1031722SN/A 1041722SN/Avoid 1051722SN/ATableWalker::init() 1061722SN/A{ 1071722SN/A fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n"); 1081722SN/A fatal_if(!port, "Table walker must have a valid port\n"); 1091722SN/A fatal_if(!tlb, "Table walker must have a valid TLB\n"); 1101722SN/A} 1111722SN/A 1121722SN/ABaseMasterPort& 1131722SN/ATableWalker::getMasterPort(const std::string &if_name, PortID idx) 1141722SN/A{ 1151722SN/A if (if_name == "port") { 1161722SN/A if (!isStage2) { 1171722SN/A return *port; 1181722SN/A } else { 1191722SN/A fatal("Cannot access table walker port through stage-two walker\n"); 1201722SN/A } 1211722SN/A } 1221722SN/A return MemObject::getMasterPort(if_name, idx); 1231722SN/A} 1241722SN/A 1251722SN/ATableWalker::WalkerState::WalkerState() : 1261722SN/A tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr), 1271722SN/A asid(0), vmid(0), isHyp(false), transState(nullptr), 1281722SN/A vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false), 1291722SN/A secureLookup(false), rwTable(false), userTable(false), xnTable(false), 1301722SN/A pxnTable(false), stage2Req(false), doingStage2(false), 1311722SN/A stage2Tran(nullptr), timing(false), functional(false), 1321722SN/A mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc), 1331722SN/A delayed(false), tableWalker(nullptr) 1341722SN/A{ 1351722SN/A} 1361722SN/A 1371722SN/Avoid 1381722SN/ATableWalker::completeDrain() 1391722SN/A{ 1401722SN/A if (drainState() == DrainState::Draining && 1411722SN/A stateQueues[L1].empty() && stateQueues[L2].empty() && 1421722SN/A pendingQueue.empty()) { 1431722SN/A 1441722SN/A DPRINTF(Drain, "TableWalker done draining, processing drain event\n"); 1451722SN/A signalDrainDone(); 1461722SN/A } 1471722SN/A} 1481722SN/A 1491722SN/ADrainState 1501722SN/ATableWalker::drain() 1511722SN/A{ 1521722SN/A bool state_queues_not_empty = false; 1531722SN/A 1541722SN/A for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) { 1551722SN/A if (!stateQueues[i].empty()) { 1561722SN/A state_queues_not_empty = true; 1571722SN/A break; 1581722SN/A } 1591722SN/A } 1601722SN/A 1611722SN/A if (state_queues_not_empty || pendingQueue.size()) { 1621722SN/A DPRINTF(Drain, "TableWalker not drained\n"); 1631722SN/A return DrainState::Draining; 1641722SN/A } else { 1651722SN/A DPRINTF(Drain, "TableWalker free, no need to drain\n"); 1661722SN/A return DrainState::Drained; 1671722SN/A } 1681722SN/A} 1691722SN/A 1701722SN/Avoid 1711722SN/ATableWalker::drainResume() 1721722SN/A{ 1731722SN/A if (params()->sys->isTimingMode() && currState) { 1741722SN/A delete currState; 1751722SN/A currState = NULL; 1761722SN/A pendingChange(); 1771722SN/A } 1781722SN/A} 1791722SN/A 1801722SN/AFault 1811722SN/ATableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid, 1821722SN/A uint8_t _vmid, bool _isHyp, TLB::Mode _mode, 1831722SN/A TLB::Translation *_trans, bool _timing, bool _functional, 1841722SN/A bool secure, TLB::ArmTranslationType tranType) 1851722SN/A{ 1861722SN/A assert(!(_functional && _timing)); 1871722SN/A ++statWalks; 1881722SN/A 1891722SN/A WalkerState *savedCurrState = NULL; 1901722SN/A 1911722SN/A if (!currState && !_functional) { 1921722SN/A // For atomic mode, a new WalkerState instance should be only created 1931722SN/A // once per TLB. For timing mode, a new instance is generated for every 1941722SN/A // TLB miss. 1951722SN/A DPRINTF(TLBVerbose, "creating new instance of WalkerState\n"); 1961722SN/A 1971722SN/A currState = new WalkerState(); 198 currState->tableWalker = this; 199 } else if (_functional) { 200 // If we are mixing functional mode with timing (or even 201 // atomic), we need to to be careful and clean up after 202 // ourselves to not risk getting into an inconsistent state. 203 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n"); 204 savedCurrState = currState; 205 currState = new WalkerState(); 206 currState->tableWalker = this; 207 } else if (_timing) { 208 // This is a translation that was completed and then faulted again 209 // because some underlying parameters that affect the translation 210 // changed out from under us (e.g. asid). It will either be a 211 // misprediction, in which case nothing will happen or we'll use 212 // this fault to re-execute the faulting instruction which should clean 213 // up everything. 214 if (currState->vaddr_tainted == _req->getVaddr()) { 215 ++statSquashedBefore; 216 return std::make_shared<ReExec>(); 217 } 218 } 219 pendingChange(); 220 221 currState->startTime = curTick(); 222 currState->tc = _tc; 223 currState->aarch64 = opModeIs64(currOpMode(_tc)); 224 currState->el = currEL(_tc); 225 currState->transState = _trans; 226 currState->req = _req; 227 currState->fault = NoFault; 228 currState->asid = _asid; 229 currState->vmid = _vmid; 230 currState->isHyp = _isHyp; 231 currState->timing = _timing; 232 currState->functional = _functional; 233 currState->mode = _mode; 234 currState->tranType = tranType; 235 currState->isSecure = secure; 236 currState->physAddrRange = physAddrRange; 237 238 /** @todo These should be cached or grabbed from cached copies in 239 the TLB, all these miscreg reads are expensive */ 240 currState->vaddr_tainted = currState->req->getVaddr(); 241 if (currState->aarch64) 242 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted, 243 currState->tc, currState->el); 244 else 245 currState->vaddr = currState->vaddr_tainted; 246 247 if (currState->aarch64) { 248 switch (currState->el) { 249 case EL0: 250 case EL1: 251 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1); 252 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1); 253 break; 254 // @todo: uncomment this to enable Virtualization 255 // case EL2: 256 // assert(haveVirtualization); 257 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2); 258 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2); 259 // break; 260 case EL3: 261 assert(haveSecurity); 262 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3); 263 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3); 264 break; 265 default: 266 panic("Invalid exception level"); 267 break; 268 } 269 } else { 270 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 271 MISCREG_SCTLR, currState->tc, !currState->isSecure)); 272 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 273 MISCREG_TTBCR, currState->tc, !currState->isSecure)); 274 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR); 275 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR); 276 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR); 277 } 278 sctlr = currState->sctlr; 279 280 currState->isFetch = (currState->mode == TLB::Execute); 281 currState->isWrite = (currState->mode == TLB::Write); 282 283 statRequestOrigin[REQUESTED][currState->isFetch]++; 284 285 // We only do a second stage of translation if we're not secure, or in 286 // hyp mode, the second stage MMU is enabled, and this table walker 287 // instance is the first stage. 288 currState->doingStage2 = false; 289 // @todo: for now disable this in AArch64 (HCR is not set) 290 currState->stage2Req = !currState->aarch64 && currState->hcr.vm && 291 !isStage2 && !currState->isSecure && !currState->isHyp; 292 293 bool long_desc_format = currState->aarch64 || 294 (_haveLPAE && currState->ttbcr.eae) || 295 _isHyp || isStage2; 296 297 if (long_desc_format) { 298 // Helper variables used for hierarchical permissions 299 currState->secureLookup = currState->isSecure; 300 currState->rwTable = true; 301 currState->userTable = true; 302 currState->xnTable = false; 303 currState->pxnTable = false; 304 305 ++statWalksLongDescriptor; 306 } else { 307 ++statWalksShortDescriptor; 308 } 309 310 if (!currState->timing) { 311 Fault fault = NoFault; 312 if (currState->aarch64) 313 fault = processWalkAArch64(); 314 else if (long_desc_format) 315 fault = processWalkLPAE(); 316 else 317 fault = processWalk(); 318 319 // If this was a functional non-timing access restore state to 320 // how we found it. 321 if (currState->functional) { 322 delete currState; 323 currState = savedCurrState; 324 } 325 return fault; 326 } 327 328 if (pending || pendingQueue.size()) { 329 pendingQueue.push_back(currState); 330 currState = NULL; 331 pendingChange(); 332 } else { 333 pending = true; 334 pendingChange(); 335 if (currState->aarch64) 336 return processWalkAArch64(); 337 else if (long_desc_format) 338 return processWalkLPAE(); 339 else 340 return processWalk(); 341 } 342 343 return NoFault; 344} 345 346void 347TableWalker::processWalkWrapper() 348{ 349 assert(!currState); 350 assert(pendingQueue.size()); 351 pendingChange(); 352 currState = pendingQueue.front(); 353 354 ExceptionLevel target_el = EL0; 355 if (currState->aarch64) 356 target_el = currEL(currState->tc); 357 else 358 target_el = EL1; 359 360 // Check if a previous walk filled this request already 361 // @TODO Should this always be the TLB or should we look in the stage2 TLB? 362 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid, 363 currState->vmid, currState->isHyp, currState->isSecure, true, false, 364 target_el); 365 366 // Check if we still need to have a walk for this request. If the requesting 367 // instruction has been squashed, or a previous walk has filled the TLB with 368 // a match, we just want to get rid of the walk. The latter could happen 369 // when there are multiple outstanding misses to a single page and a 370 // previous request has been successfully translated. 371 if (!currState->transState->squashed() && !te) { 372 // We've got a valid request, lets process it 373 pending = true; 374 pendingQueue.pop_front(); 375 // Keep currState in case one of the processWalk... calls NULLs it 376 WalkerState *curr_state_copy = currState; 377 Fault f; 378 if (currState->aarch64) 379 f = processWalkAArch64(); 380 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2) 381 f = processWalkLPAE(); 382 else 383 f = processWalk(); 384 385 if (f != NoFault) { 386 curr_state_copy->transState->finish(f, curr_state_copy->req, 387 curr_state_copy->tc, curr_state_copy->mode); 388 389 delete curr_state_copy; 390 } 391 return; 392 } 393 394 395 // If the instruction that we were translating for has been 396 // squashed we shouldn't bother. 397 unsigned num_squashed = 0; 398 ThreadContext *tc = currState->tc; 399 while ((num_squashed < numSquashable) && currState && 400 (currState->transState->squashed() || te)) { 401 pendingQueue.pop_front(); 402 num_squashed++; 403 statSquashedBefore++; 404 405 DPRINTF(TLB, "Squashing table walk for address %#x\n", 406 currState->vaddr_tainted); 407 408 if (currState->transState->squashed()) { 409 // finish the translation which will delete the translation object 410 currState->transState->finish( 411 std::make_shared<UnimpFault>("Squashed Inst"), 412 currState->req, currState->tc, currState->mode); 413 } else { 414 // translate the request now that we know it will work 415 statWalkServiceTime.sample(curTick() - currState->startTime); 416 tlb->translateTiming(currState->req, currState->tc, 417 currState->transState, currState->mode); 418 419 } 420 421 // delete the current request 422 delete currState; 423 424 // peak at the next one 425 if (pendingQueue.size()) { 426 currState = pendingQueue.front(); 427 te = tlb->lookup(currState->vaddr, currState->asid, 428 currState->vmid, currState->isHyp, currState->isSecure, true, 429 false, target_el); 430 } else { 431 // Terminate the loop, nothing more to do 432 currState = NULL; 433 } 434 } 435 pendingChange(); 436 437 // if we still have pending translations, schedule more work 438 nextWalk(tc); 439 currState = NULL; 440} 441 442Fault 443TableWalker::processWalk() 444{ 445 Addr ttbr = 0; 446 447 // If translation isn't enabled, we shouldn't be here 448 assert(currState->sctlr.m || isStage2); 449 450 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n", 451 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31, 452 32 - currState->ttbcr.n)); 453 454 statWalkWaitTime.sample(curTick() - currState->startTime); 455 456 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31, 457 32 - currState->ttbcr.n)) { 458 DPRINTF(TLB, " - Selecting TTBR0\n"); 459 // Check if table walk is allowed when Security Extensions are enabled 460 if (haveSecurity && currState->ttbcr.pd0) { 461 if (currState->isFetch) 462 return std::make_shared<PrefetchAbort>( 463 currState->vaddr_tainted, 464 ArmFault::TranslationLL + L1, 465 isStage2, 466 ArmFault::VmsaTran); 467 else 468 return std::make_shared<DataAbort>( 469 currState->vaddr_tainted, 470 TlbEntry::DomainType::NoAccess, currState->isWrite, 471 ArmFault::TranslationLL + L1, isStage2, 472 ArmFault::VmsaTran); 473 } 474 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 475 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 476 } else { 477 DPRINTF(TLB, " - Selecting TTBR1\n"); 478 // Check if table walk is allowed when Security Extensions are enabled 479 if (haveSecurity && currState->ttbcr.pd1) { 480 if (currState->isFetch) 481 return std::make_shared<PrefetchAbort>( 482 currState->vaddr_tainted, 483 ArmFault::TranslationLL + L1, 484 isStage2, 485 ArmFault::VmsaTran); 486 else 487 return std::make_shared<DataAbort>( 488 currState->vaddr_tainted, 489 TlbEntry::DomainType::NoAccess, currState->isWrite, 490 ArmFault::TranslationLL + L1, isStage2, 491 ArmFault::VmsaTran); 492 } 493 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 494 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 495 currState->ttbcr.n = 0; 496 } 497 498 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) | 499 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2); 500 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr, 501 currState->isSecure ? "s" : "ns"); 502 503 // Trickbox address check 504 Fault f; 505 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure, 506 currState->vaddr, sizeof(uint32_t), currState->isFetch, 507 currState->isWrite, TlbEntry::DomainType::NoAccess, L1); 508 if (f) { 509 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 510 if (currState->timing) { 511 pending = false; 512 nextWalk(currState->tc); 513 currState = NULL; 514 } else { 515 currState->tc = NULL; 516 currState->req = NULL; 517 } 518 return f; 519 } 520 521 Request::Flags flag = Request::PT_WALK; 522 if (currState->sctlr.c == 0) { 523 flag.set(Request::UNCACHEABLE); 524 } 525 526 if (currState->isSecure) { 527 flag.set(Request::SECURE); 528 } 529 530 bool delayed; 531 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data, 532 sizeof(uint32_t), flag, L1, &doL1DescEvent, 533 &TableWalker::doL1Descriptor); 534 if (!delayed) { 535 f = currState->fault; 536 } 537 538 return f; 539} 540 541Fault 542TableWalker::processWalkLPAE() 543{ 544 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr; 545 int tsz, n; 546 LookupLevel start_lookup_level = L1; 547 548 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n", 549 currState->vaddr_tainted, currState->ttbcr); 550 551 statWalkWaitTime.sample(curTick() - currState->startTime); 552 553 Request::Flags flag = Request::PT_WALK; 554 if (currState->isSecure) 555 flag.set(Request::SECURE); 556 557 // work out which base address register to use, if in hyp mode we always 558 // use HTTBR 559 if (isStage2) { 560 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n"); 561 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR); 562 tsz = sext<4>(currState->vtcr.t0sz); 563 start_lookup_level = currState->vtcr.sl0 ? L1 : L2; 564 } else if (currState->isHyp) { 565 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n"); 566 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR); 567 tsz = currState->htcr.t0sz; 568 } else { 569 assert(_haveLPAE && currState->ttbcr.eae); 570 571 // Determine boundaries of TTBR0/1 regions 572 if (currState->ttbcr.t0sz) 573 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1; 574 else if (currState->ttbcr.t1sz) 575 ttbr0_max = (1ULL << 32) - 576 (1ULL << (32 - currState->ttbcr.t1sz)) - 1; 577 else 578 ttbr0_max = (1ULL << 32) - 1; 579 if (currState->ttbcr.t1sz) 580 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz)); 581 else 582 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz)); 583 584 // The following code snippet selects the appropriate translation table base 585 // address (TTBR0 or TTBR1) and the appropriate starting lookup level 586 // depending on the address range supported by the translation table (ARM 587 // ARM issue C B3.6.4) 588 if (currState->vaddr <= ttbr0_max) { 589 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n"); 590 // Check if table walk is allowed 591 if (currState->ttbcr.epd0) { 592 if (currState->isFetch) 593 return std::make_shared<PrefetchAbort>( 594 currState->vaddr_tainted, 595 ArmFault::TranslationLL + L1, 596 isStage2, 597 ArmFault::LpaeTran); 598 else 599 return std::make_shared<DataAbort>( 600 currState->vaddr_tainted, 601 TlbEntry::DomainType::NoAccess, 602 currState->isWrite, 603 ArmFault::TranslationLL + L1, 604 isStage2, 605 ArmFault::LpaeTran); 606 } 607 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 608 MISCREG_TTBR0, currState->tc, !currState->isSecure)); 609 tsz = currState->ttbcr.t0sz; 610 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB 611 start_lookup_level = L2; 612 } else if (currState->vaddr >= ttbr1_min) { 613 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n"); 614 // Check if table walk is allowed 615 if (currState->ttbcr.epd1) { 616 if (currState->isFetch) 617 return std::make_shared<PrefetchAbort>( 618 currState->vaddr_tainted, 619 ArmFault::TranslationLL + L1, 620 isStage2, 621 ArmFault::LpaeTran); 622 else 623 return std::make_shared<DataAbort>( 624 currState->vaddr_tainted, 625 TlbEntry::DomainType::NoAccess, 626 currState->isWrite, 627 ArmFault::TranslationLL + L1, 628 isStage2, 629 ArmFault::LpaeTran); 630 } 631 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked( 632 MISCREG_TTBR1, currState->tc, !currState->isSecure)); 633 tsz = currState->ttbcr.t1sz; 634 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB 635 start_lookup_level = L2; 636 } else { 637 // Out of boundaries -> translation fault 638 if (currState->isFetch) 639 return std::make_shared<PrefetchAbort>( 640 currState->vaddr_tainted, 641 ArmFault::TranslationLL + L1, 642 isStage2, 643 ArmFault::LpaeTran); 644 else 645 return std::make_shared<DataAbort>( 646 currState->vaddr_tainted, 647 TlbEntry::DomainType::NoAccess, 648 currState->isWrite, ArmFault::TranslationLL + L1, 649 isStage2, ArmFault::LpaeTran); 650 } 651 652 } 653 654 // Perform lookup (ARM ARM issue C B3.6.6) 655 if (start_lookup_level == L1) { 656 n = 5 - tsz; 657 desc_addr = mbits(ttbr, 39, n) | 658 (bits(currState->vaddr, n + 26, 30) << 3); 659 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 660 desc_addr, currState->isSecure ? "s" : "ns"); 661 } else { 662 // Skip first-level lookup 663 n = (tsz >= 2 ? 14 - tsz : 12); 664 desc_addr = mbits(ttbr, 39, n) | 665 (bits(currState->vaddr, n + 17, 21) << 3); 666 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n", 667 desc_addr, currState->isSecure ? "s" : "ns"); 668 } 669 670 // Trickbox address check 671 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure, 672 currState->vaddr, sizeof(uint64_t), currState->isFetch, 673 currState->isWrite, TlbEntry::DomainType::NoAccess, 674 start_lookup_level); 675 if (f) { 676 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 677 if (currState->timing) { 678 pending = false; 679 nextWalk(currState->tc); 680 currState = NULL; 681 } else { 682 currState->tc = NULL; 683 currState->req = NULL; 684 } 685 return f; 686 } 687 688 if (currState->sctlr.c == 0) { 689 flag.set(Request::UNCACHEABLE); 690 } 691 692 currState->longDesc.lookupLevel = start_lookup_level; 693 currState->longDesc.aarch64 = false; 694 currState->longDesc.grainSize = Grain4KB; 695 696 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent 697 : (Event *) &doL2LongDescEvent; 698 699 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data, 700 sizeof(uint64_t), flag, start_lookup_level, 701 event, &TableWalker::doLongDescriptor); 702 if (!delayed) { 703 f = currState->fault; 704 } 705 706 return f; 707} 708 709unsigned 710TableWalker::adjustTableSizeAArch64(unsigned tsz) 711{ 712 if (tsz < 25) 713 return 25; 714 if (tsz > 48) 715 return 48; 716 return tsz; 717} 718 719bool 720TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange) 721{ 722 return (currPhysAddrRange != MaxPhysAddrRange && 723 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange)); 724} 725 726Fault 727TableWalker::processWalkAArch64() 728{ 729 assert(currState->aarch64); 730 731 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n", 732 currState->vaddr_tainted, currState->tcr); 733 734 static const GrainSize GrainMapDefault[] = 735 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain }; 736 static const GrainSize GrainMap_EL1_tg1[] = 737 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB }; 738 739 statWalkWaitTime.sample(curTick() - currState->startTime); 740 741 // Determine TTBR, table size, granule size and phys. address range 742 Addr ttbr = 0; 743 int tsz = 0, ps = 0; 744 GrainSize tg = Grain4KB; // grain size computed from tg* field 745 bool fault = false; 746 switch (currState->el) { 747 case EL0: 748 case EL1: 749 switch (bits(currState->vaddr, 63,48)) { 750 case 0: 751 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 752 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1); 753 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 754 tg = GrainMapDefault[currState->tcr.tg0]; 755 if (bits(currState->vaddr, 63, tsz) != 0x0 || 756 currState->tcr.epd0) 757 fault = true; 758 break; 759 case 0xffff: 760 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n"); 761 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1); 762 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz); 763 tg = GrainMap_EL1_tg1[currState->tcr.tg1]; 764 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) || 765 currState->tcr.epd1) 766 fault = true; 767 break; 768 default: 769 // top two bytes must be all 0s or all 1s, else invalid addr 770 fault = true; 771 } 772 ps = currState->tcr.ips; 773 break; 774 case EL2: 775 case EL3: 776 switch(bits(currState->vaddr, 63,48)) { 777 case 0: 778 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n"); 779 if (currState->el == EL2) 780 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2); 781 else 782 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3); 783 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz); 784 tg = GrainMapDefault[currState->tcr.tg0]; 785 break; 786 default: 787 // invalid addr if top two bytes are not all 0s 788 fault = true; 789 } 790 ps = currState->tcr.ips; 791 break; 792 } 793 794 if (fault) { 795 Fault f; 796 if (currState->isFetch) 797 f = std::make_shared<PrefetchAbort>( 798 currState->vaddr_tainted, 799 ArmFault::TranslationLL + L0, isStage2, 800 ArmFault::LpaeTran); 801 else 802 f = std::make_shared<DataAbort>( 803 currState->vaddr_tainted, 804 TlbEntry::DomainType::NoAccess, 805 currState->isWrite, 806 ArmFault::TranslationLL + L0, 807 isStage2, ArmFault::LpaeTran); 808 809 if (currState->timing) { 810 pending = false; 811 nextWalk(currState->tc); 812 currState = NULL; 813 } else { 814 currState->tc = NULL; 815 currState->req = NULL; 816 } 817 return f; 818 819 } 820 821 if (tg == ReservedGrain) { 822 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION " 823 "DEFINED behavior takes this to mean 4KB granules\n"); 824 tg = Grain4KB; 825 } 826 827 int stride = tg - 3; 828 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS; 829 830 // Determine starting lookup level 831 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library 832 // in ARM DDI 0487A. These table values correspond to the cascading tests 833 // to compute the lookup level and are of the form 834 // (grain_size + N*stride), for N = {1, 2, 3}. 835 // A value of 64 will never succeed and a value of 0 will always succeed. 836 { 837 struct GrainMap { 838 GrainSize grain_size; 839 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS]; 840 }; 841 static const GrainMap GM[] = { 842 { Grain4KB, { 39, 30, 0, 0 } }, 843 { Grain16KB, { 47, 36, 25, 0 } }, 844 { Grain64KB, { 64, 42, 29, 0 } } 845 }; 846 847 const unsigned *lookup = NULL; // points to a lookup_level_cutoff 848 849 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[] 850 if (tg == GM[i].grain_size) { 851 lookup = GM[i].lookup_level_cutoff; 852 break; 853 } 854 } 855 assert(lookup); 856 857 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) { 858 if (tsz > lookup[L]) { 859 start_lookup_level = (LookupLevel) L; 860 break; 861 } 862 } 863 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS, 864 "Table walker couldn't find lookup level\n"); 865 } 866 867 // Determine table base address 868 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg; 869 Addr base_addr = mbits(ttbr, 47, base_addr_lo); 870 871 // Determine physical address size and raise an Address Size Fault if 872 // necessary 873 int pa_range = decodePhysAddrRange64(ps); 874 // Clamp to lower limit 875 if (pa_range > physAddrRange) 876 currState->physAddrRange = physAddrRange; 877 else 878 currState->physAddrRange = pa_range; 879 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) { 880 DPRINTF(TLB, "Address size fault before any lookup\n"); 881 Fault f; 882 if (currState->isFetch) 883 f = std::make_shared<PrefetchAbort>( 884 currState->vaddr_tainted, 885 ArmFault::AddressSizeLL + start_lookup_level, 886 isStage2, 887 ArmFault::LpaeTran); 888 else 889 f = std::make_shared<DataAbort>( 890 currState->vaddr_tainted, 891 TlbEntry::DomainType::NoAccess, 892 currState->isWrite, 893 ArmFault::AddressSizeLL + start_lookup_level, 894 isStage2, 895 ArmFault::LpaeTran); 896 897 898 if (currState->timing) { 899 pending = false; 900 nextWalk(currState->tc); 901 currState = NULL; 902 } else { 903 currState->tc = NULL; 904 currState->req = NULL; 905 } 906 return f; 907 908 } 909 910 // Determine descriptor address 911 Addr desc_addr = base_addr | 912 (bits(currState->vaddr, tsz - 1, 913 stride * (3 - start_lookup_level) + tg) << 3); 914 915 // Trickbox address check 916 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure, 917 currState->vaddr, sizeof(uint64_t), currState->isFetch, 918 currState->isWrite, TlbEntry::DomainType::NoAccess, 919 start_lookup_level); 920 if (f) { 921 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted); 922 if (currState->timing) { 923 pending = false; 924 nextWalk(currState->tc); 925 currState = NULL; 926 } else { 927 currState->tc = NULL; 928 currState->req = NULL; 929 } 930 return f; 931 } 932 933 Request::Flags flag = Request::PT_WALK; 934 if (currState->sctlr.c == 0) { 935 flag.set(Request::UNCACHEABLE); 936 } 937 938 if (currState->isSecure) { 939 flag.set(Request::SECURE); 940 } 941 942 currState->longDesc.lookupLevel = start_lookup_level; 943 currState->longDesc.aarch64 = true; 944 currState->longDesc.grainSize = tg; 945 946 if (currState->timing) { 947 Event *event; 948 switch (start_lookup_level) { 949 case L0: 950 event = (Event *) &doL0LongDescEvent; 951 break; 952 case L1: 953 event = (Event *) &doL1LongDescEvent; 954 break; 955 case L2: 956 event = (Event *) &doL2LongDescEvent; 957 break; 958 case L3: 959 event = (Event *) &doL3LongDescEvent; 960 break; 961 default: 962 panic("Invalid table lookup level"); 963 break; 964 } 965 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 966 event, (uint8_t*) &currState->longDesc.data, 967 currState->tc->getCpuPtr()->clockPeriod(), flag); 968 DPRINTF(TLBVerbose, 969 "Adding to walker fifo: queue size before adding: %d\n", 970 stateQueues[start_lookup_level].size()); 971 stateQueues[start_lookup_level].push_back(currState); 972 currState = NULL; 973 } else if (!currState->functional) { 974 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t), 975 NULL, (uint8_t*) &currState->longDesc.data, 976 currState->tc->getCpuPtr()->clockPeriod(), flag); 977 doLongDescriptor(); 978 f = currState->fault; 979 } else { 980 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag, 981 masterId); 982 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 983 pkt->dataStatic((uint8_t*) &currState->longDesc.data); 984 port->sendFunctional(pkt); 985 doLongDescriptor(); 986 delete req; 987 delete pkt; 988 f = currState->fault; 989 } 990 991 return f; 992} 993 994void 995TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr, 996 uint8_t texcb, bool s) 997{ 998 // Note: tc and sctlr local variables are hiding tc and sctrl class 999 // variables 1000 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s); 1001 te.shareable = false; // default value 1002 te.nonCacheable = false; 1003 te.outerShareable = false; 1004 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) { 1005 switch(texcb) { 1006 case 0: // Stongly-ordered 1007 te.nonCacheable = true; 1008 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1009 te.shareable = true; 1010 te.innerAttrs = 1; 1011 te.outerAttrs = 0; 1012 break; 1013 case 1: // Shareable Device 1014 te.nonCacheable = true; 1015 te.mtype = TlbEntry::MemoryType::Device; 1016 te.shareable = true; 1017 te.innerAttrs = 3; 1018 te.outerAttrs = 0; 1019 break; 1020 case 2: // Outer and Inner Write-Through, no Write-Allocate 1021 te.mtype = TlbEntry::MemoryType::Normal; 1022 te.shareable = s; 1023 te.innerAttrs = 6; 1024 te.outerAttrs = bits(texcb, 1, 0); 1025 break; 1026 case 3: // Outer and Inner Write-Back, no Write-Allocate 1027 te.mtype = TlbEntry::MemoryType::Normal; 1028 te.shareable = s; 1029 te.innerAttrs = 7; 1030 te.outerAttrs = bits(texcb, 1, 0); 1031 break; 1032 case 4: // Outer and Inner Non-cacheable 1033 te.nonCacheable = true; 1034 te.mtype = TlbEntry::MemoryType::Normal; 1035 te.shareable = s; 1036 te.innerAttrs = 0; 1037 te.outerAttrs = bits(texcb, 1, 0); 1038 break; 1039 case 5: // Reserved 1040 panic("Reserved texcb value!\n"); 1041 break; 1042 case 6: // Implementation Defined 1043 panic("Implementation-defined texcb value!\n"); 1044 break; 1045 case 7: // Outer and Inner Write-Back, Write-Allocate 1046 te.mtype = TlbEntry::MemoryType::Normal; 1047 te.shareable = s; 1048 te.innerAttrs = 5; 1049 te.outerAttrs = 1; 1050 break; 1051 case 8: // Non-shareable Device 1052 te.nonCacheable = true; 1053 te.mtype = TlbEntry::MemoryType::Device; 1054 te.shareable = false; 1055 te.innerAttrs = 3; 1056 te.outerAttrs = 0; 1057 break; 1058 case 9 ... 15: // Reserved 1059 panic("Reserved texcb value!\n"); 1060 break; 1061 case 16 ... 31: // Cacheable Memory 1062 te.mtype = TlbEntry::MemoryType::Normal; 1063 te.shareable = s; 1064 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0) 1065 te.nonCacheable = true; 1066 te.innerAttrs = bits(texcb, 1, 0); 1067 te.outerAttrs = bits(texcb, 3, 2); 1068 break; 1069 default: 1070 panic("More than 32 states for 5 bits?\n"); 1071 } 1072 } else { 1073 assert(tc); 1074 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR, 1075 currState->tc, !currState->isSecure)); 1076 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR, 1077 currState->tc, !currState->isSecure)); 1078 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr); 1079 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0; 1080 switch(bits(texcb, 2,0)) { 1081 case 0: 1082 curr_tr = prrr.tr0; 1083 curr_ir = nmrr.ir0; 1084 curr_or = nmrr.or0; 1085 te.outerShareable = (prrr.nos0 == 0); 1086 break; 1087 case 1: 1088 curr_tr = prrr.tr1; 1089 curr_ir = nmrr.ir1; 1090 curr_or = nmrr.or1; 1091 te.outerShareable = (prrr.nos1 == 0); 1092 break; 1093 case 2: 1094 curr_tr = prrr.tr2; 1095 curr_ir = nmrr.ir2; 1096 curr_or = nmrr.or2; 1097 te.outerShareable = (prrr.nos2 == 0); 1098 break; 1099 case 3: 1100 curr_tr = prrr.tr3; 1101 curr_ir = nmrr.ir3; 1102 curr_or = nmrr.or3; 1103 te.outerShareable = (prrr.nos3 == 0); 1104 break; 1105 case 4: 1106 curr_tr = prrr.tr4; 1107 curr_ir = nmrr.ir4; 1108 curr_or = nmrr.or4; 1109 te.outerShareable = (prrr.nos4 == 0); 1110 break; 1111 case 5: 1112 curr_tr = prrr.tr5; 1113 curr_ir = nmrr.ir5; 1114 curr_or = nmrr.or5; 1115 te.outerShareable = (prrr.nos5 == 0); 1116 break; 1117 case 6: 1118 panic("Imp defined type\n"); 1119 case 7: 1120 curr_tr = prrr.tr7; 1121 curr_ir = nmrr.ir7; 1122 curr_or = nmrr.or7; 1123 te.outerShareable = (prrr.nos7 == 0); 1124 break; 1125 } 1126 1127 switch(curr_tr) { 1128 case 0: 1129 DPRINTF(TLBVerbose, "StronglyOrdered\n"); 1130 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1131 te.nonCacheable = true; 1132 te.innerAttrs = 1; 1133 te.outerAttrs = 0; 1134 te.shareable = true; 1135 break; 1136 case 1: 1137 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n", 1138 prrr.ds1, prrr.ds0, s); 1139 te.mtype = TlbEntry::MemoryType::Device; 1140 te.nonCacheable = true; 1141 te.innerAttrs = 3; 1142 te.outerAttrs = 0; 1143 if (prrr.ds1 && s) 1144 te.shareable = true; 1145 if (prrr.ds0 && !s) 1146 te.shareable = true; 1147 break; 1148 case 2: 1149 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n", 1150 prrr.ns1, prrr.ns0, s); 1151 te.mtype = TlbEntry::MemoryType::Normal; 1152 if (prrr.ns1 && s) 1153 te.shareable = true; 1154 if (prrr.ns0 && !s) 1155 te.shareable = true; 1156 break; 1157 case 3: 1158 panic("Reserved type"); 1159 } 1160 1161 if (te.mtype == TlbEntry::MemoryType::Normal){ 1162 switch(curr_ir) { 1163 case 0: 1164 te.nonCacheable = true; 1165 te.innerAttrs = 0; 1166 break; 1167 case 1: 1168 te.innerAttrs = 5; 1169 break; 1170 case 2: 1171 te.innerAttrs = 6; 1172 break; 1173 case 3: 1174 te.innerAttrs = 7; 1175 break; 1176 } 1177 1178 switch(curr_or) { 1179 case 0: 1180 te.nonCacheable = true; 1181 te.outerAttrs = 0; 1182 break; 1183 case 1: 1184 te.outerAttrs = 1; 1185 break; 1186 case 2: 1187 te.outerAttrs = 2; 1188 break; 1189 case 3: 1190 te.outerAttrs = 3; 1191 break; 1192 } 1193 } 1194 } 1195 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, " 1196 "outerAttrs: %d\n", 1197 te.shareable, te.innerAttrs, te.outerAttrs); 1198 te.setAttributes(false); 1199} 1200 1201void 1202TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te, 1203 LongDescriptor &lDescriptor) 1204{ 1205 assert(_haveLPAE); 1206 1207 uint8_t attr; 1208 uint8_t sh = lDescriptor.sh(); 1209 // Different format and source of attributes if this is a stage 2 1210 // translation 1211 if (isStage2) { 1212 attr = lDescriptor.memAttr(); 1213 uint8_t attr_3_2 = (attr >> 2) & 0x3; 1214 uint8_t attr_1_0 = attr & 0x3; 1215 1216 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh); 1217 1218 if (attr_3_2 == 0) { 1219 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered 1220 : TlbEntry::MemoryType::Device; 1221 te.outerAttrs = 0; 1222 te.innerAttrs = attr_1_0 == 0 ? 1 : 3; 1223 te.nonCacheable = true; 1224 } else { 1225 te.mtype = TlbEntry::MemoryType::Normal; 1226 te.outerAttrs = attr_3_2 == 1 ? 0 : 1227 attr_3_2 == 2 ? 2 : 1; 1228 te.innerAttrs = attr_1_0 == 1 ? 0 : 1229 attr_1_0 == 2 ? 6 : 5; 1230 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1); 1231 } 1232 } else { 1233 uint8_t attrIndx = lDescriptor.attrIndx(); 1234 1235 // LPAE always uses remapping of memory attributes, irrespective of the 1236 // value of SCTLR.TRE 1237 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0; 1238 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc, 1239 !currState->isSecure); 1240 uint32_t mair = currState->tc->readMiscReg(reg_as_int); 1241 attr = (mair >> (8 * (attrIndx % 4))) & 0xff; 1242 uint8_t attr_7_4 = bits(attr, 7, 4); 1243 uint8_t attr_3_0 = bits(attr, 3, 0); 1244 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr); 1245 1246 // Note: the memory subsystem only cares about the 'cacheable' memory 1247 // attribute. The other attributes are only used to fill the PAR register 1248 // accordingly to provide the illusion of full support 1249 te.nonCacheable = false; 1250 1251 switch (attr_7_4) { 1252 case 0x0: 1253 // Strongly-ordered or Device memory 1254 if (attr_3_0 == 0x0) 1255 te.mtype = TlbEntry::MemoryType::StronglyOrdered; 1256 else if (attr_3_0 == 0x4) 1257 te.mtype = TlbEntry::MemoryType::Device; 1258 else 1259 panic("Unpredictable behavior\n"); 1260 te.nonCacheable = true; 1261 te.outerAttrs = 0; 1262 break; 1263 case 0x4: 1264 // Normal memory, Outer Non-cacheable 1265 te.mtype = TlbEntry::MemoryType::Normal; 1266 te.outerAttrs = 0; 1267 if (attr_3_0 == 0x4) 1268 // Inner Non-cacheable 1269 te.nonCacheable = true; 1270 else if (attr_3_0 < 0x8) 1271 panic("Unpredictable behavior\n"); 1272 break; 1273 case 0x8: 1274 case 0x9: 1275 case 0xa: 1276 case 0xb: 1277 case 0xc: 1278 case 0xd: 1279 case 0xe: 1280 case 0xf: 1281 if (attr_7_4 & 0x4) { 1282 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3; 1283 } else { 1284 te.outerAttrs = 0x2; 1285 } 1286 // Normal memory, Outer Cacheable 1287 te.mtype = TlbEntry::MemoryType::Normal; 1288 if (attr_3_0 != 0x4 && attr_3_0 < 0x8) 1289 panic("Unpredictable behavior\n"); 1290 break; 1291 default: 1292 panic("Unpredictable behavior\n"); 1293 break; 1294 } 1295 1296 switch (attr_3_0) { 1297 case 0x0: 1298 te.innerAttrs = 0x1; 1299 break; 1300 case 0x4: 1301 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0; 1302 break; 1303 case 0x8: 1304 case 0x9: 1305 case 0xA: 1306 case 0xB: 1307 te.innerAttrs = 6; 1308 break; 1309 case 0xC: 1310 case 0xD: 1311 case 0xE: 1312 case 0xF: 1313 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7; 1314 break; 1315 default: 1316 panic("Unpredictable behavior\n"); 1317 break; 1318 } 1319 } 1320 1321 te.outerShareable = sh == 2; 1322 te.shareable = (sh & 0x2) ? true : false; 1323 te.setAttributes(true); 1324 te.attributes |= (uint64_t) attr << 56; 1325} 1326 1327void 1328TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx, 1329 uint8_t sh) 1330{ 1331 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh); 1332 1333 // Select MAIR 1334 uint64_t mair; 1335 switch (currState->el) { 1336 case EL0: 1337 case EL1: 1338 mair = tc->readMiscReg(MISCREG_MAIR_EL1); 1339 break; 1340 case EL2: 1341 mair = tc->readMiscReg(MISCREG_MAIR_EL2); 1342 break; 1343 case EL3: 1344 mair = tc->readMiscReg(MISCREG_MAIR_EL3); 1345 break; 1346 default: 1347 panic("Invalid exception level"); 1348 break; 1349 } 1350 1351 // Select attributes 1352 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx); 1353 uint8_t attr_lo = bits(attr, 3, 0); 1354 uint8_t attr_hi = bits(attr, 7, 4); 1355 1356 // Memory type 1357 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal; 1358 1359 // Cacheability 1360 te.nonCacheable = false; 1361 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory 1362 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable 1363 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable 1364 te.nonCacheable = true; 1365 } 1366 1367 te.shareable = sh == 2; 1368 te.outerShareable = (sh & 0x2) ? true : false; 1369 // Attributes formatted according to the 64-bit PAR 1370 te.attributes = ((uint64_t) attr << 56) | 1371 (1 << 11) | // LPAE bit 1372 (te.ns << 9) | // NS bit 1373 (sh << 7); 1374} 1375 1376void 1377TableWalker::doL1Descriptor() 1378{ 1379 if (currState->fault != NoFault) { 1380 return; 1381 } 1382 1383 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n", 1384 currState->vaddr_tainted, currState->l1Desc.data); 1385 TlbEntry te; 1386 1387 switch (currState->l1Desc.type()) { 1388 case L1Descriptor::Ignore: 1389 case L1Descriptor::Reserved: 1390 if (!currState->timing) { 1391 currState->tc = NULL; 1392 currState->req = NULL; 1393 } 1394 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n"); 1395 if (currState->isFetch) 1396 currState->fault = 1397 std::make_shared<PrefetchAbort>( 1398 currState->vaddr_tainted, 1399 ArmFault::TranslationLL + L1, 1400 isStage2, 1401 ArmFault::VmsaTran); 1402 else 1403 currState->fault = 1404 std::make_shared<DataAbort>( 1405 currState->vaddr_tainted, 1406 TlbEntry::DomainType::NoAccess, 1407 currState->isWrite, 1408 ArmFault::TranslationLL + L1, isStage2, 1409 ArmFault::VmsaTran); 1410 return; 1411 case L1Descriptor::Section: 1412 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) { 1413 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is 1414 * enabled if set, do l1.Desc.setAp0() instead of generating 1415 * AccessFlag0 1416 */ 1417 1418 currState->fault = std::make_shared<DataAbort>( 1419 currState->vaddr_tainted, 1420 currState->l1Desc.domain(), 1421 currState->isWrite, 1422 ArmFault::AccessFlagLL + L1, 1423 isStage2, 1424 ArmFault::VmsaTran); 1425 } 1426 if (currState->l1Desc.supersection()) { 1427 panic("Haven't implemented supersections\n"); 1428 } 1429 insertTableEntry(currState->l1Desc, false); 1430 return; 1431 case L1Descriptor::PageTable: 1432 { 1433 Addr l2desc_addr; 1434 l2desc_addr = currState->l1Desc.l2Addr() | 1435 (bits(currState->vaddr, 19, 12) << 2); 1436 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n", 1437 l2desc_addr, currState->isSecure ? "s" : "ns"); 1438 1439 // Trickbox address check 1440 currState->fault = tlb->walkTrickBoxCheck( 1441 l2desc_addr, currState->isSecure, currState->vaddr, 1442 sizeof(uint32_t), currState->isFetch, currState->isWrite, 1443 currState->l1Desc.domain(), L2); 1444 1445 if (currState->fault) { 1446 if (!currState->timing) { 1447 currState->tc = NULL; 1448 currState->req = NULL; 1449 } 1450 return; 1451 } 1452 1453 Request::Flags flag = Request::PT_WALK; 1454 if (currState->isSecure) 1455 flag.set(Request::SECURE); 1456 1457 bool delayed; 1458 delayed = fetchDescriptor(l2desc_addr, 1459 (uint8_t*)&currState->l2Desc.data, 1460 sizeof(uint32_t), flag, -1, &doL2DescEvent, 1461 &TableWalker::doL2Descriptor); 1462 if (delayed) { 1463 currState->delayed = true; 1464 } 1465 1466 return; 1467 } 1468 default: 1469 panic("A new type in a 2 bit field?\n"); 1470 } 1471} 1472 1473void 1474TableWalker::doLongDescriptor() 1475{ 1476 if (currState->fault != NoFault) { 1477 return; 1478 } 1479 1480 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n", 1481 currState->longDesc.lookupLevel, currState->vaddr_tainted, 1482 currState->longDesc.data, 1483 currState->aarch64 ? "AArch64" : "long-desc."); 1484 1485 if ((currState->longDesc.type() == LongDescriptor::Block) || 1486 (currState->longDesc.type() == LongDescriptor::Page)) { 1487 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, " 1488 "xn: %d, ap: %d, af: %d, type: %d\n", 1489 currState->longDesc.lookupLevel, 1490 currState->longDesc.data, 1491 currState->longDesc.pxn(), 1492 currState->longDesc.xn(), 1493 currState->longDesc.ap(), 1494 currState->longDesc.af(), 1495 currState->longDesc.type()); 1496 } else { 1497 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n", 1498 currState->longDesc.lookupLevel, 1499 currState->longDesc.data, 1500 currState->longDesc.type()); 1501 } 1502 1503 TlbEntry te; 1504 1505 switch (currState->longDesc.type()) { 1506 case LongDescriptor::Invalid: 1507 if (!currState->timing) { 1508 currState->tc = NULL; 1509 currState->req = NULL; 1510 } 1511 1512 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n", 1513 currState->longDesc.lookupLevel, 1514 ArmFault::TranslationLL + currState->longDesc.lookupLevel); 1515 if (currState->isFetch) 1516 currState->fault = std::make_shared<PrefetchAbort>( 1517 currState->vaddr_tainted, 1518 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1519 isStage2, 1520 ArmFault::LpaeTran); 1521 else 1522 currState->fault = std::make_shared<DataAbort>( 1523 currState->vaddr_tainted, 1524 TlbEntry::DomainType::NoAccess, 1525 currState->isWrite, 1526 ArmFault::TranslationLL + currState->longDesc.lookupLevel, 1527 isStage2, 1528 ArmFault::LpaeTran); 1529 return; 1530 case LongDescriptor::Block: 1531 case LongDescriptor::Page: 1532 { 1533 bool fault = false; 1534 bool aff = false; 1535 // Check for address size fault 1536 if (checkAddrSizeFaultAArch64( 1537 mbits(currState->longDesc.data, MaxPhysAddrRange - 1, 1538 currState->longDesc.offsetBits()), 1539 currState->physAddrRange)) { 1540 fault = true; 1541 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1542 currState->longDesc.lookupLevel); 1543 // Check for access fault 1544 } else if (currState->longDesc.af() == 0) { 1545 fault = true; 1546 DPRINTF(TLB, "L%d descriptor causing Access Fault\n", 1547 currState->longDesc.lookupLevel); 1548 aff = true; 1549 } 1550 if (fault) { 1551 if (currState->isFetch) 1552 currState->fault = std::make_shared<PrefetchAbort>( 1553 currState->vaddr_tainted, 1554 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1555 currState->longDesc.lookupLevel, 1556 isStage2, 1557 ArmFault::LpaeTran); 1558 else 1559 currState->fault = std::make_shared<DataAbort>( 1560 currState->vaddr_tainted, 1561 TlbEntry::DomainType::NoAccess, currState->isWrite, 1562 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) + 1563 currState->longDesc.lookupLevel, 1564 isStage2, 1565 ArmFault::LpaeTran); 1566 } else { 1567 insertTableEntry(currState->longDesc, true); 1568 } 1569 } 1570 return; 1571 case LongDescriptor::Table: 1572 { 1573 // Set hierarchical permission flags 1574 currState->secureLookup = currState->secureLookup && 1575 currState->longDesc.secureTable(); 1576 currState->rwTable = currState->rwTable && 1577 currState->longDesc.rwTable(); 1578 currState->userTable = currState->userTable && 1579 currState->longDesc.userTable(); 1580 currState->xnTable = currState->xnTable || 1581 currState->longDesc.xnTable(); 1582 currState->pxnTable = currState->pxnTable || 1583 currState->longDesc.pxnTable(); 1584 1585 // Set up next level lookup 1586 Addr next_desc_addr = currState->longDesc.nextDescAddr( 1587 currState->vaddr); 1588 1589 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n", 1590 currState->longDesc.lookupLevel, 1591 currState->longDesc.lookupLevel + 1, 1592 next_desc_addr, 1593 currState->secureLookup ? "s" : "ns"); 1594 1595 // Check for address size fault 1596 if (currState->aarch64 && checkAddrSizeFaultAArch64( 1597 next_desc_addr, currState->physAddrRange)) { 1598 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n", 1599 currState->longDesc.lookupLevel); 1600 if (currState->isFetch) 1601 currState->fault = std::make_shared<PrefetchAbort>( 1602 currState->vaddr_tainted, 1603 ArmFault::AddressSizeLL 1604 + currState->longDesc.lookupLevel, 1605 isStage2, 1606 ArmFault::LpaeTran); 1607 else 1608 currState->fault = std::make_shared<DataAbort>( 1609 currState->vaddr_tainted, 1610 TlbEntry::DomainType::NoAccess, currState->isWrite, 1611 ArmFault::AddressSizeLL 1612 + currState->longDesc.lookupLevel, 1613 isStage2, 1614 ArmFault::LpaeTran); 1615 return; 1616 } 1617 1618 // Trickbox address check 1619 currState->fault = tlb->walkTrickBoxCheck( 1620 next_desc_addr, currState->vaddr, 1621 currState->vaddr, sizeof(uint64_t), 1622 currState->isFetch, currState->isWrite, 1623 TlbEntry::DomainType::Client, 1624 toLookupLevel(currState->longDesc.lookupLevel +1)); 1625 1626 if (currState->fault) { 1627 if (!currState->timing) { 1628 currState->tc = NULL; 1629 currState->req = NULL; 1630 } 1631 return; 1632 } 1633 1634 Request::Flags flag = Request::PT_WALK; 1635 if (currState->secureLookup) 1636 flag.set(Request::SECURE); 1637 1638 currState->longDesc.lookupLevel = 1639 (LookupLevel) (currState->longDesc.lookupLevel + 1); 1640 Event *event = NULL; 1641 switch (currState->longDesc.lookupLevel) { 1642 case L1: 1643 assert(currState->aarch64); 1644 event = &doL1LongDescEvent; 1645 break; 1646 case L2: 1647 event = &doL2LongDescEvent; 1648 break; 1649 case L3: 1650 event = &doL3LongDescEvent; 1651 break; 1652 default: 1653 panic("Wrong lookup level in table walk\n"); 1654 break; 1655 } 1656 1657 bool delayed; 1658 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data, 1659 sizeof(uint64_t), flag, -1, event, 1660 &TableWalker::doLongDescriptor); 1661 if (delayed) { 1662 currState->delayed = true; 1663 } 1664 } 1665 return; 1666 default: 1667 panic("A new type in a 2 bit field?\n"); 1668 } 1669} 1670 1671void 1672TableWalker::doL2Descriptor() 1673{ 1674 if (currState->fault != NoFault) { 1675 return; 1676 } 1677 1678 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n", 1679 currState->vaddr_tainted, currState->l2Desc.data); 1680 TlbEntry te; 1681 1682 if (currState->l2Desc.invalid()) { 1683 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n"); 1684 if (!currState->timing) { 1685 currState->tc = NULL; 1686 currState->req = NULL; 1687 } 1688 if (currState->isFetch) 1689 currState->fault = std::make_shared<PrefetchAbort>( 1690 currState->vaddr_tainted, 1691 ArmFault::TranslationLL + L2, 1692 isStage2, 1693 ArmFault::VmsaTran); 1694 else 1695 currState->fault = std::make_shared<DataAbort>( 1696 currState->vaddr_tainted, currState->l1Desc.domain(), 1697 currState->isWrite, ArmFault::TranslationLL + L2, 1698 isStage2, 1699 ArmFault::VmsaTran); 1700 return; 1701 } 1702 1703 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) { 1704 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled 1705 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0 1706 */ 1707 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n", 1708 currState->sctlr.afe, currState->l2Desc.ap()); 1709 1710 currState->fault = std::make_shared<DataAbort>( 1711 currState->vaddr_tainted, 1712 TlbEntry::DomainType::NoAccess, currState->isWrite, 1713 ArmFault::AccessFlagLL + L2, isStage2, 1714 ArmFault::VmsaTran); 1715 } 1716 1717 insertTableEntry(currState->l2Desc, false); 1718} 1719 1720void 1721TableWalker::doL1DescriptorWrapper() 1722{ 1723 currState = stateQueues[L1].front(); 1724 currState->delayed = false; 1725 // if there's a stage2 translation object we don't need it any more 1726 if (currState->stage2Tran) { 1727 delete currState->stage2Tran; 1728 currState->stage2Tran = NULL; 1729 } 1730 1731 1732 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data); 1733 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data); 1734 1735 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted); 1736 doL1Descriptor(); 1737 1738 stateQueues[L1].pop_front(); 1739 // Check if fault was generated 1740 if (currState->fault != NoFault) { 1741 currState->transState->finish(currState->fault, currState->req, 1742 currState->tc, currState->mode); 1743 statWalksShortTerminatedAtLevel[0]++; 1744 1745 pending = false; 1746 nextWalk(currState->tc); 1747 1748 currState->req = NULL; 1749 currState->tc = NULL; 1750 currState->delayed = false; 1751 delete currState; 1752 } 1753 else if (!currState->delayed) { 1754 // delay is not set so there is no L2 to do 1755 // Don't finish the translation if a stage 2 look up is underway 1756 if (!currState->doingStage2) { 1757 statWalkServiceTime.sample(curTick() - currState->startTime); 1758 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1759 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1760 currState->transState, currState->mode); 1761 statWalksShortTerminatedAtLevel[0]++; 1762 } 1763 1764 pending = false; 1765 nextWalk(currState->tc); 1766 1767 currState->req = NULL; 1768 currState->tc = NULL; 1769 currState->delayed = false; 1770 delete currState; 1771 } else { 1772 // need to do L2 descriptor 1773 stateQueues[L2].push_back(currState); 1774 } 1775 currState = NULL; 1776} 1777 1778void 1779TableWalker::doL2DescriptorWrapper() 1780{ 1781 currState = stateQueues[L2].front(); 1782 assert(currState->delayed); 1783 // if there's a stage2 translation object we don't need it any more 1784 if (currState->stage2Tran) { 1785 delete currState->stage2Tran; 1786 currState->stage2Tran = NULL; 1787 } 1788 1789 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n", 1790 currState->vaddr_tainted); 1791 doL2Descriptor(); 1792 1793 // Check if fault was generated 1794 if (currState->fault != NoFault) { 1795 currState->transState->finish(currState->fault, currState->req, 1796 currState->tc, currState->mode); 1797 statWalksShortTerminatedAtLevel[1]++; 1798 } 1799 else { 1800 // Don't finish the translation if a stage 2 look up is underway 1801 if (!currState->doingStage2) { 1802 statWalkServiceTime.sample(curTick() - currState->startTime); 1803 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1804 currState->fault = tlb->translateTiming(currState->req, 1805 currState->tc, currState->transState, currState->mode); 1806 statWalksShortTerminatedAtLevel[1]++; 1807 } 1808 } 1809 1810 1811 stateQueues[L2].pop_front(); 1812 pending = false; 1813 nextWalk(currState->tc); 1814 1815 currState->req = NULL; 1816 currState->tc = NULL; 1817 currState->delayed = false; 1818 1819 delete currState; 1820 currState = NULL; 1821} 1822 1823void 1824TableWalker::doL0LongDescriptorWrapper() 1825{ 1826 doLongDescriptorWrapper(L0); 1827} 1828 1829void 1830TableWalker::doL1LongDescriptorWrapper() 1831{ 1832 doLongDescriptorWrapper(L1); 1833} 1834 1835void 1836TableWalker::doL2LongDescriptorWrapper() 1837{ 1838 doLongDescriptorWrapper(L2); 1839} 1840 1841void 1842TableWalker::doL3LongDescriptorWrapper() 1843{ 1844 doLongDescriptorWrapper(L3); 1845} 1846 1847void 1848TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level) 1849{ 1850 currState = stateQueues[curr_lookup_level].front(); 1851 assert(curr_lookup_level == currState->longDesc.lookupLevel); 1852 currState->delayed = false; 1853 1854 // if there's a stage2 translation object we don't need it any more 1855 if (currState->stage2Tran) { 1856 delete currState->stage2Tran; 1857 currState->stage2Tran = NULL; 1858 } 1859 1860 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n", 1861 currState->vaddr_tainted); 1862 doLongDescriptor(); 1863 1864 stateQueues[curr_lookup_level].pop_front(); 1865 1866 if (currState->fault != NoFault) { 1867 // A fault was generated 1868 currState->transState->finish(currState->fault, currState->req, 1869 currState->tc, currState->mode); 1870 1871 pending = false; 1872 nextWalk(currState->tc); 1873 1874 currState->req = NULL; 1875 currState->tc = NULL; 1876 currState->delayed = false; 1877 delete currState; 1878 } else if (!currState->delayed) { 1879 // No additional lookups required 1880 // Don't finish the translation if a stage 2 look up is underway 1881 if (!currState->doingStage2) { 1882 DPRINTF(TLBVerbose, "calling translateTiming again\n"); 1883 statWalkServiceTime.sample(curTick() - currState->startTime); 1884 currState->fault = tlb->translateTiming(currState->req, currState->tc, 1885 currState->transState, 1886 currState->mode); 1887 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++; 1888 } 1889 1890 pending = false; 1891 nextWalk(currState->tc); 1892 1893 currState->req = NULL; 1894 currState->tc = NULL; 1895 currState->delayed = false; 1896 delete currState; 1897 } else { 1898 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1) 1899 panic("Max. number of lookups already reached in table walk\n"); 1900 // Need to perform additional lookups 1901 stateQueues[currState->longDesc.lookupLevel].push_back(currState); 1902 } 1903 currState = NULL; 1904} 1905 1906 1907void 1908TableWalker::nextWalk(ThreadContext *tc) 1909{ 1910 if (pendingQueue.size()) 1911 schedule(doProcessEvent, clockEdge(Cycles(1))); 1912 else 1913 completeDrain(); 1914} 1915 1916bool 1917TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes, 1918 Request::Flags flags, int queueIndex, Event *event, 1919 void (TableWalker::*doDescriptor)()) 1920{ 1921 bool isTiming = currState->timing; 1922 1923 // do the requests for the page table descriptors have to go through the 1924 // second stage MMU 1925 if (currState->stage2Req) { 1926 Fault fault; 1927 flags = flags | TLB::MustBeOne; 1928 1929 if (isTiming) { 1930 Stage2MMU::Stage2Translation *tran = new 1931 Stage2MMU::Stage2Translation(*stage2Mmu, data, event, 1932 currState->vaddr); 1933 currState->stage2Tran = tran; 1934 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes, 1935 flags); 1936 fault = tran->fault; 1937 } else { 1938 fault = stage2Mmu->readDataUntimed(currState->tc, 1939 currState->vaddr, descAddr, data, numBytes, flags, 1940 currState->functional); 1941 } 1942 1943 if (fault != NoFault) { 1944 currState->fault = fault; 1945 } 1946 if (isTiming) { 1947 if (queueIndex >= 0) { 1948 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1949 stateQueues[queueIndex].size()); 1950 stateQueues[queueIndex].push_back(currState); 1951 currState = NULL; 1952 } 1953 } else { 1954 (this->*doDescriptor)(); 1955 } 1956 } else { 1957 if (isTiming) { 1958 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data, 1959 currState->tc->getCpuPtr()->clockPeriod(),flags); 1960 if (queueIndex >= 0) { 1961 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n", 1962 stateQueues[queueIndex].size()); 1963 stateQueues[queueIndex].push_back(currState); 1964 currState = NULL; 1965 } 1966 } else if (!currState->functional) { 1967 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data, 1968 currState->tc->getCpuPtr()->clockPeriod(), flags); 1969 (this->*doDescriptor)(); 1970 } else { 1971 RequestPtr req = new Request(descAddr, numBytes, flags, masterId); 1972 req->taskId(ContextSwitchTaskId::DMA); 1973 PacketPtr pkt = new Packet(req, MemCmd::ReadReq); 1974 pkt->dataStatic(data); 1975 port->sendFunctional(pkt); 1976 (this->*doDescriptor)(); 1977 delete req; 1978 delete pkt; 1979 } 1980 } 1981 return (isTiming); 1982} 1983 1984void 1985TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor) 1986{ 1987 TlbEntry te; 1988 1989 // Create and fill a new page table entry 1990 te.valid = true; 1991 te.longDescFormat = longDescriptor; 1992 te.isHyp = currState->isHyp; 1993 te.asid = currState->asid; 1994 te.vmid = currState->vmid; 1995 te.N = descriptor.offsetBits(); 1996 te.vpn = currState->vaddr >> te.N; 1997 te.size = (1<<te.N) - 1; 1998 te.pfn = descriptor.pfn(); 1999 te.domain = descriptor.domain(); 2000 te.lookupLevel = descriptor.lookupLevel; 2001 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2; 2002 te.nstid = !currState->isSecure; 2003 te.xn = descriptor.xn(); 2004 if (currState->aarch64) 2005 te.el = currState->el; 2006 else 2007 te.el = 1; 2008 2009 statPageSizes[pageSizeNtoStatBin(te.N)]++; 2010 statRequestOrigin[COMPLETED][currState->isFetch]++; 2011 2012 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries 2013 // as global 2014 te.global = descriptor.global(currState) || isStage2; 2015 if (longDescriptor) { 2016 LongDescriptor lDescriptor = 2017 dynamic_cast<LongDescriptor &>(descriptor); 2018 2019 te.xn |= currState->xnTable; 2020 te.pxn = currState->pxnTable || lDescriptor.pxn(); 2021 if (isStage2) { 2022 // this is actually the HAP field, but its stored in the same bit 2023 // possitions as the AP field in a stage 1 translation. 2024 te.hap = lDescriptor.ap(); 2025 } else { 2026 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) | 2027 (currState->userTable && (descriptor.ap() & 0x1)); 2028 } 2029 if (currState->aarch64) 2030 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(), 2031 currState->longDesc.sh()); 2032 else 2033 memAttrsLPAE(currState->tc, te, lDescriptor); 2034 } else { 2035 te.ap = descriptor.ap(); 2036 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(), 2037 descriptor.shareable()); 2038 } 2039 2040 // Debug output 2041 DPRINTF(TLB, descriptor.dbgHeader().c_str()); 2042 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n", 2043 te.N, te.pfn, te.size, te.global, te.valid); 2044 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d " 2045 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn, 2046 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp, 2047 te.nonCacheable, te.ns); 2048 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n", 2049 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()), 2050 descriptor.getRawData()); 2051 2052 // Insert the entry into the TLB 2053 tlb->insert(currState->vaddr, te); 2054 if (!currState->timing) { 2055 currState->tc = NULL; 2056 currState->req = NULL; 2057 } 2058} 2059 2060ArmISA::TableWalker * 2061ArmTableWalkerParams::create() 2062{ 2063 return new ArmISA::TableWalker(this); 2064} 2065 2066LookupLevel 2067TableWalker::toLookupLevel(uint8_t lookup_level_as_int) 2068{ 2069 switch (lookup_level_as_int) { 2070 case L1: 2071 return L1; 2072 case L2: 2073 return L2; 2074 case L3: 2075 return L3; 2076 default: 2077 panic("Invalid lookup level conversion"); 2078 } 2079} 2080 2081/* this method keeps track of the table walker queue's residency, so 2082 * needs to be called whenever requests start and complete. */ 2083void 2084TableWalker::pendingChange() 2085{ 2086 unsigned n = pendingQueue.size(); 2087 if ((currState != NULL) && (currState != pendingQueue.front())) { 2088 ++n; 2089 } 2090 2091 if (n != pendingReqs) { 2092 Tick now = curTick(); 2093 statPendingWalks.sample(pendingReqs, now - pendingChangeTick); 2094 pendingReqs = n; 2095 pendingChangeTick = now; 2096 } 2097} 2098 2099uint8_t 2100TableWalker::pageSizeNtoStatBin(uint8_t N) 2101{ 2102 /* for statPageSizes */ 2103 switch(N) { 2104 case 12: return 0; // 4K 2105 case 14: return 1; // 16K (using 16K granule in v8-64) 2106 case 16: return 2; // 64K 2107 case 20: return 3; // 1M 2108 case 21: return 4; // 2M-LPAE 2109 case 24: return 5; // 16M 2110 case 25: return 6; // 32M (using 16K granule in v8-64) 2111 case 29: return 7; // 512M (using 64K granule in v8-64) 2112 case 30: return 8; // 1G-LPAE 2113 default: 2114 panic("unknown page size"); 2115 return 255; 2116 } 2117} 2118 2119void 2120TableWalker::regStats() 2121{ 2122 statWalks 2123 .name(name() + ".walks") 2124 .desc("Table walker walks requested") 2125 ; 2126 2127 statWalksShortDescriptor 2128 .name(name() + ".walksShort") 2129 .desc("Table walker walks initiated with short descriptors") 2130 .flags(Stats::nozero) 2131 ; 2132 2133 statWalksLongDescriptor 2134 .name(name() + ".walksLong") 2135 .desc("Table walker walks initiated with long descriptors") 2136 .flags(Stats::nozero) 2137 ; 2138 2139 statWalksShortTerminatedAtLevel 2140 .init(2) 2141 .name(name() + ".walksShortTerminationLevel") 2142 .desc("Level at which table walker walks " 2143 "with short descriptors terminate") 2144 .flags(Stats::nozero) 2145 ; 2146 statWalksShortTerminatedAtLevel.subname(0, "Level1"); 2147 statWalksShortTerminatedAtLevel.subname(1, "Level2"); 2148 2149 statWalksLongTerminatedAtLevel 2150 .init(4) 2151 .name(name() + ".walksLongTerminationLevel") 2152 .desc("Level at which table walker walks " 2153 "with long descriptors terminate") 2154 .flags(Stats::nozero) 2155 ; 2156 statWalksLongTerminatedAtLevel.subname(0, "Level0"); 2157 statWalksLongTerminatedAtLevel.subname(1, "Level1"); 2158 statWalksLongTerminatedAtLevel.subname(2, "Level2"); 2159 statWalksLongTerminatedAtLevel.subname(3, "Level3"); 2160 2161 statSquashedBefore 2162 .name(name() + ".walksSquashedBefore") 2163 .desc("Table walks squashed before starting") 2164 .flags(Stats::nozero) 2165 ; 2166 2167 statSquashedAfter 2168 .name(name() + ".walksSquashedAfter") 2169 .desc("Table walks squashed after completion") 2170 .flags(Stats::nozero) 2171 ; 2172 2173 statWalkWaitTime 2174 .init(16) 2175 .name(name() + ".walkWaitTime") 2176 .desc("Table walker wait (enqueue to first request) latency") 2177 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2178 ; 2179 2180 statWalkServiceTime 2181 .init(16) 2182 .name(name() + ".walkCompletionTime") 2183 .desc("Table walker service (enqueue to completion) latency") 2184 .flags(Stats::pdf | Stats::nozero | Stats::nonan) 2185 ; 2186 2187 statPendingWalks 2188 .init(16) 2189 .name(name() + ".walksPending") 2190 .desc("Table walker pending requests distribution") 2191 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan) 2192 ; 2193 2194 statPageSizes // see DDI 0487A D4-1661 2195 .init(9) 2196 .name(name() + ".walkPageSizes") 2197 .desc("Table walker page sizes translated") 2198 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero) 2199 ; 2200 statPageSizes.subname(0, "4K"); 2201 statPageSizes.subname(1, "16K"); 2202 statPageSizes.subname(2, "64K"); 2203 statPageSizes.subname(3, "1M"); 2204 statPageSizes.subname(4, "2M"); 2205 statPageSizes.subname(5, "16M"); 2206 statPageSizes.subname(6, "32M"); 2207 statPageSizes.subname(7, "512M"); 2208 statPageSizes.subname(8, "1G"); 2209 2210 statRequestOrigin 2211 .init(2,2) // Instruction/Data, requests/completed 2212 .name(name() + ".walkRequestOrigin") 2213 .desc("Table walker requests started/completed, data/inst") 2214 .flags(Stats::total) 2215 ; 2216 statRequestOrigin.subname(0,"Requested"); 2217 statRequestOrigin.subname(1,"Completed"); 2218 statRequestOrigin.ysubname(0,"Data"); 2219 statRequestOrigin.ysubname(1,"Inst"); 2220} 2221