pagetable_walker.cc revision 10231
1/* 2 * Copyright (c) 2012 ARM Limited 3 * All rights reserved. 4 * 5 * The license below extends only to copyright in the software and shall 6 * not be construed as granting a license to any other intellectual 7 * property including but not limited to intellectual property relating 8 * to a hardware implementation of the functionality of the software 9 * licensed hereunder. You may use the software subject to the license 10 * terms below provided that you ensure that this notice is replicated 11 * unmodified and in its entirety in all distributions of the software, 12 * modified or unmodified, in source code or in binary form. 13 * 14 * Copyright (c) 2007 The Hewlett-Packard Development Company 15 * All rights reserved. 16 * 17 * The license below extends only to copyright in the software and shall 18 * not be construed as granting a license to any other intellectual 19 * property including but not limited to intellectual property relating 20 * to a hardware implementation of the functionality of the software 21 * licensed hereunder. You may use the software subject to the license 22 * terms below provided that you ensure that this notice is replicated 23 * unmodified and in its entirety in all distributions of the software, 24 * modified or unmodified, in source code or in binary form. 25 * 26 * Redistribution and use in source and binary forms, with or without 27 * modification, are permitted provided that the following conditions are 28 * met: redistributions of source code must retain the above copyright 29 * notice, this list of conditions and the following disclaimer; 30 * redistributions in binary form must reproduce the above copyright 31 * notice, this list of conditions and the following disclaimer in the 32 * documentation and/or other materials provided with the distribution; 33 * neither the name of the copyright holders nor the names of its 34 * contributors may be used to endorse or promote products derived from 35 * this software without specific prior written permission. 36 * 37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 48 * 49 * Authors: Gabe Black 50 */ 51 52#include "arch/x86/pagetable.hh" 53#include "arch/x86/pagetable_walker.hh" 54#include "arch/x86/tlb.hh" 55#include "arch/x86/vtophys.hh" 56#include "base/bitfield.hh" 57#include "base/trie.hh" 58#include "cpu/base.hh" 59#include "cpu/thread_context.hh" 60#include "debug/PageTableWalker.hh" 61#include "mem/packet_access.hh" 62#include "mem/request.hh" 63 64namespace X86ISA { 65 66// Unfortunately, the placement of the base field in a page table entry is 67// very erratic and would make a mess here. It might be moved here at some 68// point in the future. 69BitUnion64(PageTableEntry) 70 Bitfield<63> nx; 71 Bitfield<11, 9> avl; 72 Bitfield<8> g; 73 Bitfield<7> ps; 74 Bitfield<6> d; 75 Bitfield<5> a; 76 Bitfield<4> pcd; 77 Bitfield<3> pwt; 78 Bitfield<2> u; 79 Bitfield<1> w; 80 Bitfield<0> p; 81EndBitUnion(PageTableEntry) 82 83Fault 84Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation, 85 RequestPtr _req, BaseTLB::Mode _mode) 86{ 87 // TODO: in timing mode, instead of blocking when there are other 88 // outstanding requests, see if this request can be coalesced with 89 // another one (i.e. either coalesce or start walk) 90 WalkerState * newState = new WalkerState(this, _translation, _req); 91 newState->initState(_tc, _mode, sys->isTimingMode()); 92 if (currStates.size()) { 93 assert(newState->isTiming()); 94 DPRINTF(PageTableWalker, "Walks in progress: %d\n", currStates.size()); 95 currStates.push_back(newState); 96 return NoFault; 97 } else { 98 currStates.push_back(newState); 99 Fault fault = newState->startWalk(); 100 if (!newState->isTiming()) { 101 currStates.pop_front(); 102 delete newState; 103 } 104 return fault; 105 } 106} 107 108Fault 109Walker::startFunctional(ThreadContext * _tc, Addr &addr, unsigned &logBytes, 110 BaseTLB::Mode _mode) 111{ 112 funcState.initState(_tc, _mode); 113 return funcState.startFunctional(addr, logBytes); 114} 115 116bool 117Walker::WalkerPort::recvTimingResp(PacketPtr pkt) 118{ 119 return walker->recvTimingResp(pkt); 120} 121 122bool 123Walker::recvTimingResp(PacketPtr pkt) 124{ 125 WalkerSenderState * senderState = 126 dynamic_cast<WalkerSenderState *>(pkt->popSenderState()); 127 WalkerState * senderWalk = senderState->senderWalk; 128 bool walkComplete = senderWalk->recvPacket(pkt); 129 delete senderState; 130 if (walkComplete) { 131 std::list<WalkerState *>::iterator iter; 132 for (iter = currStates.begin(); iter != currStates.end(); iter++) { 133 WalkerState * walkerState = *(iter); 134 if (walkerState == senderWalk) { 135 iter = currStates.erase(iter); 136 break; 137 } 138 } 139 delete senderWalk; 140 // Since we block requests when another is outstanding, we 141 // need to check if there is a waiting request to be serviced 142 if (currStates.size()) 143 startWalkWrapper(); 144 } 145 return true; 146} 147 148void 149Walker::WalkerPort::recvRetry() 150{ 151 walker->recvRetry(); 152} 153 154void 155Walker::recvRetry() 156{ 157 std::list<WalkerState *>::iterator iter; 158 for (iter = currStates.begin(); iter != currStates.end(); iter++) { 159 WalkerState * walkerState = *(iter); 160 if (walkerState->isRetrying()) { 161 walkerState->retry(); 162 } 163 } 164} 165 166bool Walker::sendTiming(WalkerState* sendingState, PacketPtr pkt) 167{ 168 WalkerSenderState* walker_state = new WalkerSenderState(sendingState); 169 pkt->pushSenderState(walker_state); 170 if (port.sendTimingReq(pkt)) { 171 return true; 172 } else { 173 // undo the adding of the sender state and delete it, as we 174 // will do it again the next time we attempt to send it 175 pkt->popSenderState(); 176 delete walker_state; 177 return false; 178 } 179 180} 181 182BaseMasterPort & 183Walker::getMasterPort(const std::string &if_name, PortID idx) 184{ 185 if (if_name == "port") 186 return port; 187 else 188 return MemObject::getMasterPort(if_name, idx); 189} 190 191void 192Walker::WalkerState::initState(ThreadContext * _tc, 193 BaseTLB::Mode _mode, bool _isTiming) 194{ 195 assert(state == Ready); 196 started = false; 197 tc = _tc; 198 mode = _mode; 199 timing = _isTiming; 200} 201 202void 203Walker::startWalkWrapper() 204{ 205 unsigned num_squashed = 0; 206 WalkerState *currState = currStates.front(); 207 while ((num_squashed < numSquashable) && currState && 208 currState->translation->squashed()) { 209 currStates.pop_front(); 210 num_squashed++; 211 212 DPRINTF(PageTableWalker, "Squashing table walk for address %#x\n", 213 currState->req->getVaddr()); 214 215 // finish the translation which will delete the translation object 216 currState->translation->finish(new UnimpFault("Squashed Inst"), 217 currState->req, currState->tc, currState->mode); 218 219 // delete the current request 220 delete currState; 221 222 // check the next translation request, if it exists 223 if (currStates.size()) 224 currState = currStates.front(); 225 else 226 currState = NULL; 227 } 228 if (currState && !currState->wasStarted()) 229 currState->startWalk(); 230} 231 232Fault 233Walker::WalkerState::startWalk() 234{ 235 Fault fault = NoFault; 236 assert(!started); 237 started = true; 238 setupWalk(req->getVaddr()); 239 if (timing) { 240 nextState = state; 241 state = Waiting; 242 timingFault = NoFault; 243 sendPackets(); 244 } else { 245 do { 246 walker->port.sendAtomic(read); 247 PacketPtr write = NULL; 248 fault = stepWalk(write); 249 assert(fault == NoFault || read == NULL); 250 state = nextState; 251 nextState = Ready; 252 if (write) 253 walker->port.sendAtomic(write); 254 } while(read); 255 state = Ready; 256 nextState = Waiting; 257 } 258 return fault; 259} 260 261Fault 262Walker::WalkerState::startFunctional(Addr &addr, unsigned &logBytes) 263{ 264 Fault fault = NoFault; 265 assert(!started); 266 started = true; 267 setupWalk(addr); 268 269 do { 270 walker->port.sendFunctional(read); 271 // On a functional access (page table lookup), writes should 272 // not happen so this pointer is ignored after stepWalk 273 PacketPtr write = NULL; 274 fault = stepWalk(write); 275 assert(fault == NoFault || read == NULL); 276 state = nextState; 277 nextState = Ready; 278 } while(read); 279 logBytes = entry.logBytes; 280 addr = entry.paddr; 281 282 return fault; 283} 284 285Fault 286Walker::WalkerState::stepWalk(PacketPtr &write) 287{ 288 assert(state != Ready && state != Waiting); 289 Fault fault = NoFault; 290 write = NULL; 291 PageTableEntry pte; 292 if (dataSize == 8) 293 pte = read->get<uint64_t>(); 294 else 295 pte = read->get<uint32_t>(); 296 VAddr vaddr = entry.vaddr; 297 bool uncacheable = pte.pcd; 298 Addr nextRead = 0; 299 bool doWrite = false; 300 bool doTLBInsert = false; 301 bool doEndWalk = false; 302 bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX; 303 switch(state) { 304 case LongPML4: 305 DPRINTF(PageTableWalker, 306 "Got long mode PML4 entry %#016x.\n", (uint64_t)pte); 307 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * dataSize; 308 doWrite = !pte.a; 309 pte.a = 1; 310 entry.writable = pte.w; 311 entry.user = pte.u; 312 if (badNX || !pte.p) { 313 doEndWalk = true; 314 fault = pageFault(pte.p); 315 break; 316 } 317 entry.noExec = pte.nx; 318 nextState = LongPDP; 319 break; 320 case LongPDP: 321 DPRINTF(PageTableWalker, 322 "Got long mode PDP entry %#016x.\n", (uint64_t)pte); 323 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * dataSize; 324 doWrite = !pte.a; 325 pte.a = 1; 326 entry.writable = entry.writable && pte.w; 327 entry.user = entry.user && pte.u; 328 if (badNX || !pte.p) { 329 doEndWalk = true; 330 fault = pageFault(pte.p); 331 break; 332 } 333 nextState = LongPD; 334 break; 335 case LongPD: 336 DPRINTF(PageTableWalker, 337 "Got long mode PD entry %#016x.\n", (uint64_t)pte); 338 doWrite = !pte.a; 339 pte.a = 1; 340 entry.writable = entry.writable && pte.w; 341 entry.user = entry.user && pte.u; 342 if (badNX || !pte.p) { 343 doEndWalk = true; 344 fault = pageFault(pte.p); 345 break; 346 } 347 if (!pte.ps) { 348 // 4 KB page 349 entry.logBytes = 12; 350 nextRead = 351 ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * dataSize; 352 nextState = LongPTE; 353 break; 354 } else { 355 // 2 MB page 356 entry.logBytes = 21; 357 entry.paddr = (uint64_t)pte & (mask(31) << 21); 358 entry.uncacheable = uncacheable; 359 entry.global = pte.g; 360 entry.patBit = bits(pte, 12); 361 entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); 362 doTLBInsert = true; 363 doEndWalk = true; 364 break; 365 } 366 case LongPTE: 367 DPRINTF(PageTableWalker, 368 "Got long mode PTE entry %#016x.\n", (uint64_t)pte); 369 doWrite = !pte.a; 370 pte.a = 1; 371 entry.writable = entry.writable && pte.w; 372 entry.user = entry.user && pte.u; 373 if (badNX || !pte.p) { 374 doEndWalk = true; 375 fault = pageFault(pte.p); 376 break; 377 } 378 entry.paddr = (uint64_t)pte & (mask(40) << 12); 379 entry.uncacheable = uncacheable; 380 entry.global = pte.g; 381 entry.patBit = bits(pte, 12); 382 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); 383 doTLBInsert = true; 384 doEndWalk = true; 385 break; 386 case PAEPDP: 387 DPRINTF(PageTableWalker, 388 "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte); 389 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * dataSize; 390 if (!pte.p) { 391 doEndWalk = true; 392 fault = pageFault(pte.p); 393 break; 394 } 395 nextState = PAEPD; 396 break; 397 case PAEPD: 398 DPRINTF(PageTableWalker, 399 "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte); 400 doWrite = !pte.a; 401 pte.a = 1; 402 entry.writable = pte.w; 403 entry.user = pte.u; 404 if (badNX || !pte.p) { 405 doEndWalk = true; 406 fault = pageFault(pte.p); 407 break; 408 } 409 if (!pte.ps) { 410 // 4 KB page 411 entry.logBytes = 12; 412 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * dataSize; 413 nextState = PAEPTE; 414 break; 415 } else { 416 // 2 MB page 417 entry.logBytes = 21; 418 entry.paddr = (uint64_t)pte & (mask(31) << 21); 419 entry.uncacheable = uncacheable; 420 entry.global = pte.g; 421 entry.patBit = bits(pte, 12); 422 entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1); 423 doTLBInsert = true; 424 doEndWalk = true; 425 break; 426 } 427 case PAEPTE: 428 DPRINTF(PageTableWalker, 429 "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte); 430 doWrite = !pte.a; 431 pte.a = 1; 432 entry.writable = entry.writable && pte.w; 433 entry.user = entry.user && pte.u; 434 if (badNX || !pte.p) { 435 doEndWalk = true; 436 fault = pageFault(pte.p); 437 break; 438 } 439 entry.paddr = (uint64_t)pte & (mask(40) << 12); 440 entry.uncacheable = uncacheable; 441 entry.global = pte.g; 442 entry.patBit = bits(pte, 7); 443 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); 444 doTLBInsert = true; 445 doEndWalk = true; 446 break; 447 case PSEPD: 448 DPRINTF(PageTableWalker, 449 "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte); 450 doWrite = !pte.a; 451 pte.a = 1; 452 entry.writable = pte.w; 453 entry.user = pte.u; 454 if (!pte.p) { 455 doEndWalk = true; 456 fault = pageFault(pte.p); 457 break; 458 } 459 if (!pte.ps) { 460 // 4 KB page 461 entry.logBytes = 12; 462 nextRead = 463 ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize; 464 nextState = PTE; 465 break; 466 } else { 467 // 4 MB page 468 entry.logBytes = 21; 469 entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22; 470 entry.uncacheable = uncacheable; 471 entry.global = pte.g; 472 entry.patBit = bits(pte, 12); 473 entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1); 474 doTLBInsert = true; 475 doEndWalk = true; 476 break; 477 } 478 case PD: 479 DPRINTF(PageTableWalker, 480 "Got legacy mode PD entry %#08x.\n", (uint32_t)pte); 481 doWrite = !pte.a; 482 pte.a = 1; 483 entry.writable = pte.w; 484 entry.user = pte.u; 485 if (!pte.p) { 486 doEndWalk = true; 487 fault = pageFault(pte.p); 488 break; 489 } 490 // 4 KB page 491 entry.logBytes = 12; 492 nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize; 493 nextState = PTE; 494 break; 495 case PTE: 496 DPRINTF(PageTableWalker, 497 "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte); 498 doWrite = !pte.a; 499 pte.a = 1; 500 entry.writable = pte.w; 501 entry.user = pte.u; 502 if (!pte.p) { 503 doEndWalk = true; 504 fault = pageFault(pte.p); 505 break; 506 } 507 entry.paddr = (uint64_t)pte & (mask(20) << 12); 508 entry.uncacheable = uncacheable; 509 entry.global = pte.g; 510 entry.patBit = bits(pte, 7); 511 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1); 512 doTLBInsert = true; 513 doEndWalk = true; 514 break; 515 default: 516 panic("Unknown page table walker state %d!\n"); 517 } 518 if (doEndWalk) { 519 if (doTLBInsert) 520 if (!functional) 521 walker->tlb->insert(entry.vaddr, entry); 522 endWalk(); 523 } else { 524 PacketPtr oldRead = read; 525 //If we didn't return, we're setting up another read. 526 Request::Flags flags = oldRead->req->getFlags(); 527 flags.set(Request::UNCACHEABLE, uncacheable); 528 RequestPtr request = 529 new Request(nextRead, oldRead->getSize(), flags, walker->masterId); 530 read = new Packet(request, MemCmd::ReadReq); 531 read->allocate(); 532 // If we need to write, adjust the read packet to write the modified 533 // value back to memory. 534 if (doWrite) { 535 write = oldRead; 536 write->set<uint64_t>(pte); 537 write->cmd = MemCmd::WriteReq; 538 write->clearDest(); 539 } else { 540 write = NULL; 541 delete oldRead->req; 542 delete oldRead; 543 } 544 } 545 return fault; 546} 547 548void 549Walker::WalkerState::endWalk() 550{ 551 nextState = Ready; 552 delete read->req; 553 delete read; 554 read = NULL; 555} 556 557void 558Walker::WalkerState::setupWalk(Addr vaddr) 559{ 560 VAddr addr = vaddr; 561 CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3); 562 // Check if we're in long mode or not 563 Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER); 564 dataSize = 8; 565 Addr topAddr; 566 if (efer.lma) { 567 // Do long mode. 568 state = LongPML4; 569 topAddr = (cr3.longPdtb << 12) + addr.longl4 * dataSize; 570 enableNX = efer.nxe; 571 } else { 572 // We're in some flavor of legacy mode. 573 CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4); 574 if (cr4.pae) { 575 // Do legacy PAE. 576 state = PAEPDP; 577 topAddr = (cr3.paePdtb << 5) + addr.pael3 * dataSize; 578 enableNX = efer.nxe; 579 } else { 580 dataSize = 4; 581 topAddr = (cr3.pdtb << 12) + addr.norml2 * dataSize; 582 if (cr4.pse) { 583 // Do legacy PSE. 584 state = PSEPD; 585 } else { 586 // Do legacy non PSE. 587 state = PD; 588 } 589 enableNX = false; 590 } 591 } 592 593 nextState = Ready; 594 entry.vaddr = vaddr; 595 596 Request::Flags flags = Request::PHYSICAL; 597 if (cr3.pcd) 598 flags.set(Request::UNCACHEABLE); 599 RequestPtr request = new Request(topAddr, dataSize, flags, 600 walker->masterId); 601 read = new Packet(request, MemCmd::ReadReq); 602 read->allocate(); 603} 604 605bool 606Walker::WalkerState::recvPacket(PacketPtr pkt) 607{ 608 assert(pkt->isResponse()); 609 assert(inflight); 610 assert(state == Waiting); 611 assert(!read); 612 inflight--; 613 if (pkt->isRead()) { 614 // @todo someone should pay for this 615 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0; 616 617 state = nextState; 618 nextState = Ready; 619 PacketPtr write = NULL; 620 read = pkt; 621 timingFault = stepWalk(write); 622 state = Waiting; 623 assert(timingFault == NoFault || read == NULL); 624 if (write) { 625 writes.push_back(write); 626 } 627 sendPackets(); 628 } else { 629 sendPackets(); 630 } 631 if (inflight == 0 && read == NULL && writes.size() == 0) { 632 state = Ready; 633 nextState = Waiting; 634 if (timingFault == NoFault) { 635 /* 636 * Finish the translation. Now that we now the right entry is 637 * in the TLB, this should work with no memory accesses. 638 * There could be new faults unrelated to the table walk like 639 * permissions violations, so we'll need the return value as 640 * well. 641 */ 642 bool delayedResponse; 643 Fault fault = walker->tlb->translate(req, tc, NULL, mode, 644 delayedResponse, true); 645 assert(!delayedResponse); 646 // Let the CPU continue. 647 translation->finish(fault, req, tc, mode); 648 } else { 649 // There was a fault during the walk. Let the CPU know. 650 translation->finish(timingFault, req, tc, mode); 651 } 652 return true; 653 } 654 655 return false; 656} 657 658void 659Walker::WalkerState::sendPackets() 660{ 661 //If we're already waiting for the port to become available, just return. 662 if (retrying) 663 return; 664 665 //Reads always have priority 666 if (read) { 667 PacketPtr pkt = read; 668 read = NULL; 669 inflight++; 670 if (!walker->sendTiming(this, pkt)) { 671 retrying = true; 672 read = pkt; 673 inflight--; 674 return; 675 } 676 } 677 //Send off as many of the writes as we can. 678 while (writes.size()) { 679 PacketPtr write = writes.back(); 680 writes.pop_back(); 681 inflight++; 682 if (!walker->sendTiming(this, write)) { 683 retrying = true; 684 writes.push_back(write); 685 inflight--; 686 return; 687 } 688 } 689} 690 691bool 692Walker::WalkerState::isRetrying() 693{ 694 return retrying; 695} 696 697bool 698Walker::WalkerState::isTiming() 699{ 700 return timing; 701} 702 703bool 704Walker::WalkerState::wasStarted() 705{ 706 return started; 707} 708 709void 710Walker::WalkerState::retry() 711{ 712 retrying = false; 713 sendPackets(); 714} 715 716Fault 717Walker::WalkerState::pageFault(bool present) 718{ 719 DPRINTF(PageTableWalker, "Raising page fault.\n"); 720 HandyM5Reg m5reg = tc->readMiscRegNoEffect(MISCREG_M5_REG); 721 if (mode == BaseTLB::Execute && !enableNX) 722 mode = BaseTLB::Read; 723 return new PageFault(entry.vaddr, present, mode, m5reg.cpl == 3, false); 724} 725 726/* end namespace X86ISA */ } 727 728X86ISA::Walker * 729X86PagetableWalkerParams::create() 730{ 731 return new X86ISA::Walker(this); 732} 733