1/*
2 * Copyright (c) 2012 ARM Limited
3 * All rights reserved.
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2007 The Hewlett-Packard Development Company
15 * All rights reserved.
16 *
17 * The license below extends only to copyright in the software and shall
18 * not be construed as granting a license to any other intellectual
19 * property including but not limited to intellectual property relating
20 * to a hardware implementation of the functionality of the software
21 * licensed hereunder. You may use the software subject to the license
22 * terms below provided that you ensure that this notice is replicated
23 * unmodified and in its entirety in all distributions of the software,
24 * modified or unmodified, in source code or in binary form.
25 *
26 * Redistribution and use in source and binary forms, with or without
27 * modification, are permitted provided that the following conditions are
28 * met: redistributions of source code must retain the above copyright
29 * notice, this list of conditions and the following disclaimer;
30 * redistributions in binary form must reproduce the above copyright
31 * notice, this list of conditions and the following disclaimer in the
32 * documentation and/or other materials provided with the distribution;
33 * neither the name of the copyright holders nor the names of its
34 * contributors may be used to endorse or promote products derived from
35 * this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 * Authors: Gabe Black
50 */
51
52#include "arch/x86/pagetable.hh"
53#include "arch/x86/pagetable_walker.hh"
54#include "arch/x86/tlb.hh"
55#include "arch/x86/vtophys.hh"
56#include "base/bitfield.hh"
57#include "base/trie.hh"
58#include "cpu/base.hh"
59#include "cpu/thread_context.hh"
60#include "debug/PageTableWalker.hh"
61#include "mem/packet_access.hh"
62#include "mem/request.hh"
63
64namespace X86ISA {
65
66// Unfortunately, the placement of the base field in a page table entry is
67// very erratic and would make a mess here. It might be moved here at some
68// point in the future.
69BitUnion64(PageTableEntry)
70 Bitfield<63> nx;
71 Bitfield<11, 9> avl;
72 Bitfield<8> g;
73 Bitfield<7> ps;
74 Bitfield<6> d;
75 Bitfield<5> a;
76 Bitfield<4> pcd;
77 Bitfield<3> pwt;
78 Bitfield<2> u;
79 Bitfield<1> w;
80 Bitfield<0> p;
81EndBitUnion(PageTableEntry)
82
83Fault
84Walker::start(ThreadContext * _tc, BaseTLB::Translation *_translation,
85 RequestPtr _req, BaseTLB::Mode _mode)
86{
87 // TODO: in timing mode, instead of blocking when there are other
88 // outstanding requests, see if this request can be coalesced with
89 // another one (i.e. either coalesce or start walk)
90 WalkerState * newState = new WalkerState(this, _translation, _req);
91 newState->initState(_tc, _mode, sys->isTimingMode());
92 if (currStates.size()) {
93 assert(newState->isTiming());
94 DPRINTF(PageTableWalker, "Walks in progress: %d\n", currStates.size());
95 currStates.push_back(newState);
96 return NoFault;
97 } else {
98 currStates.push_back(newState);
99 Fault fault = newState->startWalk();
100 if (!newState->isTiming()) {
101 currStates.pop_front();
102 delete newState;
103 }
104 return fault;
105 }
106}
107
108Fault
109Walker::startFunctional(ThreadContext * _tc, Addr &addr, unsigned &logBytes,
110 BaseTLB::Mode _mode)
111{
112 funcState.initState(_tc, _mode);
113 return funcState.startFunctional(addr, logBytes);
114}
115
116bool
117Walker::WalkerPort::recvTimingResp(PacketPtr pkt)
118{
119 return walker->recvTimingResp(pkt);
120}
121
122bool
123Walker::recvTimingResp(PacketPtr pkt)
124{
125 WalkerSenderState * senderState =
126 dynamic_cast<WalkerSenderState *>(pkt->popSenderState());
127 WalkerState * senderWalk = senderState->senderWalk;
128 bool walkComplete = senderWalk->recvPacket(pkt);
129 delete senderState;
130 if (walkComplete) {
131 std::list<WalkerState *>::iterator iter;
132 for (iter = currStates.begin(); iter != currStates.end(); iter++) {
133 WalkerState * walkerState = *(iter);
134 if (walkerState == senderWalk) {
135 iter = currStates.erase(iter);
136 break;
137 }
138 }
139 delete senderWalk;
140 // Since we block requests when another is outstanding, we
141 // need to check if there is a waiting request to be serviced
142 if (currStates.size()) {
143 WalkerState * newState = currStates.front();
144 if (!newState->wasStarted())
145 newState->startWalk();
146 }
147 }
148 return true;
149}
150
151void
152Walker::WalkerPort::recvRetry()
153{
154 walker->recvRetry();
155}
156
157void
158Walker::recvRetry()
159{
160 std::list<WalkerState *>::iterator iter;
161 for (iter = currStates.begin(); iter != currStates.end(); iter++) {
162 WalkerState * walkerState = *(iter);
163 if (walkerState->isRetrying()) {
164 walkerState->retry();
165 }
166 }
167}
168
169bool Walker::sendTiming(WalkerState* sendingState, PacketPtr pkt)
170{
171 pkt->pushSenderState(new WalkerSenderState(sendingState));
172 return port.sendTimingReq(pkt);
173}
174
175BaseMasterPort &
176Walker::getMasterPort(const std::string &if_name, PortID idx)
177{
178 if (if_name == "port")
179 return port;
180 else
181 return MemObject::getMasterPort(if_name, idx);
182}
183
184void
185Walker::WalkerState::initState(ThreadContext * _tc,
186 BaseTLB::Mode _mode, bool _isTiming)
187{
188 assert(state == Ready);
189 started = false;
190 tc = _tc;
191 mode = _mode;
192 timing = _isTiming;
193}
194
195Fault
196Walker::WalkerState::startWalk()
197{
198 Fault fault = NoFault;
199 assert(started == false);
200 started = true;
201 setupWalk(req->getVaddr());
202 if (timing) {
203 nextState = state;
204 state = Waiting;
205 timingFault = NoFault;
206 sendPackets();
207 } else {
208 do {
209 walker->port.sendAtomic(read);
210 PacketPtr write = NULL;
211 fault = stepWalk(write);
212 assert(fault == NoFault || read == NULL);
213 state = nextState;
214 nextState = Ready;
215 if (write)
216 walker->port.sendAtomic(write);
217 } while(read);
218 state = Ready;
219 nextState = Waiting;
220 }
221 return fault;
222}
223
224Fault
225Walker::WalkerState::startFunctional(Addr &addr, unsigned &logBytes)
226{
227 Fault fault = NoFault;
228 assert(started == false);
229 started = true;
230 setupWalk(addr);
231
232 do {
233 walker->port.sendFunctional(read);
234 // On a functional access (page table lookup), writes should
235 // not happen so this pointer is ignored after stepWalk
236 PacketPtr write = NULL;
237 fault = stepWalk(write);
238 assert(fault == NoFault || read == NULL);
239 state = nextState;
240 nextState = Ready;
241 } while(read);
242 logBytes = entry.logBytes;
243 addr = entry.paddr;
244
245 return fault;
246}
247
248Fault
249Walker::WalkerState::stepWalk(PacketPtr &write)
250{
251 assert(state != Ready && state != Waiting);
252 Fault fault = NoFault;
253 write = NULL;
254 PageTableEntry pte;
255 if (dataSize == 8)
256 pte = read->get<uint64_t>();
257 else
258 pte = read->get<uint32_t>();
259 VAddr vaddr = entry.vaddr;
260 bool uncacheable = pte.pcd;
261 Addr nextRead = 0;
262 bool doWrite = false;
263 bool doTLBInsert = false;
264 bool doEndWalk = false;
265 bool badNX = pte.nx && mode == BaseTLB::Execute && enableNX;
266 switch(state) {
267 case LongPML4:
268 DPRINTF(PageTableWalker,
269 "Got long mode PML4 entry %#016x.\n", (uint64_t)pte);
270 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl3 * dataSize;
271 doWrite = !pte.a;
272 pte.a = 1;
273 entry.writable = pte.w;
274 entry.user = pte.u;
275 if (badNX || !pte.p) {
276 doEndWalk = true;
277 fault = pageFault(pte.p);
278 break;
279 }
280 entry.noExec = pte.nx;
281 nextState = LongPDP;
282 break;
283 case LongPDP:
284 DPRINTF(PageTableWalker,
285 "Got long mode PDP entry %#016x.\n", (uint64_t)pte);
286 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl2 * dataSize;
287 doWrite = !pte.a;
288 pte.a = 1;
289 entry.writable = entry.writable && pte.w;
290 entry.user = entry.user && pte.u;
291 if (badNX || !pte.p) {
292 doEndWalk = true;
293 fault = pageFault(pte.p);
294 break;
295 }
296 nextState = LongPD;
297 break;
298 case LongPD:
299 DPRINTF(PageTableWalker,
300 "Got long mode PD entry %#016x.\n", (uint64_t)pte);
301 doWrite = !pte.a;
302 pte.a = 1;
303 entry.writable = entry.writable && pte.w;
304 entry.user = entry.user && pte.u;
305 if (badNX || !pte.p) {
306 doEndWalk = true;
307 fault = pageFault(pte.p);
308 break;
309 }
310 if (!pte.ps) {
311 // 4 KB page
312 entry.logBytes = 12;
313 nextRead =
314 ((uint64_t)pte & (mask(40) << 12)) + vaddr.longl1 * dataSize;
315 nextState = LongPTE;
316 break;
317 } else {
318 // 2 MB page
319 entry.logBytes = 21;
320 entry.paddr = (uint64_t)pte & (mask(31) << 21);
321 entry.uncacheable = uncacheable;
322 entry.global = pte.g;
323 entry.patBit = bits(pte, 12);
324 entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
325 doTLBInsert = true;
326 doEndWalk = true;
327 break;
328 }
329 case LongPTE:
330 DPRINTF(PageTableWalker,
331 "Got long mode PTE entry %#016x.\n", (uint64_t)pte);
332 doWrite = !pte.a;
333 pte.a = 1;
334 entry.writable = entry.writable && pte.w;
335 entry.user = entry.user && pte.u;
336 if (badNX || !pte.p) {
337 doEndWalk = true;
338 fault = pageFault(pte.p);
339 break;
340 }
341 entry.paddr = (uint64_t)pte & (mask(40) << 12);
342 entry.uncacheable = uncacheable;
343 entry.global = pte.g;
344 entry.patBit = bits(pte, 12);
345 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
346 doTLBInsert = true;
347 doEndWalk = true;
348 break;
349 case PAEPDP:
350 DPRINTF(PageTableWalker,
351 "Got legacy mode PAE PDP entry %#08x.\n", (uint32_t)pte);
352 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael2 * dataSize;
353 if (!pte.p) {
354 doEndWalk = true;
355 fault = pageFault(pte.p);
356 break;
357 }
358 nextState = PAEPD;
359 break;
360 case PAEPD:
361 DPRINTF(PageTableWalker,
362 "Got legacy mode PAE PD entry %#08x.\n", (uint32_t)pte);
363 doWrite = !pte.a;
364 pte.a = 1;
365 entry.writable = pte.w;
366 entry.user = pte.u;
367 if (badNX || !pte.p) {
368 doEndWalk = true;
369 fault = pageFault(pte.p);
370 break;
371 }
372 if (!pte.ps) {
373 // 4 KB page
374 entry.logBytes = 12;
375 nextRead = ((uint64_t)pte & (mask(40) << 12)) + vaddr.pael1 * dataSize;
376 nextState = PAEPTE;
377 break;
378 } else {
379 // 2 MB page
380 entry.logBytes = 21;
381 entry.paddr = (uint64_t)pte & (mask(31) << 21);
382 entry.uncacheable = uncacheable;
383 entry.global = pte.g;
384 entry.patBit = bits(pte, 12);
385 entry.vaddr = entry.vaddr & ~((2 * (1 << 20)) - 1);
386 doTLBInsert = true;
387 doEndWalk = true;
388 break;
389 }
390 case PAEPTE:
391 DPRINTF(PageTableWalker,
392 "Got legacy mode PAE PTE entry %#08x.\n", (uint32_t)pte);
393 doWrite = !pte.a;
394 pte.a = 1;
395 entry.writable = entry.writable && pte.w;
396 entry.user = entry.user && pte.u;
397 if (badNX || !pte.p) {
398 doEndWalk = true;
399 fault = pageFault(pte.p);
400 break;
401 }
402 entry.paddr = (uint64_t)pte & (mask(40) << 12);
403 entry.uncacheable = uncacheable;
404 entry.global = pte.g;
405 entry.patBit = bits(pte, 7);
406 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
407 doTLBInsert = true;
408 doEndWalk = true;
409 break;
410 case PSEPD:
411 DPRINTF(PageTableWalker,
412 "Got legacy mode PSE PD entry %#08x.\n", (uint32_t)pte);
413 doWrite = !pte.a;
414 pte.a = 1;
415 entry.writable = pte.w;
416 entry.user = pte.u;
417 if (!pte.p) {
418 doEndWalk = true;
419 fault = pageFault(pte.p);
420 break;
421 }
422 if (!pte.ps) {
423 // 4 KB page
424 entry.logBytes = 12;
425 nextRead =
426 ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize;
427 nextState = PTE;
428 break;
429 } else {
430 // 4 MB page
431 entry.logBytes = 21;
432 entry.paddr = bits(pte, 20, 13) << 32 | bits(pte, 31, 22) << 22;
433 entry.uncacheable = uncacheable;
434 entry.global = pte.g;
435 entry.patBit = bits(pte, 12);
436 entry.vaddr = entry.vaddr & ~((4 * (1 << 20)) - 1);
437 doTLBInsert = true;
438 doEndWalk = true;
439 break;
440 }
441 case PD:
442 DPRINTF(PageTableWalker,
443 "Got legacy mode PD entry %#08x.\n", (uint32_t)pte);
444 doWrite = !pte.a;
445 pte.a = 1;
446 entry.writable = pte.w;
447 entry.user = pte.u;
448 if (!pte.p) {
449 doEndWalk = true;
450 fault = pageFault(pte.p);
451 break;
452 }
453 // 4 KB page
454 entry.logBytes = 12;
455 nextRead = ((uint64_t)pte & (mask(20) << 12)) + vaddr.norml2 * dataSize;
456 nextState = PTE;
457 break;
458 case PTE:
459 DPRINTF(PageTableWalker,
460 "Got legacy mode PTE entry %#08x.\n", (uint32_t)pte);
461 doWrite = !pte.a;
462 pte.a = 1;
463 entry.writable = pte.w;
464 entry.user = pte.u;
465 if (!pte.p) {
466 doEndWalk = true;
467 fault = pageFault(pte.p);
468 break;
469 }
470 entry.paddr = (uint64_t)pte & (mask(20) << 12);
471 entry.uncacheable = uncacheable;
472 entry.global = pte.g;
473 entry.patBit = bits(pte, 7);
474 entry.vaddr = entry.vaddr & ~((4 * (1 << 10)) - 1);
475 doTLBInsert = true;
476 doEndWalk = true;
477 break;
478 default:
479 panic("Unknown page table walker state %d!\n");
480 }
481 if (doEndWalk) {
482 if (doTLBInsert)
483 if (!functional)
484 walker->tlb->insert(entry.vaddr, entry);
485 endWalk();
486 } else {
487 PacketPtr oldRead = read;
488 //If we didn't return, we're setting up another read.
489 Request::Flags flags = oldRead->req->getFlags();
490 flags.set(Request::UNCACHEABLE, uncacheable);
491 RequestPtr request =
492 new Request(nextRead, oldRead->getSize(), flags, walker->masterId);
493 read = new Packet(request, MemCmd::ReadReq);
494 read->allocate();
495 // If we need to write, adjust the read packet to write the modified
496 // value back to memory.
497 if (doWrite) {
498 write = oldRead;
499 write->set<uint64_t>(pte);
500 write->cmd = MemCmd::WriteReq;
501 write->clearDest();
502 } else {
503 write = NULL;
504 delete oldRead->req;
505 delete oldRead;
506 }
507 }
508 return fault;
509}
510
511void
512Walker::WalkerState::endWalk()
513{
514 nextState = Ready;
515 delete read->req;
516 delete read;
517 read = NULL;
518}
519
520void
521Walker::WalkerState::setupWalk(Addr vaddr)
522{
523 VAddr addr = vaddr;
524 CR3 cr3 = tc->readMiscRegNoEffect(MISCREG_CR3);
525 // Check if we're in long mode or not
526 Efer efer = tc->readMiscRegNoEffect(MISCREG_EFER);
527 dataSize = 8;
528 Addr topAddr;
529 if (efer.lma) {
530 // Do long mode.
531 state = LongPML4;
532 topAddr = (cr3.longPdtb << 12) + addr.longl4 * dataSize;
533 enableNX = efer.nxe;
534 } else {
535 // We're in some flavor of legacy mode.
536 CR4 cr4 = tc->readMiscRegNoEffect(MISCREG_CR4);
537 if (cr4.pae) {
538 // Do legacy PAE.
539 state = PAEPDP;
540 topAddr = (cr3.paePdtb << 5) + addr.pael3 * dataSize;
541 enableNX = efer.nxe;
542 } else {
543 dataSize = 4;
544 topAddr = (cr3.pdtb << 12) + addr.norml2 * dataSize;
545 if (cr4.pse) {
546 // Do legacy PSE.
547 state = PSEPD;
548 } else {
549 // Do legacy non PSE.
550 state = PD;
551 }
552 enableNX = false;
553 }
554 }
555
556 nextState = Ready;
557 entry.vaddr = vaddr;
558
559 Request::Flags flags = Request::PHYSICAL;
560 if (cr3.pcd)
561 flags.set(Request::UNCACHEABLE);
562 RequestPtr request = new Request(topAddr, dataSize, flags,
563 walker->masterId);
564 read = new Packet(request, MemCmd::ReadReq);
565 read->allocate();
566}
567
568bool
569Walker::WalkerState::recvPacket(PacketPtr pkt)
570{
571 assert(pkt->isResponse());
572 assert(inflight);
573 assert(state == Waiting);
574 assert(!read);
575 inflight--;
576 if (pkt->isRead()) {
577 // @todo someone should pay for this
578 pkt->busFirstWordDelay = pkt->busLastWordDelay = 0;
579
580 state = nextState;
581 nextState = Ready;
582 PacketPtr write = NULL;
583 read = pkt;
584 timingFault = stepWalk(write);
585 state = Waiting;
586 assert(timingFault == NoFault || read == NULL);
587 if (write) {
588 writes.push_back(write);
589 }
590 sendPackets();
591 } else {
592 sendPackets();
593 }
594 if (inflight == 0 && read == NULL && writes.size() == 0) {
595 state = Ready;
596 nextState = Waiting;
597 if (timingFault == NoFault) {
598 /*
599 * Finish the translation. Now that we now the right entry is
600 * in the TLB, this should work with no memory accesses.
601 * There could be new faults unrelated to the table walk like
602 * permissions violations, so we'll need the return value as
603 * well.
604 */
605 bool delayedResponse;
606 Fault fault = walker->tlb->translate(req, tc, NULL, mode,
607 delayedResponse, true);
608 assert(!delayedResponse);
609 // Let the CPU continue.
610 translation->finish(fault, req, tc, mode);
611 } else {
612 // There was a fault during the walk. Let the CPU know.
613 translation->finish(timingFault, req, tc, mode);
614 }
615 return true;
616 }
617
618 return false;
619}
620
621void
622Walker::WalkerState::sendPackets()
623{
624 //If we're already waiting for the port to become available, just return.
625 if (retrying)
626 return;
627
628 //Reads always have priority
629 if (read) {
630 PacketPtr pkt = read;
631 read = NULL;
632 inflight++;
633 if (!walker->sendTiming(this, pkt)) {
634 retrying = true;
635 read = pkt;
636 inflight--;
637 return;
638 }
639 }
640 //Send off as many of the writes as we can.
641 while (writes.size()) {
642 PacketPtr write = writes.back();
643 writes.pop_back();
644 inflight++;
645 if (!walker->sendTiming(this, write)) {
646 retrying = true;
647 writes.push_back(write);
648 inflight--;
649 return;
650 }
651 }
652}
653
654bool
655Walker::WalkerState::isRetrying()
656{
657 return retrying;
658}
659
660bool
661Walker::WalkerState::isTiming()
662{
663 return timing;
664}
665
666bool
667Walker::WalkerState::wasStarted()
668{
669 return started;
670}
671
672void
673Walker::WalkerState::retry()
674{
675 retrying = false;
676 sendPackets();
677}
678
679Fault
680Walker::WalkerState::pageFault(bool present)
681{
682 DPRINTF(PageTableWalker, "Raising page fault.\n");
683 HandyM5Reg m5reg = tc->readMiscRegNoEffect(MISCREG_M5_REG);
684 if (mode == BaseTLB::Execute && !enableNX)
685 mode = BaseTLB::Read;
686 return new PageFault(entry.vaddr, present, mode, m5reg.cpl == 3, false);
687}
688
689/* end namespace X86ISA */ }
690
691X86ISA::Walker *
692X86PagetableWalkerParams::create()
693{
694 return new X86ISA::Walker(this);
695}