170 DPRINTF(Drain, "TableWalker free, no need to drain\n");
171
172 // table walker is drained, but its ports may still need to be drained
173 return 0;
174 }
175}
176
177void
178TableWalker::drainResume()
179{
180 Drainable::drainResume();
181 if (params()->sys->isTimingMode() && currState) {
182 delete currState;
183 currState = NULL;
184 pendingChange();
185 }
186}
187
188Fault
189TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
190 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
191 TLB::Translation *_trans, bool _timing, bool _functional,
192 bool secure, TLB::ArmTranslationType tranType)
193{
194 assert(!(_functional && _timing));
195 ++statWalks;
196
197 WalkerState *savedCurrState = NULL;
198
199 if (!currState && !_functional) {
200 // For atomic mode, a new WalkerState instance should be only created
201 // once per TLB. For timing mode, a new instance is generated for every
202 // TLB miss.
203 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
204
205 currState = new WalkerState();
206 currState->tableWalker = this;
207 } else if (_functional) {
208 // If we are mixing functional mode with timing (or even
209 // atomic), we need to to be careful and clean up after
210 // ourselves to not risk getting into an inconsistent state.
211 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
212 savedCurrState = currState;
213 currState = new WalkerState();
214 currState->tableWalker = this;
215 } else if (_timing) {
216 // This is a translation that was completed and then faulted again
217 // because some underlying parameters that affect the translation
218 // changed out from under us (e.g. asid). It will either be a
219 // misprediction, in which case nothing will happen or we'll use
220 // this fault to re-execute the faulting instruction which should clean
221 // up everything.
222 if (currState->vaddr_tainted == _req->getVaddr()) {
223 ++statSquashedBefore;
224 return std::make_shared<ReExec>();
225 }
226 }
227 pendingChange();
228
229 currState->startTime = curTick();
230 currState->tc = _tc;
231 currState->aarch64 = opModeIs64(currOpMode(_tc));
232 currState->el = currEL(_tc);
233 currState->transState = _trans;
234 currState->req = _req;
235 currState->fault = NoFault;
236 currState->asid = _asid;
237 currState->vmid = _vmid;
238 currState->isHyp = _isHyp;
239 currState->timing = _timing;
240 currState->functional = _functional;
241 currState->mode = _mode;
242 currState->tranType = tranType;
243 currState->isSecure = secure;
244 currState->physAddrRange = physAddrRange;
245
246 /** @todo These should be cached or grabbed from cached copies in
247 the TLB, all these miscreg reads are expensive */
248 currState->vaddr_tainted = currState->req->getVaddr();
249 if (currState->aarch64)
250 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
251 currState->tc, currState->el);
252 else
253 currState->vaddr = currState->vaddr_tainted;
254
255 if (currState->aarch64) {
256 switch (currState->el) {
257 case EL0:
258 case EL1:
259 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
260 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
261 break;
262 // @todo: uncomment this to enable Virtualization
263 // case EL2:
264 // assert(haveVirtualization);
265 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
266 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
267 // break;
268 case EL3:
269 assert(haveSecurity);
270 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
271 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
272 break;
273 default:
274 panic("Invalid exception level");
275 break;
276 }
277 } else {
278 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
279 MISCREG_SCTLR, currState->tc, !currState->isSecure));
280 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
281 MISCREG_TTBCR, currState->tc, !currState->isSecure));
282 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
283 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
284 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
285 }
286 sctlr = currState->sctlr;
287
288 currState->isFetch = (currState->mode == TLB::Execute);
289 currState->isWrite = (currState->mode == TLB::Write);
290
291 statRequestOrigin[REQUESTED][currState->isFetch]++;
292
293 // We only do a second stage of translation if we're not secure, or in
294 // hyp mode, the second stage MMU is enabled, and this table walker
295 // instance is the first stage.
296 currState->doingStage2 = false;
297 // @todo: for now disable this in AArch64 (HCR is not set)
298 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
299 !isStage2 && !currState->isSecure && !currState->isHyp;
300
301 bool long_desc_format = currState->aarch64 ||
302 (_haveLPAE && currState->ttbcr.eae) ||
303 _isHyp || isStage2;
304
305 if (long_desc_format) {
306 // Helper variables used for hierarchical permissions
307 currState->secureLookup = currState->isSecure;
308 currState->rwTable = true;
309 currState->userTable = true;
310 currState->xnTable = false;
311 currState->pxnTable = false;
312
313 ++statWalksLongDescriptor;
314 } else {
315 ++statWalksShortDescriptor;
316 }
317
318 if (!currState->timing) {
319 Fault fault = NoFault;
320 if (currState->aarch64)
321 fault = processWalkAArch64();
322 else if (long_desc_format)
323 fault = processWalkLPAE();
324 else
325 fault = processWalk();
326
327 // If this was a functional non-timing access restore state to
328 // how we found it.
329 if (currState->functional) {
330 delete currState;
331 currState = savedCurrState;
332 }
333 return fault;
334 }
335
336 if (pending || pendingQueue.size()) {
337 pendingQueue.push_back(currState);
338 currState = NULL;
339 pendingChange();
340 } else {
341 pending = true;
342 pendingChange();
343 if (currState->aarch64)
344 return processWalkAArch64();
345 else if (long_desc_format)
346 return processWalkLPAE();
347 else
348 return processWalk();
349 }
350
351 return NoFault;
352}
353
354void
355TableWalker::processWalkWrapper()
356{
357 assert(!currState);
358 assert(pendingQueue.size());
359 pendingChange();
360 currState = pendingQueue.front();
361
362 ExceptionLevel target_el = EL0;
363 if (currState->aarch64)
364 target_el = currEL(currState->tc);
365 else
366 target_el = EL1;
367
368 // Check if a previous walk filled this request already
369 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
370 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
371 currState->vmid, currState->isHyp, currState->isSecure, true, false,
372 target_el);
373
374 // Check if we still need to have a walk for this request. If the requesting
375 // instruction has been squashed, or a previous walk has filled the TLB with
376 // a match, we just want to get rid of the walk. The latter could happen
377 // when there are multiple outstanding misses to a single page and a
378 // previous request has been successfully translated.
379 if (!currState->transState->squashed() && !te) {
380 // We've got a valid request, lets process it
381 pending = true;
382 pendingQueue.pop_front();
383 // Keep currState in case one of the processWalk... calls NULLs it
384 WalkerState *curr_state_copy = currState;
385 Fault f;
386 if (currState->aarch64)
387 f = processWalkAArch64();
388 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
389 f = processWalkLPAE();
390 else
391 f = processWalk();
392
393 if (f != NoFault) {
394 curr_state_copy->transState->finish(f, curr_state_copy->req,
395 curr_state_copy->tc, curr_state_copy->mode);
396
397 delete curr_state_copy;
398 }
399 return;
400 }
401
402
403 // If the instruction that we were translating for has been
404 // squashed we shouldn't bother.
405 unsigned num_squashed = 0;
406 ThreadContext *tc = currState->tc;
407 while ((num_squashed < numSquashable) && currState &&
408 (currState->transState->squashed() || te)) {
409 pendingQueue.pop_front();
410 num_squashed++;
411 statSquashedBefore++;
412
413 DPRINTF(TLB, "Squashing table walk for address %#x\n",
414 currState->vaddr_tainted);
415
416 if (currState->transState->squashed()) {
417 // finish the translation which will delete the translation object
418 currState->transState->finish(
419 std::make_shared<UnimpFault>("Squashed Inst"),
420 currState->req, currState->tc, currState->mode);
421 } else {
422 // translate the request now that we know it will work
423 statWalkServiceTime.sample(curTick() - currState->startTime);
424 tlb->translateTiming(currState->req, currState->tc,
425 currState->transState, currState->mode);
426
427 }
428
429 // delete the current request
430 delete currState;
431
432 // peak at the next one
433 if (pendingQueue.size()) {
434 currState = pendingQueue.front();
435 te = tlb->lookup(currState->vaddr, currState->asid,
436 currState->vmid, currState->isHyp, currState->isSecure, true,
437 false, target_el);
438 } else {
439 // Terminate the loop, nothing more to do
440 currState = NULL;
441 }
442 }
443 pendingChange();
444
445 // if we still have pending translations, schedule more work
446 nextWalk(tc);
447 currState = NULL;
448}
449
450Fault
451TableWalker::processWalk()
452{
453 Addr ttbr = 0;
454
455 // If translation isn't enabled, we shouldn't be here
456 assert(currState->sctlr.m || isStage2);
457
458 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
459 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
460 32 - currState->ttbcr.n));
461
462 statWalkWaitTime.sample(curTick() - currState->startTime);
463
464 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
465 32 - currState->ttbcr.n)) {
466 DPRINTF(TLB, " - Selecting TTBR0\n");
467 // Check if table walk is allowed when Security Extensions are enabled
468 if (haveSecurity && currState->ttbcr.pd0) {
469 if (currState->isFetch)
470 return std::make_shared<PrefetchAbort>(
471 currState->vaddr_tainted,
472 ArmFault::TranslationLL + L1,
473 isStage2,
474 ArmFault::VmsaTran);
475 else
476 return std::make_shared<DataAbort>(
477 currState->vaddr_tainted,
478 TlbEntry::DomainType::NoAccess, currState->isWrite,
479 ArmFault::TranslationLL + L1, isStage2,
480 ArmFault::VmsaTran);
481 }
482 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
483 MISCREG_TTBR0, currState->tc, !currState->isSecure));
484 } else {
485 DPRINTF(TLB, " - Selecting TTBR1\n");
486 // Check if table walk is allowed when Security Extensions are enabled
487 if (haveSecurity && currState->ttbcr.pd1) {
488 if (currState->isFetch)
489 return std::make_shared<PrefetchAbort>(
490 currState->vaddr_tainted,
491 ArmFault::TranslationLL + L1,
492 isStage2,
493 ArmFault::VmsaTran);
494 else
495 return std::make_shared<DataAbort>(
496 currState->vaddr_tainted,
497 TlbEntry::DomainType::NoAccess, currState->isWrite,
498 ArmFault::TranslationLL + L1, isStage2,
499 ArmFault::VmsaTran);
500 }
501 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
502 MISCREG_TTBR1, currState->tc, !currState->isSecure));
503 currState->ttbcr.n = 0;
504 }
505
506 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
507 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
508 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
509 currState->isSecure ? "s" : "ns");
510
511 // Trickbox address check
512 Fault f;
513 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
514 currState->vaddr, sizeof(uint32_t), currState->isFetch,
515 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
516 if (f) {
517 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
518 if (currState->timing) {
519 pending = false;
520 nextWalk(currState->tc);
521 currState = NULL;
522 } else {
523 currState->tc = NULL;
524 currState->req = NULL;
525 }
526 return f;
527 }
528
529 Request::Flags flag = Request::PT_WALK;
530 if (currState->sctlr.c == 0) {
531 flag.set(Request::UNCACHEABLE);
532 }
533
534 bool delayed;
535 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
536 sizeof(uint32_t), flag, L1, &doL1DescEvent,
537 &TableWalker::doL1Descriptor);
538 if (!delayed) {
539 f = currState->fault;
540 }
541
542 return f;
543}
544
545Fault
546TableWalker::processWalkLPAE()
547{
548 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
549 int tsz, n;
550 LookupLevel start_lookup_level = L1;
551
552 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
553 currState->vaddr_tainted, currState->ttbcr);
554
555 statWalkWaitTime.sample(curTick() - currState->startTime);
556
557 Request::Flags flag = Request::PT_WALK;
558 if (currState->isSecure)
559 flag.set(Request::SECURE);
560
561 // work out which base address register to use, if in hyp mode we always
562 // use HTTBR
563 if (isStage2) {
564 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
565 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
566 tsz = sext<4>(currState->vtcr.t0sz);
567 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
568 } else if (currState->isHyp) {
569 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
570 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
571 tsz = currState->htcr.t0sz;
572 } else {
573 assert(_haveLPAE && currState->ttbcr.eae);
574
575 // Determine boundaries of TTBR0/1 regions
576 if (currState->ttbcr.t0sz)
577 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
578 else if (currState->ttbcr.t1sz)
579 ttbr0_max = (1ULL << 32) -
580 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
581 else
582 ttbr0_max = (1ULL << 32) - 1;
583 if (currState->ttbcr.t1sz)
584 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
585 else
586 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
587
588 // The following code snippet selects the appropriate translation table base
589 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
590 // depending on the address range supported by the translation table (ARM
591 // ARM issue C B3.6.4)
592 if (currState->vaddr <= ttbr0_max) {
593 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
594 // Check if table walk is allowed
595 if (currState->ttbcr.epd0) {
596 if (currState->isFetch)
597 return std::make_shared<PrefetchAbort>(
598 currState->vaddr_tainted,
599 ArmFault::TranslationLL + L1,
600 isStage2,
601 ArmFault::LpaeTran);
602 else
603 return std::make_shared<DataAbort>(
604 currState->vaddr_tainted,
605 TlbEntry::DomainType::NoAccess,
606 currState->isWrite,
607 ArmFault::TranslationLL + L1,
608 isStage2,
609 ArmFault::LpaeTran);
610 }
611 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
612 MISCREG_TTBR0, currState->tc, !currState->isSecure));
613 tsz = currState->ttbcr.t0sz;
614 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
615 start_lookup_level = L2;
616 } else if (currState->vaddr >= ttbr1_min) {
617 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
618 // Check if table walk is allowed
619 if (currState->ttbcr.epd1) {
620 if (currState->isFetch)
621 return std::make_shared<PrefetchAbort>(
622 currState->vaddr_tainted,
623 ArmFault::TranslationLL + L1,
624 isStage2,
625 ArmFault::LpaeTran);
626 else
627 return std::make_shared<DataAbort>(
628 currState->vaddr_tainted,
629 TlbEntry::DomainType::NoAccess,
630 currState->isWrite,
631 ArmFault::TranslationLL + L1,
632 isStage2,
633 ArmFault::LpaeTran);
634 }
635 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
636 MISCREG_TTBR1, currState->tc, !currState->isSecure));
637 tsz = currState->ttbcr.t1sz;
638 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
639 start_lookup_level = L2;
640 } else {
641 // Out of boundaries -> translation fault
642 if (currState->isFetch)
643 return std::make_shared<PrefetchAbort>(
644 currState->vaddr_tainted,
645 ArmFault::TranslationLL + L1,
646 isStage2,
647 ArmFault::LpaeTran);
648 else
649 return std::make_shared<DataAbort>(
650 currState->vaddr_tainted,
651 TlbEntry::DomainType::NoAccess,
652 currState->isWrite, ArmFault::TranslationLL + L1,
653 isStage2, ArmFault::LpaeTran);
654 }
655
656 }
657
658 // Perform lookup (ARM ARM issue C B3.6.6)
659 if (start_lookup_level == L1) {
660 n = 5 - tsz;
661 desc_addr = mbits(ttbr, 39, n) |
662 (bits(currState->vaddr, n + 26, 30) << 3);
663 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
664 desc_addr, currState->isSecure ? "s" : "ns");
665 } else {
666 // Skip first-level lookup
667 n = (tsz >= 2 ? 14 - tsz : 12);
668 desc_addr = mbits(ttbr, 39, n) |
669 (bits(currState->vaddr, n + 17, 21) << 3);
670 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
671 desc_addr, currState->isSecure ? "s" : "ns");
672 }
673
674 // Trickbox address check
675 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
676 currState->vaddr, sizeof(uint64_t), currState->isFetch,
677 currState->isWrite, TlbEntry::DomainType::NoAccess,
678 start_lookup_level);
679 if (f) {
680 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
681 if (currState->timing) {
682 pending = false;
683 nextWalk(currState->tc);
684 currState = NULL;
685 } else {
686 currState->tc = NULL;
687 currState->req = NULL;
688 }
689 return f;
690 }
691
692 if (currState->sctlr.c == 0) {
693 flag.set(Request::UNCACHEABLE);
694 }
695
696 if (currState->isSecure)
697 flag.set(Request::SECURE);
698
699 currState->longDesc.lookupLevel = start_lookup_level;
700 currState->longDesc.aarch64 = false;
701 currState->longDesc.grainSize = Grain4KB;
702
703 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
704 : (Event *) &doL2LongDescEvent;
705
706 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
707 sizeof(uint64_t), flag, start_lookup_level,
708 event, &TableWalker::doLongDescriptor);
709 if (!delayed) {
710 f = currState->fault;
711 }
712
713 return f;
714}
715
716unsigned
717TableWalker::adjustTableSizeAArch64(unsigned tsz)
718{
719 if (tsz < 25)
720 return 25;
721 if (tsz > 48)
722 return 48;
723 return tsz;
724}
725
726bool
727TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
728{
729 return (currPhysAddrRange != MaxPhysAddrRange &&
730 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
731}
732
733Fault
734TableWalker::processWalkAArch64()
735{
736 assert(currState->aarch64);
737
738 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
739 currState->vaddr_tainted, currState->tcr);
740
741 static const GrainSize GrainMapDefault[] =
742 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
743 static const GrainSize GrainMap_EL1_tg1[] =
744 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
745
746 statWalkWaitTime.sample(curTick() - currState->startTime);
747
748 // Determine TTBR, table size, granule size and phys. address range
749 Addr ttbr = 0;
750 int tsz = 0, ps = 0;
751 GrainSize tg = Grain4KB; // grain size computed from tg* field
752 bool fault = false;
753 switch (currState->el) {
754 case EL0:
755 case EL1:
756 switch (bits(currState->vaddr, 63,48)) {
757 case 0:
758 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
759 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
760 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
761 tg = GrainMapDefault[currState->tcr.tg0];
762 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
763 currState->tcr.epd0)
764 fault = true;
765 break;
766 case 0xffff:
767 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
768 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
769 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
770 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
771 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
772 currState->tcr.epd1)
773 fault = true;
774 break;
775 default:
776 // top two bytes must be all 0s or all 1s, else invalid addr
777 fault = true;
778 }
779 ps = currState->tcr.ips;
780 break;
781 case EL2:
782 case EL3:
783 switch(bits(currState->vaddr, 63,48)) {
784 case 0:
785 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
786 if (currState->el == EL2)
787 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
788 else
789 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
790 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
791 tg = GrainMapDefault[currState->tcr.tg0];
792 break;
793 default:
794 // invalid addr if top two bytes are not all 0s
795 fault = true;
796 }
797 ps = currState->tcr.ips;
798 break;
799 }
800
801 if (fault) {
802 Fault f;
803 if (currState->isFetch)
804 f = std::make_shared<PrefetchAbort>(
805 currState->vaddr_tainted,
806 ArmFault::TranslationLL + L0, isStage2,
807 ArmFault::LpaeTran);
808 else
809 f = std::make_shared<DataAbort>(
810 currState->vaddr_tainted,
811 TlbEntry::DomainType::NoAccess,
812 currState->isWrite,
813 ArmFault::TranslationLL + L0,
814 isStage2, ArmFault::LpaeTran);
815
816 if (currState->timing) {
817 pending = false;
818 nextWalk(currState->tc);
819 currState = NULL;
820 } else {
821 currState->tc = NULL;
822 currState->req = NULL;
823 }
824 return f;
825
826 }
827
828 if (tg == ReservedGrain) {
829 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
830 "DEFINED behavior takes this to mean 4KB granules\n");
831 tg = Grain4KB;
832 }
833
834 int stride = tg - 3;
835 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
836
837 // Determine starting lookup level
838 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
839 // in ARM DDI 0487A. These table values correspond to the cascading tests
840 // to compute the lookup level and are of the form
841 // (grain_size + N*stride), for N = {1, 2, 3}.
842 // A value of 64 will never succeed and a value of 0 will always succeed.
843 {
844 struct GrainMap {
845 GrainSize grain_size;
846 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
847 };
848 static const GrainMap GM[] = {
849 { Grain4KB, { 39, 30, 0, 0 } },
850 { Grain16KB, { 47, 36, 25, 0 } },
851 { Grain64KB, { 64, 42, 29, 0 } }
852 };
853
854 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
855
856 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
857 if (tg == GM[i].grain_size) {
858 lookup = GM[i].lookup_level_cutoff;
859 break;
860 }
861 }
862 assert(lookup);
863
864 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
865 if (tsz > lookup[L]) {
866 start_lookup_level = (LookupLevel) L;
867 break;
868 }
869 }
870 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
871 "Table walker couldn't find lookup level\n");
872 }
873
874 // Determine table base address
875 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
876 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
877
878 // Determine physical address size and raise an Address Size Fault if
879 // necessary
880 int pa_range = decodePhysAddrRange64(ps);
881 // Clamp to lower limit
882 if (pa_range > physAddrRange)
883 currState->physAddrRange = physAddrRange;
884 else
885 currState->physAddrRange = pa_range;
886 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
887 DPRINTF(TLB, "Address size fault before any lookup\n");
888 Fault f;
889 if (currState->isFetch)
890 f = std::make_shared<PrefetchAbort>(
891 currState->vaddr_tainted,
892 ArmFault::AddressSizeLL + start_lookup_level,
893 isStage2,
894 ArmFault::LpaeTran);
895 else
896 f = std::make_shared<DataAbort>(
897 currState->vaddr_tainted,
898 TlbEntry::DomainType::NoAccess,
899 currState->isWrite,
900 ArmFault::AddressSizeLL + start_lookup_level,
901 isStage2,
902 ArmFault::LpaeTran);
903
904
905 if (currState->timing) {
906 pending = false;
907 nextWalk(currState->tc);
908 currState = NULL;
909 } else {
910 currState->tc = NULL;
911 currState->req = NULL;
912 }
913 return f;
914
915 }
916
917 // Determine descriptor address
918 Addr desc_addr = base_addr |
919 (bits(currState->vaddr, tsz - 1,
920 stride * (3 - start_lookup_level) + tg) << 3);
921
922 // Trickbox address check
923 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
924 currState->vaddr, sizeof(uint64_t), currState->isFetch,
925 currState->isWrite, TlbEntry::DomainType::NoAccess,
926 start_lookup_level);
927 if (f) {
928 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
929 if (currState->timing) {
930 pending = false;
931 nextWalk(currState->tc);
932 currState = NULL;
933 } else {
934 currState->tc = NULL;
935 currState->req = NULL;
936 }
937 return f;
938 }
939
940 Request::Flags flag = Request::PT_WALK;
941 if (currState->sctlr.c == 0) {
942 flag.set(Request::UNCACHEABLE);
943 }
944
945 currState->longDesc.lookupLevel = start_lookup_level;
946 currState->longDesc.aarch64 = true;
947 currState->longDesc.grainSize = tg;
948
949 if (currState->timing) {
950 Event *event;
951 switch (start_lookup_level) {
952 case L0:
953 event = (Event *) &doL0LongDescEvent;
954 break;
955 case L1:
956 event = (Event *) &doL1LongDescEvent;
957 break;
958 case L2:
959 event = (Event *) &doL2LongDescEvent;
960 break;
961 case L3:
962 event = (Event *) &doL3LongDescEvent;
963 break;
964 default:
965 panic("Invalid table lookup level");
966 break;
967 }
968 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
969 event, (uint8_t*) &currState->longDesc.data,
970 currState->tc->getCpuPtr()->clockPeriod(), flag);
971 DPRINTF(TLBVerbose,
972 "Adding to walker fifo: queue size before adding: %d\n",
973 stateQueues[start_lookup_level].size());
974 stateQueues[start_lookup_level].push_back(currState);
975 currState = NULL;
976 } else if (!currState->functional) {
977 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
978 NULL, (uint8_t*) &currState->longDesc.data,
979 currState->tc->getCpuPtr()->clockPeriod(), flag);
980 doLongDescriptor();
981 f = currState->fault;
982 } else {
983 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
984 masterId);
985 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
986 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
987 port->sendFunctional(pkt);
988 doLongDescriptor();
989 delete req;
990 delete pkt;
991 f = currState->fault;
992 }
993
994 return f;
995}
996
997void
998TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
999 uint8_t texcb, bool s)
1000{
1001 // Note: tc and sctlr local variables are hiding tc and sctrl class
1002 // variables
1003 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1004 te.shareable = false; // default value
1005 te.nonCacheable = false;
1006 te.outerShareable = false;
1007 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1008 switch(texcb) {
1009 case 0: // Stongly-ordered
1010 te.nonCacheable = true;
1011 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1012 te.shareable = true;
1013 te.innerAttrs = 1;
1014 te.outerAttrs = 0;
1015 break;
1016 case 1: // Shareable Device
1017 te.nonCacheable = true;
1018 te.mtype = TlbEntry::MemoryType::Device;
1019 te.shareable = true;
1020 te.innerAttrs = 3;
1021 te.outerAttrs = 0;
1022 break;
1023 case 2: // Outer and Inner Write-Through, no Write-Allocate
1024 te.mtype = TlbEntry::MemoryType::Normal;
1025 te.shareable = s;
1026 te.innerAttrs = 6;
1027 te.outerAttrs = bits(texcb, 1, 0);
1028 break;
1029 case 3: // Outer and Inner Write-Back, no Write-Allocate
1030 te.mtype = TlbEntry::MemoryType::Normal;
1031 te.shareable = s;
1032 te.innerAttrs = 7;
1033 te.outerAttrs = bits(texcb, 1, 0);
1034 break;
1035 case 4: // Outer and Inner Non-cacheable
1036 te.nonCacheable = true;
1037 te.mtype = TlbEntry::MemoryType::Normal;
1038 te.shareable = s;
1039 te.innerAttrs = 0;
1040 te.outerAttrs = bits(texcb, 1, 0);
1041 break;
1042 case 5: // Reserved
1043 panic("Reserved texcb value!\n");
1044 break;
1045 case 6: // Implementation Defined
1046 panic("Implementation-defined texcb value!\n");
1047 break;
1048 case 7: // Outer and Inner Write-Back, Write-Allocate
1049 te.mtype = TlbEntry::MemoryType::Normal;
1050 te.shareable = s;
1051 te.innerAttrs = 5;
1052 te.outerAttrs = 1;
1053 break;
1054 case 8: // Non-shareable Device
1055 te.nonCacheable = true;
1056 te.mtype = TlbEntry::MemoryType::Device;
1057 te.shareable = false;
1058 te.innerAttrs = 3;
1059 te.outerAttrs = 0;
1060 break;
1061 case 9 ... 15: // Reserved
1062 panic("Reserved texcb value!\n");
1063 break;
1064 case 16 ... 31: // Cacheable Memory
1065 te.mtype = TlbEntry::MemoryType::Normal;
1066 te.shareable = s;
1067 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1068 te.nonCacheable = true;
1069 te.innerAttrs = bits(texcb, 1, 0);
1070 te.outerAttrs = bits(texcb, 3, 2);
1071 break;
1072 default:
1073 panic("More than 32 states for 5 bits?\n");
1074 }
1075 } else {
1076 assert(tc);
1077 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1078 currState->tc, !currState->isSecure));
1079 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1080 currState->tc, !currState->isSecure));
1081 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1082 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1083 switch(bits(texcb, 2,0)) {
1084 case 0:
1085 curr_tr = prrr.tr0;
1086 curr_ir = nmrr.ir0;
1087 curr_or = nmrr.or0;
1088 te.outerShareable = (prrr.nos0 == 0);
1089 break;
1090 case 1:
1091 curr_tr = prrr.tr1;
1092 curr_ir = nmrr.ir1;
1093 curr_or = nmrr.or1;
1094 te.outerShareable = (prrr.nos1 == 0);
1095 break;
1096 case 2:
1097 curr_tr = prrr.tr2;
1098 curr_ir = nmrr.ir2;
1099 curr_or = nmrr.or2;
1100 te.outerShareable = (prrr.nos2 == 0);
1101 break;
1102 case 3:
1103 curr_tr = prrr.tr3;
1104 curr_ir = nmrr.ir3;
1105 curr_or = nmrr.or3;
1106 te.outerShareable = (prrr.nos3 == 0);
1107 break;
1108 case 4:
1109 curr_tr = prrr.tr4;
1110 curr_ir = nmrr.ir4;
1111 curr_or = nmrr.or4;
1112 te.outerShareable = (prrr.nos4 == 0);
1113 break;
1114 case 5:
1115 curr_tr = prrr.tr5;
1116 curr_ir = nmrr.ir5;
1117 curr_or = nmrr.or5;
1118 te.outerShareable = (prrr.nos5 == 0);
1119 break;
1120 case 6:
1121 panic("Imp defined type\n");
1122 case 7:
1123 curr_tr = prrr.tr7;
1124 curr_ir = nmrr.ir7;
1125 curr_or = nmrr.or7;
1126 te.outerShareable = (prrr.nos7 == 0);
1127 break;
1128 }
1129
1130 switch(curr_tr) {
1131 case 0:
1132 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1133 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1134 te.nonCacheable = true;
1135 te.innerAttrs = 1;
1136 te.outerAttrs = 0;
1137 te.shareable = true;
1138 break;
1139 case 1:
1140 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1141 prrr.ds1, prrr.ds0, s);
1142 te.mtype = TlbEntry::MemoryType::Device;
1143 te.nonCacheable = true;
1144 te.innerAttrs = 3;
1145 te.outerAttrs = 0;
1146 if (prrr.ds1 && s)
1147 te.shareable = true;
1148 if (prrr.ds0 && !s)
1149 te.shareable = true;
1150 break;
1151 case 2:
1152 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1153 prrr.ns1, prrr.ns0, s);
1154 te.mtype = TlbEntry::MemoryType::Normal;
1155 if (prrr.ns1 && s)
1156 te.shareable = true;
1157 if (prrr.ns0 && !s)
1158 te.shareable = true;
1159 break;
1160 case 3:
1161 panic("Reserved type");
1162 }
1163
1164 if (te.mtype == TlbEntry::MemoryType::Normal){
1165 switch(curr_ir) {
1166 case 0:
1167 te.nonCacheable = true;
1168 te.innerAttrs = 0;
1169 break;
1170 case 1:
1171 te.innerAttrs = 5;
1172 break;
1173 case 2:
1174 te.innerAttrs = 6;
1175 break;
1176 case 3:
1177 te.innerAttrs = 7;
1178 break;
1179 }
1180
1181 switch(curr_or) {
1182 case 0:
1183 te.nonCacheable = true;
1184 te.outerAttrs = 0;
1185 break;
1186 case 1:
1187 te.outerAttrs = 1;
1188 break;
1189 case 2:
1190 te.outerAttrs = 2;
1191 break;
1192 case 3:
1193 te.outerAttrs = 3;
1194 break;
1195 }
1196 }
1197 }
1198 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1199 "outerAttrs: %d\n",
1200 te.shareable, te.innerAttrs, te.outerAttrs);
1201 te.setAttributes(false);
1202}
1203
1204void
1205TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1206 LongDescriptor &lDescriptor)
1207{
1208 assert(_haveLPAE);
1209
1210 uint8_t attr;
1211 uint8_t sh = lDescriptor.sh();
1212 // Different format and source of attributes if this is a stage 2
1213 // translation
1214 if (isStage2) {
1215 attr = lDescriptor.memAttr();
1216 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1217 uint8_t attr_1_0 = attr & 0x3;
1218
1219 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1220
1221 if (attr_3_2 == 0) {
1222 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1223 : TlbEntry::MemoryType::Device;
1224 te.outerAttrs = 0;
1225 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1226 te.nonCacheable = true;
1227 } else {
1228 te.mtype = TlbEntry::MemoryType::Normal;
1229 te.outerAttrs = attr_3_2 == 1 ? 0 :
1230 attr_3_2 == 2 ? 2 : 1;
1231 te.innerAttrs = attr_1_0 == 1 ? 0 :
1232 attr_1_0 == 2 ? 6 : 5;
1233 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1234 }
1235 } else {
1236 uint8_t attrIndx = lDescriptor.attrIndx();
1237
1238 // LPAE always uses remapping of memory attributes, irrespective of the
1239 // value of SCTLR.TRE
1240 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1241 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1242 !currState->isSecure);
1243 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1244 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1245 uint8_t attr_7_4 = bits(attr, 7, 4);
1246 uint8_t attr_3_0 = bits(attr, 3, 0);
1247 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1248
1249 // Note: the memory subsystem only cares about the 'cacheable' memory
1250 // attribute. The other attributes are only used to fill the PAR register
1251 // accordingly to provide the illusion of full support
1252 te.nonCacheable = false;
1253
1254 switch (attr_7_4) {
1255 case 0x0:
1256 // Strongly-ordered or Device memory
1257 if (attr_3_0 == 0x0)
1258 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1259 else if (attr_3_0 == 0x4)
1260 te.mtype = TlbEntry::MemoryType::Device;
1261 else
1262 panic("Unpredictable behavior\n");
1263 te.nonCacheable = true;
1264 te.outerAttrs = 0;
1265 break;
1266 case 0x4:
1267 // Normal memory, Outer Non-cacheable
1268 te.mtype = TlbEntry::MemoryType::Normal;
1269 te.outerAttrs = 0;
1270 if (attr_3_0 == 0x4)
1271 // Inner Non-cacheable
1272 te.nonCacheable = true;
1273 else if (attr_3_0 < 0x8)
1274 panic("Unpredictable behavior\n");
1275 break;
1276 case 0x8:
1277 case 0x9:
1278 case 0xa:
1279 case 0xb:
1280 case 0xc:
1281 case 0xd:
1282 case 0xe:
1283 case 0xf:
1284 if (attr_7_4 & 0x4) {
1285 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1286 } else {
1287 te.outerAttrs = 0x2;
1288 }
1289 // Normal memory, Outer Cacheable
1290 te.mtype = TlbEntry::MemoryType::Normal;
1291 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1292 panic("Unpredictable behavior\n");
1293 break;
1294 default:
1295 panic("Unpredictable behavior\n");
1296 break;
1297 }
1298
1299 switch (attr_3_0) {
1300 case 0x0:
1301 te.innerAttrs = 0x1;
1302 break;
1303 case 0x4:
1304 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1305 break;
1306 case 0x8:
1307 case 0x9:
1308 case 0xA:
1309 case 0xB:
1310 te.innerAttrs = 6;
1311 break;
1312 case 0xC:
1313 case 0xD:
1314 case 0xE:
1315 case 0xF:
1316 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1317 break;
1318 default:
1319 panic("Unpredictable behavior\n");
1320 break;
1321 }
1322 }
1323
1324 te.outerShareable = sh == 2;
1325 te.shareable = (sh & 0x2) ? true : false;
1326 te.setAttributes(true);
1327 te.attributes |= (uint64_t) attr << 56;
1328}
1329
1330void
1331TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1332 uint8_t sh)
1333{
1334 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1335
1336 // Select MAIR
1337 uint64_t mair;
1338 switch (currState->el) {
1339 case EL0:
1340 case EL1:
1341 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1342 break;
1343 case EL2:
1344 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1345 break;
1346 case EL3:
1347 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1348 break;
1349 default:
1350 panic("Invalid exception level");
1351 break;
1352 }
1353
1354 // Select attributes
1355 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1356 uint8_t attr_lo = bits(attr, 3, 0);
1357 uint8_t attr_hi = bits(attr, 7, 4);
1358
1359 // Memory type
1360 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1361
1362 // Cacheability
1363 te.nonCacheable = false;
1364 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1365 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1366 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1367 te.nonCacheable = true;
1368 }
1369
1370 te.shareable = sh == 2;
1371 te.outerShareable = (sh & 0x2) ? true : false;
1372 // Attributes formatted according to the 64-bit PAR
1373 te.attributes = ((uint64_t) attr << 56) |
1374 (1 << 11) | // LPAE bit
1375 (te.ns << 9) | // NS bit
1376 (sh << 7);
1377}
1378
1379void
1380TableWalker::doL1Descriptor()
1381{
1382 if (currState->fault != NoFault) {
1383 return;
1384 }
1385
1386 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1387 currState->vaddr_tainted, currState->l1Desc.data);
1388 TlbEntry te;
1389
1390 switch (currState->l1Desc.type()) {
1391 case L1Descriptor::Ignore:
1392 case L1Descriptor::Reserved:
1393 if (!currState->timing) {
1394 currState->tc = NULL;
1395 currState->req = NULL;
1396 }
1397 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1398 if (currState->isFetch)
1399 currState->fault =
1400 std::make_shared<PrefetchAbort>(
1401 currState->vaddr_tainted,
1402 ArmFault::TranslationLL + L1,
1403 isStage2,
1404 ArmFault::VmsaTran);
1405 else
1406 currState->fault =
1407 std::make_shared<DataAbort>(
1408 currState->vaddr_tainted,
1409 TlbEntry::DomainType::NoAccess,
1410 currState->isWrite,
1411 ArmFault::TranslationLL + L1, isStage2,
1412 ArmFault::VmsaTran);
1413 return;
1414 case L1Descriptor::Section:
1415 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1416 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1417 * enabled if set, do l1.Desc.setAp0() instead of generating
1418 * AccessFlag0
1419 */
1420
1421 currState->fault = std::make_shared<DataAbort>(
1422 currState->vaddr_tainted,
1423 currState->l1Desc.domain(),
1424 currState->isWrite,
1425 ArmFault::AccessFlagLL + L1,
1426 isStage2,
1427 ArmFault::VmsaTran);
1428 }
1429 if (currState->l1Desc.supersection()) {
1430 panic("Haven't implemented supersections\n");
1431 }
1432 insertTableEntry(currState->l1Desc, false);
1433 return;
1434 case L1Descriptor::PageTable:
1435 {
1436 Addr l2desc_addr;
1437 l2desc_addr = currState->l1Desc.l2Addr() |
1438 (bits(currState->vaddr, 19, 12) << 2);
1439 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1440 l2desc_addr, currState->isSecure ? "s" : "ns");
1441
1442 // Trickbox address check
1443 currState->fault = tlb->walkTrickBoxCheck(
1444 l2desc_addr, currState->isSecure, currState->vaddr,
1445 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1446 currState->l1Desc.domain(), L2);
1447
1448 if (currState->fault) {
1449 if (!currState->timing) {
1450 currState->tc = NULL;
1451 currState->req = NULL;
1452 }
1453 return;
1454 }
1455
1456 Request::Flags flag = Request::PT_WALK;
1457 if (currState->isSecure)
1458 flag.set(Request::SECURE);
1459
1460 bool delayed;
1461 delayed = fetchDescriptor(l2desc_addr,
1462 (uint8_t*)&currState->l2Desc.data,
1463 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1464 &TableWalker::doL2Descriptor);
1465 if (delayed) {
1466 currState->delayed = true;
1467 }
1468
1469 return;
1470 }
1471 default:
1472 panic("A new type in a 2 bit field?\n");
1473 }
1474}
1475
1476void
1477TableWalker::doLongDescriptor()
1478{
1479 if (currState->fault != NoFault) {
1480 return;
1481 }
1482
1483 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1484 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1485 currState->longDesc.data,
1486 currState->aarch64 ? "AArch64" : "long-desc.");
1487
1488 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1489 (currState->longDesc.type() == LongDescriptor::Page)) {
1490 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1491 "xn: %d, ap: %d, af: %d, type: %d\n",
1492 currState->longDesc.lookupLevel,
1493 currState->longDesc.data,
1494 currState->longDesc.pxn(),
1495 currState->longDesc.xn(),
1496 currState->longDesc.ap(),
1497 currState->longDesc.af(),
1498 currState->longDesc.type());
1499 } else {
1500 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1501 currState->longDesc.lookupLevel,
1502 currState->longDesc.data,
1503 currState->longDesc.type());
1504 }
1505
1506 TlbEntry te;
1507
1508 switch (currState->longDesc.type()) {
1509 case LongDescriptor::Invalid:
1510 if (!currState->timing) {
1511 currState->tc = NULL;
1512 currState->req = NULL;
1513 }
1514
1515 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1516 currState->longDesc.lookupLevel,
1517 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1518 if (currState->isFetch)
1519 currState->fault = std::make_shared<PrefetchAbort>(
1520 currState->vaddr_tainted,
1521 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1522 isStage2,
1523 ArmFault::LpaeTran);
1524 else
1525 currState->fault = std::make_shared<DataAbort>(
1526 currState->vaddr_tainted,
1527 TlbEntry::DomainType::NoAccess,
1528 currState->isWrite,
1529 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1530 isStage2,
1531 ArmFault::LpaeTran);
1532 return;
1533 case LongDescriptor::Block:
1534 case LongDescriptor::Page:
1535 {
1536 bool fault = false;
1537 bool aff = false;
1538 // Check for address size fault
1539 if (checkAddrSizeFaultAArch64(
1540 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1541 currState->longDesc.offsetBits()),
1542 currState->physAddrRange)) {
1543 fault = true;
1544 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1545 currState->longDesc.lookupLevel);
1546 // Check for access fault
1547 } else if (currState->longDesc.af() == 0) {
1548 fault = true;
1549 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1550 currState->longDesc.lookupLevel);
1551 aff = true;
1552 }
1553 if (fault) {
1554 if (currState->isFetch)
1555 currState->fault = std::make_shared<PrefetchAbort>(
1556 currState->vaddr_tainted,
1557 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1558 currState->longDesc.lookupLevel,
1559 isStage2,
1560 ArmFault::LpaeTran);
1561 else
1562 currState->fault = std::make_shared<DataAbort>(
1563 currState->vaddr_tainted,
1564 TlbEntry::DomainType::NoAccess, currState->isWrite,
1565 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1566 currState->longDesc.lookupLevel,
1567 isStage2,
1568 ArmFault::LpaeTran);
1569 } else {
1570 insertTableEntry(currState->longDesc, true);
1571 }
1572 }
1573 return;
1574 case LongDescriptor::Table:
1575 {
1576 // Set hierarchical permission flags
1577 currState->secureLookup = currState->secureLookup &&
1578 currState->longDesc.secureTable();
1579 currState->rwTable = currState->rwTable &&
1580 currState->longDesc.rwTable();
1581 currState->userTable = currState->userTable &&
1582 currState->longDesc.userTable();
1583 currState->xnTable = currState->xnTable ||
1584 currState->longDesc.xnTable();
1585 currState->pxnTable = currState->pxnTable ||
1586 currState->longDesc.pxnTable();
1587
1588 // Set up next level lookup
1589 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1590 currState->vaddr);
1591
1592 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1593 currState->longDesc.lookupLevel,
1594 currState->longDesc.lookupLevel + 1,
1595 next_desc_addr,
1596 currState->secureLookup ? "s" : "ns");
1597
1598 // Check for address size fault
1599 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1600 next_desc_addr, currState->physAddrRange)) {
1601 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1602 currState->longDesc.lookupLevel);
1603 if (currState->isFetch)
1604 currState->fault = std::make_shared<PrefetchAbort>(
1605 currState->vaddr_tainted,
1606 ArmFault::AddressSizeLL
1607 + currState->longDesc.lookupLevel,
1608 isStage2,
1609 ArmFault::LpaeTran);
1610 else
1611 currState->fault = std::make_shared<DataAbort>(
1612 currState->vaddr_tainted,
1613 TlbEntry::DomainType::NoAccess, currState->isWrite,
1614 ArmFault::AddressSizeLL
1615 + currState->longDesc.lookupLevel,
1616 isStage2,
1617 ArmFault::LpaeTran);
1618 return;
1619 }
1620
1621 // Trickbox address check
1622 currState->fault = tlb->walkTrickBoxCheck(
1623 next_desc_addr, currState->vaddr,
1624 currState->vaddr, sizeof(uint64_t),
1625 currState->isFetch, currState->isWrite,
1626 TlbEntry::DomainType::Client,
1627 toLookupLevel(currState->longDesc.lookupLevel +1));
1628
1629 if (currState->fault) {
1630 if (!currState->timing) {
1631 currState->tc = NULL;
1632 currState->req = NULL;
1633 }
1634 return;
1635 }
1636
1637 Request::Flags flag = Request::PT_WALK;
1638 if (currState->secureLookup)
1639 flag.set(Request::SECURE);
1640
1641 currState->longDesc.lookupLevel =
1642 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1643 Event *event = NULL;
1644 switch (currState->longDesc.lookupLevel) {
1645 case L1:
1646 assert(currState->aarch64);
1647 event = &doL1LongDescEvent;
1648 break;
1649 case L2:
1650 event = &doL2LongDescEvent;
1651 break;
1652 case L3:
1653 event = &doL3LongDescEvent;
1654 break;
1655 default:
1656 panic("Wrong lookup level in table walk\n");
1657 break;
1658 }
1659
1660 bool delayed;
1661 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1662 sizeof(uint64_t), flag, -1, event,
1663 &TableWalker::doLongDescriptor);
1664 if (delayed) {
1665 currState->delayed = true;
1666 }
1667 }
1668 return;
1669 default:
1670 panic("A new type in a 2 bit field?\n");
1671 }
1672}
1673
1674void
1675TableWalker::doL2Descriptor()
1676{
1677 if (currState->fault != NoFault) {
1678 return;
1679 }
1680
1681 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1682 currState->vaddr_tainted, currState->l2Desc.data);
1683 TlbEntry te;
1684
1685 if (currState->l2Desc.invalid()) {
1686 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1687 if (!currState->timing) {
1688 currState->tc = NULL;
1689 currState->req = NULL;
1690 }
1691 if (currState->isFetch)
1692 currState->fault = std::make_shared<PrefetchAbort>(
1693 currState->vaddr_tainted,
1694 ArmFault::TranslationLL + L2,
1695 isStage2,
1696 ArmFault::VmsaTran);
1697 else
1698 currState->fault = std::make_shared<DataAbort>(
1699 currState->vaddr_tainted, currState->l1Desc.domain(),
1700 currState->isWrite, ArmFault::TranslationLL + L2,
1701 isStage2,
1702 ArmFault::VmsaTran);
1703 return;
1704 }
1705
1706 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1707 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1708 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1709 */
1710 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1711 currState->sctlr.afe, currState->l2Desc.ap());
1712
1713 currState->fault = std::make_shared<DataAbort>(
1714 currState->vaddr_tainted,
1715 TlbEntry::DomainType::NoAccess, currState->isWrite,
1716 ArmFault::AccessFlagLL + L2, isStage2,
1717 ArmFault::VmsaTran);
1718 }
1719
1720 insertTableEntry(currState->l2Desc, false);
1721}
1722
1723void
1724TableWalker::doL1DescriptorWrapper()
1725{
1726 currState = stateQueues[L1].front();
1727 currState->delayed = false;
1728 // if there's a stage2 translation object we don't need it any more
1729 if (currState->stage2Tran) {
1730 delete currState->stage2Tran;
1731 currState->stage2Tran = NULL;
1732 }
1733
1734
1735 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1736 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1737
1738 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1739 doL1Descriptor();
1740
1741 stateQueues[L1].pop_front();
1742 // Check if fault was generated
1743 if (currState->fault != NoFault) {
1744 currState->transState->finish(currState->fault, currState->req,
1745 currState->tc, currState->mode);
1746 statWalksShortTerminatedAtLevel[0]++;
1747
1748 pending = false;
1749 nextWalk(currState->tc);
1750
1751 currState->req = NULL;
1752 currState->tc = NULL;
1753 currState->delayed = false;
1754 delete currState;
1755 }
1756 else if (!currState->delayed) {
1757 // delay is not set so there is no L2 to do
1758 // Don't finish the translation if a stage 2 look up is underway
1759 if (!currState->doingStage2) {
1760 statWalkServiceTime.sample(curTick() - currState->startTime);
1761 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1762 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1763 currState->transState, currState->mode);
1764 statWalksShortTerminatedAtLevel[0]++;
1765 }
1766
1767 pending = false;
1768 nextWalk(currState->tc);
1769
1770 currState->req = NULL;
1771 currState->tc = NULL;
1772 currState->delayed = false;
1773 delete currState;
1774 } else {
1775 // need to do L2 descriptor
1776 stateQueues[L2].push_back(currState);
1777 }
1778 currState = NULL;
1779}
1780
1781void
1782TableWalker::doL2DescriptorWrapper()
1783{
1784 currState = stateQueues[L2].front();
1785 assert(currState->delayed);
1786 // if there's a stage2 translation object we don't need it any more
1787 if (currState->stage2Tran) {
1788 delete currState->stage2Tran;
1789 currState->stage2Tran = NULL;
1790 }
1791
1792 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1793 currState->vaddr_tainted);
1794 doL2Descriptor();
1795
1796 // Check if fault was generated
1797 if (currState->fault != NoFault) {
1798 currState->transState->finish(currState->fault, currState->req,
1799 currState->tc, currState->mode);
1800 statWalksShortTerminatedAtLevel[1]++;
1801 }
1802 else {
1803 // Don't finish the translation if a stage 2 look up is underway
1804 if (!currState->doingStage2) {
1805 statWalkServiceTime.sample(curTick() - currState->startTime);
1806 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1807 currState->fault = tlb->translateTiming(currState->req,
1808 currState->tc, currState->transState, currState->mode);
1809 statWalksShortTerminatedAtLevel[1]++;
1810 }
1811 }
1812
1813
1814 stateQueues[L2].pop_front();
1815 pending = false;
1816 nextWalk(currState->tc);
1817
1818 currState->req = NULL;
1819 currState->tc = NULL;
1820 currState->delayed = false;
1821
1822 delete currState;
1823 currState = NULL;
1824}
1825
1826void
1827TableWalker::doL0LongDescriptorWrapper()
1828{
1829 doLongDescriptorWrapper(L0);
1830}
1831
1832void
1833TableWalker::doL1LongDescriptorWrapper()
1834{
1835 doLongDescriptorWrapper(L1);
1836}
1837
1838void
1839TableWalker::doL2LongDescriptorWrapper()
1840{
1841 doLongDescriptorWrapper(L2);
1842}
1843
1844void
1845TableWalker::doL3LongDescriptorWrapper()
1846{
1847 doLongDescriptorWrapper(L3);
1848}
1849
1850void
1851TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1852{
1853 currState = stateQueues[curr_lookup_level].front();
1854 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1855 currState->delayed = false;
1856
1857 // if there's a stage2 translation object we don't need it any more
1858 if (currState->stage2Tran) {
1859 delete currState->stage2Tran;
1860 currState->stage2Tran = NULL;
1861 }
1862
1863 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1864 currState->vaddr_tainted);
1865 doLongDescriptor();
1866
1867 stateQueues[curr_lookup_level].pop_front();
1868
1869 if (currState->fault != NoFault) {
1870 // A fault was generated
1871 currState->transState->finish(currState->fault, currState->req,
1872 currState->tc, currState->mode);
1873
1874 pending = false;
1875 nextWalk(currState->tc);
1876
1877 currState->req = NULL;
1878 currState->tc = NULL;
1879 currState->delayed = false;
1880 delete currState;
1881 } else if (!currState->delayed) {
1882 // No additional lookups required
1883 // Don't finish the translation if a stage 2 look up is underway
1884 if (!currState->doingStage2) {
1885 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1886 statWalkServiceTime.sample(curTick() - currState->startTime);
1887 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1888 currState->transState,
1889 currState->mode);
1890 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1891 }
1892
1893 pending = false;
1894 nextWalk(currState->tc);
1895
1896 currState->req = NULL;
1897 currState->tc = NULL;
1898 currState->delayed = false;
1899 delete currState;
1900 } else {
1901 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1902 panic("Max. number of lookups already reached in table walk\n");
1903 // Need to perform additional lookups
1904 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1905 }
1906 currState = NULL;
1907}
1908
1909
1910void
1911TableWalker::nextWalk(ThreadContext *tc)
1912{
1913 if (pendingQueue.size())
1914 schedule(doProcessEvent, clockEdge(Cycles(1)));
1915 else
1916 completeDrain();
1917}
1918
1919bool
1920TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1921 Request::Flags flags, int queueIndex, Event *event,
1922 void (TableWalker::*doDescriptor)())
1923{
1924 bool isTiming = currState->timing;
1925
1926 // do the requests for the page table descriptors have to go through the
1927 // second stage MMU
1928 if (currState->stage2Req) {
1929 Fault fault;
1930 flags = flags | TLB::MustBeOne;
1931
1932 if (isTiming) {
1933 Stage2MMU::Stage2Translation *tran = new
1934 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1935 currState->vaddr);
1936 currState->stage2Tran = tran;
1937 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1938 flags);
1939 fault = tran->fault;
1940 } else {
1941 fault = stage2Mmu->readDataUntimed(currState->tc,
1942 currState->vaddr, descAddr, data, numBytes, flags,
1943 currState->functional);
1944 }
1945
1946 if (fault != NoFault) {
1947 currState->fault = fault;
1948 }
1949 if (isTiming) {
1950 if (queueIndex >= 0) {
1951 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1952 stateQueues[queueIndex].size());
1953 stateQueues[queueIndex].push_back(currState);
1954 currState = NULL;
1955 }
1956 } else {
1957 (this->*doDescriptor)();
1958 }
1959 } else {
1960 if (isTiming) {
1961 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1962 currState->tc->getCpuPtr()->clockPeriod(),flags);
1963 if (queueIndex >= 0) {
1964 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1965 stateQueues[queueIndex].size());
1966 stateQueues[queueIndex].push_back(currState);
1967 currState = NULL;
1968 }
1969 } else if (!currState->functional) {
1970 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1971 currState->tc->getCpuPtr()->clockPeriod(), flags);
1972 (this->*doDescriptor)();
1973 } else {
1974 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1975 req->taskId(ContextSwitchTaskId::DMA);
1976 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1977 pkt->dataStatic(data);
1978 port->sendFunctional(pkt);
1979 (this->*doDescriptor)();
1980 delete req;
1981 delete pkt;
1982 }
1983 }
1984 return (isTiming);
1985}
1986
1987void
1988TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1989{
1990 TlbEntry te;
1991
1992 // Create and fill a new page table entry
1993 te.valid = true;
1994 te.longDescFormat = longDescriptor;
1995 te.isHyp = currState->isHyp;
1996 te.asid = currState->asid;
1997 te.vmid = currState->vmid;
1998 te.N = descriptor.offsetBits();
1999 te.vpn = currState->vaddr >> te.N;
2000 te.size = (1<<te.N) - 1;
2001 te.pfn = descriptor.pfn();
2002 te.domain = descriptor.domain();
2003 te.lookupLevel = descriptor.lookupLevel;
2004 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
2005 te.nstid = !currState->isSecure;
2006 te.xn = descriptor.xn();
2007 if (currState->aarch64)
2008 te.el = currState->el;
2009 else
2010 te.el = 1;
2011
2012 statPageSizes[pageSizeNtoStatBin(te.N)]++;
2013 statRequestOrigin[COMPLETED][currState->isFetch]++;
2014
2015 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2016 // as global
2017 te.global = descriptor.global(currState) || isStage2;
2018 if (longDescriptor) {
2019 LongDescriptor lDescriptor =
2020 dynamic_cast<LongDescriptor &>(descriptor);
2021
2022 te.xn |= currState->xnTable;
2023 te.pxn = currState->pxnTable || lDescriptor.pxn();
2024 if (isStage2) {
2025 // this is actually the HAP field, but its stored in the same bit
2026 // possitions as the AP field in a stage 1 translation.
2027 te.hap = lDescriptor.ap();
2028 } else {
2029 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2030 (currState->userTable && (descriptor.ap() & 0x1));
2031 }
2032 if (currState->aarch64)
2033 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2034 currState->longDesc.sh());
2035 else
2036 memAttrsLPAE(currState->tc, te, lDescriptor);
2037 } else {
2038 te.ap = descriptor.ap();
2039 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2040 descriptor.shareable());
2041 }
2042
2043 // Debug output
2044 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2045 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2046 te.N, te.pfn, te.size, te.global, te.valid);
2047 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2048 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2049 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2050 te.nonCacheable, te.ns);
2051 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2052 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2053 descriptor.getRawData());
2054
2055 // Insert the entry into the TLB
2056 tlb->insert(currState->vaddr, te);
2057 if (!currState->timing) {
2058 currState->tc = NULL;
2059 currState->req = NULL;
2060 }
2061}
2062
2063ArmISA::TableWalker *
2064ArmTableWalkerParams::create()
2065{
2066 return new ArmISA::TableWalker(this);
2067}
2068
2069LookupLevel
2070TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2071{
2072 switch (lookup_level_as_int) {
2073 case L1:
2074 return L1;
2075 case L2:
2076 return L2;
2077 case L3:
2078 return L3;
2079 default:
2080 panic("Invalid lookup level conversion");
2081 }
2082}
2083
2084/* this method keeps track of the table walker queue's residency, so
2085 * needs to be called whenever requests start and complete. */
2086void
2087TableWalker::pendingChange()
2088{
2089 unsigned n = pendingQueue.size();
2090 if ((currState != NULL) && (currState != pendingQueue.front())) {
2091 ++n;
2092 }
2093
2094 if (n != pendingReqs) {
2095 Tick now = curTick();
2096 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2097 pendingReqs = n;
2098 pendingChangeTick = now;
2099 }
2100}
2101
2102uint8_t
2103TableWalker::pageSizeNtoStatBin(uint8_t N)
2104{
2105 /* for statPageSizes */
2106 switch(N) {
2107 case 12: return 0; // 4K
2108 case 14: return 1; // 16K (using 16K granule in v8-64)
2109 case 16: return 2; // 64K
2110 case 20: return 3; // 1M
2111 case 21: return 4; // 2M-LPAE
2112 case 24: return 5; // 16M
2113 case 25: return 6; // 32M (using 16K granule in v8-64)
2114 case 29: return 7; // 512M (using 64K granule in v8-64)
2115 case 30: return 8; // 1G-LPAE
2116 default:
2117 panic("unknown page size");
2118 return 255;
2119 }
2120}
2121
2122void
2123TableWalker::regStats()
2124{
2125 statWalks
2126 .name(name() + ".walks")
2127 .desc("Table walker walks requested")
2128 ;
2129
2130 statWalksShortDescriptor
2131 .name(name() + ".walksShort")
2132 .desc("Table walker walks initiated with short descriptors")
2133 .flags(Stats::nozero)
2134 ;
2135
2136 statWalksLongDescriptor
2137 .name(name() + ".walksLong")
2138 .desc("Table walker walks initiated with long descriptors")
2139 .flags(Stats::nozero)
2140 ;
2141
2142 statWalksShortTerminatedAtLevel
2143 .init(2)
2144 .name(name() + ".walksShortTerminationLevel")
2145 .desc("Level at which table walker walks "
2146 "with short descriptors terminate")
2147 .flags(Stats::nozero)
2148 ;
2149 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2150 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2151
2152 statWalksLongTerminatedAtLevel
2153 .init(4)
2154 .name(name() + ".walksLongTerminationLevel")
2155 .desc("Level at which table walker walks "
2156 "with long descriptors terminate")
2157 .flags(Stats::nozero)
2158 ;
2159 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2160 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2161 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2162 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2163
2164 statSquashedBefore
2165 .name(name() + ".walksSquashedBefore")
2166 .desc("Table walks squashed before starting")
2167 .flags(Stats::nozero)
2168 ;
2169
2170 statSquashedAfter
2171 .name(name() + ".walksSquashedAfter")
2172 .desc("Table walks squashed after completion")
2173 .flags(Stats::nozero)
2174 ;
2175
2176 statWalkWaitTime
2177 .init(16)
2178 .name(name() + ".walkWaitTime")
2179 .desc("Table walker wait (enqueue to first request) latency")
2180 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2181 ;
2182
2183 statWalkServiceTime
2184 .init(16)
2185 .name(name() + ".walkCompletionTime")
2186 .desc("Table walker service (enqueue to completion) latency")
2187 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2188 ;
2189
2190 statPendingWalks
2191 .init(16)
2192 .name(name() + ".walksPending")
2193 .desc("Table walker pending requests distribution")
2194 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2195 ;
2196
2197 statPageSizes // see DDI 0487A D4-1661
2198 .init(9)
2199 .name(name() + ".walkPageSizes")
2200 .desc("Table walker page sizes translated")
2201 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2202 ;
2203 statPageSizes.subname(0, "4K");
2204 statPageSizes.subname(1, "16K");
2205 statPageSizes.subname(2, "64K");
2206 statPageSizes.subname(3, "1M");
2207 statPageSizes.subname(4, "2M");
2208 statPageSizes.subname(5, "16M");
2209 statPageSizes.subname(6, "32M");
2210 statPageSizes.subname(7, "512M");
2211 statPageSizes.subname(8, "1G");
2212
2213 statRequestOrigin
2214 .init(2,2) // Instruction/Data, requests/completed
2215 .name(name() + ".walkRequestOrigin")
2216 .desc("Table walker requests started/completed, data/inst")
2217 .flags(Stats::total)
2218 ;
2219 statRequestOrigin.subname(0,"Requested");
2220 statRequestOrigin.subname(1,"Completed");
2221 statRequestOrigin.ysubname(0,"Data");
2222 statRequestOrigin.ysubname(1,"Inst");
2223}
| 170 DPRINTF(Drain, "TableWalker free, no need to drain\n");
171
172 // table walker is drained, but its ports may still need to be drained
173 return 0;
174 }
175}
176
177void
178TableWalker::drainResume()
179{
180 Drainable::drainResume();
181 if (params()->sys->isTimingMode() && currState) {
182 delete currState;
183 currState = NULL;
184 pendingChange();
185 }
186}
187
188Fault
189TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
190 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
191 TLB::Translation *_trans, bool _timing, bool _functional,
192 bool secure, TLB::ArmTranslationType tranType)
193{
194 assert(!(_functional && _timing));
195 ++statWalks;
196
197 WalkerState *savedCurrState = NULL;
198
199 if (!currState && !_functional) {
200 // For atomic mode, a new WalkerState instance should be only created
201 // once per TLB. For timing mode, a new instance is generated for every
202 // TLB miss.
203 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
204
205 currState = new WalkerState();
206 currState->tableWalker = this;
207 } else if (_functional) {
208 // If we are mixing functional mode with timing (or even
209 // atomic), we need to to be careful and clean up after
210 // ourselves to not risk getting into an inconsistent state.
211 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
212 savedCurrState = currState;
213 currState = new WalkerState();
214 currState->tableWalker = this;
215 } else if (_timing) {
216 // This is a translation that was completed and then faulted again
217 // because some underlying parameters that affect the translation
218 // changed out from under us (e.g. asid). It will either be a
219 // misprediction, in which case nothing will happen or we'll use
220 // this fault to re-execute the faulting instruction which should clean
221 // up everything.
222 if (currState->vaddr_tainted == _req->getVaddr()) {
223 ++statSquashedBefore;
224 return std::make_shared<ReExec>();
225 }
226 }
227 pendingChange();
228
229 currState->startTime = curTick();
230 currState->tc = _tc;
231 currState->aarch64 = opModeIs64(currOpMode(_tc));
232 currState->el = currEL(_tc);
233 currState->transState = _trans;
234 currState->req = _req;
235 currState->fault = NoFault;
236 currState->asid = _asid;
237 currState->vmid = _vmid;
238 currState->isHyp = _isHyp;
239 currState->timing = _timing;
240 currState->functional = _functional;
241 currState->mode = _mode;
242 currState->tranType = tranType;
243 currState->isSecure = secure;
244 currState->physAddrRange = physAddrRange;
245
246 /** @todo These should be cached or grabbed from cached copies in
247 the TLB, all these miscreg reads are expensive */
248 currState->vaddr_tainted = currState->req->getVaddr();
249 if (currState->aarch64)
250 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
251 currState->tc, currState->el);
252 else
253 currState->vaddr = currState->vaddr_tainted;
254
255 if (currState->aarch64) {
256 switch (currState->el) {
257 case EL0:
258 case EL1:
259 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
260 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
261 break;
262 // @todo: uncomment this to enable Virtualization
263 // case EL2:
264 // assert(haveVirtualization);
265 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
266 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
267 // break;
268 case EL3:
269 assert(haveSecurity);
270 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
271 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
272 break;
273 default:
274 panic("Invalid exception level");
275 break;
276 }
277 } else {
278 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
279 MISCREG_SCTLR, currState->tc, !currState->isSecure));
280 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
281 MISCREG_TTBCR, currState->tc, !currState->isSecure));
282 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
283 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
284 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
285 }
286 sctlr = currState->sctlr;
287
288 currState->isFetch = (currState->mode == TLB::Execute);
289 currState->isWrite = (currState->mode == TLB::Write);
290
291 statRequestOrigin[REQUESTED][currState->isFetch]++;
292
293 // We only do a second stage of translation if we're not secure, or in
294 // hyp mode, the second stage MMU is enabled, and this table walker
295 // instance is the first stage.
296 currState->doingStage2 = false;
297 // @todo: for now disable this in AArch64 (HCR is not set)
298 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
299 !isStage2 && !currState->isSecure && !currState->isHyp;
300
301 bool long_desc_format = currState->aarch64 ||
302 (_haveLPAE && currState->ttbcr.eae) ||
303 _isHyp || isStage2;
304
305 if (long_desc_format) {
306 // Helper variables used for hierarchical permissions
307 currState->secureLookup = currState->isSecure;
308 currState->rwTable = true;
309 currState->userTable = true;
310 currState->xnTable = false;
311 currState->pxnTable = false;
312
313 ++statWalksLongDescriptor;
314 } else {
315 ++statWalksShortDescriptor;
316 }
317
318 if (!currState->timing) {
319 Fault fault = NoFault;
320 if (currState->aarch64)
321 fault = processWalkAArch64();
322 else if (long_desc_format)
323 fault = processWalkLPAE();
324 else
325 fault = processWalk();
326
327 // If this was a functional non-timing access restore state to
328 // how we found it.
329 if (currState->functional) {
330 delete currState;
331 currState = savedCurrState;
332 }
333 return fault;
334 }
335
336 if (pending || pendingQueue.size()) {
337 pendingQueue.push_back(currState);
338 currState = NULL;
339 pendingChange();
340 } else {
341 pending = true;
342 pendingChange();
343 if (currState->aarch64)
344 return processWalkAArch64();
345 else if (long_desc_format)
346 return processWalkLPAE();
347 else
348 return processWalk();
349 }
350
351 return NoFault;
352}
353
354void
355TableWalker::processWalkWrapper()
356{
357 assert(!currState);
358 assert(pendingQueue.size());
359 pendingChange();
360 currState = pendingQueue.front();
361
362 ExceptionLevel target_el = EL0;
363 if (currState->aarch64)
364 target_el = currEL(currState->tc);
365 else
366 target_el = EL1;
367
368 // Check if a previous walk filled this request already
369 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
370 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
371 currState->vmid, currState->isHyp, currState->isSecure, true, false,
372 target_el);
373
374 // Check if we still need to have a walk for this request. If the requesting
375 // instruction has been squashed, or a previous walk has filled the TLB with
376 // a match, we just want to get rid of the walk. The latter could happen
377 // when there are multiple outstanding misses to a single page and a
378 // previous request has been successfully translated.
379 if (!currState->transState->squashed() && !te) {
380 // We've got a valid request, lets process it
381 pending = true;
382 pendingQueue.pop_front();
383 // Keep currState in case one of the processWalk... calls NULLs it
384 WalkerState *curr_state_copy = currState;
385 Fault f;
386 if (currState->aarch64)
387 f = processWalkAArch64();
388 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
389 f = processWalkLPAE();
390 else
391 f = processWalk();
392
393 if (f != NoFault) {
394 curr_state_copy->transState->finish(f, curr_state_copy->req,
395 curr_state_copy->tc, curr_state_copy->mode);
396
397 delete curr_state_copy;
398 }
399 return;
400 }
401
402
403 // If the instruction that we were translating for has been
404 // squashed we shouldn't bother.
405 unsigned num_squashed = 0;
406 ThreadContext *tc = currState->tc;
407 while ((num_squashed < numSquashable) && currState &&
408 (currState->transState->squashed() || te)) {
409 pendingQueue.pop_front();
410 num_squashed++;
411 statSquashedBefore++;
412
413 DPRINTF(TLB, "Squashing table walk for address %#x\n",
414 currState->vaddr_tainted);
415
416 if (currState->transState->squashed()) {
417 // finish the translation which will delete the translation object
418 currState->transState->finish(
419 std::make_shared<UnimpFault>("Squashed Inst"),
420 currState->req, currState->tc, currState->mode);
421 } else {
422 // translate the request now that we know it will work
423 statWalkServiceTime.sample(curTick() - currState->startTime);
424 tlb->translateTiming(currState->req, currState->tc,
425 currState->transState, currState->mode);
426
427 }
428
429 // delete the current request
430 delete currState;
431
432 // peak at the next one
433 if (pendingQueue.size()) {
434 currState = pendingQueue.front();
435 te = tlb->lookup(currState->vaddr, currState->asid,
436 currState->vmid, currState->isHyp, currState->isSecure, true,
437 false, target_el);
438 } else {
439 // Terminate the loop, nothing more to do
440 currState = NULL;
441 }
442 }
443 pendingChange();
444
445 // if we still have pending translations, schedule more work
446 nextWalk(tc);
447 currState = NULL;
448}
449
450Fault
451TableWalker::processWalk()
452{
453 Addr ttbr = 0;
454
455 // If translation isn't enabled, we shouldn't be here
456 assert(currState->sctlr.m || isStage2);
457
458 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
459 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
460 32 - currState->ttbcr.n));
461
462 statWalkWaitTime.sample(curTick() - currState->startTime);
463
464 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
465 32 - currState->ttbcr.n)) {
466 DPRINTF(TLB, " - Selecting TTBR0\n");
467 // Check if table walk is allowed when Security Extensions are enabled
468 if (haveSecurity && currState->ttbcr.pd0) {
469 if (currState->isFetch)
470 return std::make_shared<PrefetchAbort>(
471 currState->vaddr_tainted,
472 ArmFault::TranslationLL + L1,
473 isStage2,
474 ArmFault::VmsaTran);
475 else
476 return std::make_shared<DataAbort>(
477 currState->vaddr_tainted,
478 TlbEntry::DomainType::NoAccess, currState->isWrite,
479 ArmFault::TranslationLL + L1, isStage2,
480 ArmFault::VmsaTran);
481 }
482 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
483 MISCREG_TTBR0, currState->tc, !currState->isSecure));
484 } else {
485 DPRINTF(TLB, " - Selecting TTBR1\n");
486 // Check if table walk is allowed when Security Extensions are enabled
487 if (haveSecurity && currState->ttbcr.pd1) {
488 if (currState->isFetch)
489 return std::make_shared<PrefetchAbort>(
490 currState->vaddr_tainted,
491 ArmFault::TranslationLL + L1,
492 isStage2,
493 ArmFault::VmsaTran);
494 else
495 return std::make_shared<DataAbort>(
496 currState->vaddr_tainted,
497 TlbEntry::DomainType::NoAccess, currState->isWrite,
498 ArmFault::TranslationLL + L1, isStage2,
499 ArmFault::VmsaTran);
500 }
501 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
502 MISCREG_TTBR1, currState->tc, !currState->isSecure));
503 currState->ttbcr.n = 0;
504 }
505
506 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
507 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
508 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
509 currState->isSecure ? "s" : "ns");
510
511 // Trickbox address check
512 Fault f;
513 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
514 currState->vaddr, sizeof(uint32_t), currState->isFetch,
515 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
516 if (f) {
517 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
518 if (currState->timing) {
519 pending = false;
520 nextWalk(currState->tc);
521 currState = NULL;
522 } else {
523 currState->tc = NULL;
524 currState->req = NULL;
525 }
526 return f;
527 }
528
529 Request::Flags flag = Request::PT_WALK;
530 if (currState->sctlr.c == 0) {
531 flag.set(Request::UNCACHEABLE);
532 }
533
534 bool delayed;
535 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
536 sizeof(uint32_t), flag, L1, &doL1DescEvent,
537 &TableWalker::doL1Descriptor);
538 if (!delayed) {
539 f = currState->fault;
540 }
541
542 return f;
543}
544
545Fault
546TableWalker::processWalkLPAE()
547{
548 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
549 int tsz, n;
550 LookupLevel start_lookup_level = L1;
551
552 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
553 currState->vaddr_tainted, currState->ttbcr);
554
555 statWalkWaitTime.sample(curTick() - currState->startTime);
556
557 Request::Flags flag = Request::PT_WALK;
558 if (currState->isSecure)
559 flag.set(Request::SECURE);
560
561 // work out which base address register to use, if in hyp mode we always
562 // use HTTBR
563 if (isStage2) {
564 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
565 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
566 tsz = sext<4>(currState->vtcr.t0sz);
567 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
568 } else if (currState->isHyp) {
569 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
570 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
571 tsz = currState->htcr.t0sz;
572 } else {
573 assert(_haveLPAE && currState->ttbcr.eae);
574
575 // Determine boundaries of TTBR0/1 regions
576 if (currState->ttbcr.t0sz)
577 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
578 else if (currState->ttbcr.t1sz)
579 ttbr0_max = (1ULL << 32) -
580 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
581 else
582 ttbr0_max = (1ULL << 32) - 1;
583 if (currState->ttbcr.t1sz)
584 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
585 else
586 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
587
588 // The following code snippet selects the appropriate translation table base
589 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
590 // depending on the address range supported by the translation table (ARM
591 // ARM issue C B3.6.4)
592 if (currState->vaddr <= ttbr0_max) {
593 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
594 // Check if table walk is allowed
595 if (currState->ttbcr.epd0) {
596 if (currState->isFetch)
597 return std::make_shared<PrefetchAbort>(
598 currState->vaddr_tainted,
599 ArmFault::TranslationLL + L1,
600 isStage2,
601 ArmFault::LpaeTran);
602 else
603 return std::make_shared<DataAbort>(
604 currState->vaddr_tainted,
605 TlbEntry::DomainType::NoAccess,
606 currState->isWrite,
607 ArmFault::TranslationLL + L1,
608 isStage2,
609 ArmFault::LpaeTran);
610 }
611 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
612 MISCREG_TTBR0, currState->tc, !currState->isSecure));
613 tsz = currState->ttbcr.t0sz;
614 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
615 start_lookup_level = L2;
616 } else if (currState->vaddr >= ttbr1_min) {
617 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
618 // Check if table walk is allowed
619 if (currState->ttbcr.epd1) {
620 if (currState->isFetch)
621 return std::make_shared<PrefetchAbort>(
622 currState->vaddr_tainted,
623 ArmFault::TranslationLL + L1,
624 isStage2,
625 ArmFault::LpaeTran);
626 else
627 return std::make_shared<DataAbort>(
628 currState->vaddr_tainted,
629 TlbEntry::DomainType::NoAccess,
630 currState->isWrite,
631 ArmFault::TranslationLL + L1,
632 isStage2,
633 ArmFault::LpaeTran);
634 }
635 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
636 MISCREG_TTBR1, currState->tc, !currState->isSecure));
637 tsz = currState->ttbcr.t1sz;
638 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
639 start_lookup_level = L2;
640 } else {
641 // Out of boundaries -> translation fault
642 if (currState->isFetch)
643 return std::make_shared<PrefetchAbort>(
644 currState->vaddr_tainted,
645 ArmFault::TranslationLL + L1,
646 isStage2,
647 ArmFault::LpaeTran);
648 else
649 return std::make_shared<DataAbort>(
650 currState->vaddr_tainted,
651 TlbEntry::DomainType::NoAccess,
652 currState->isWrite, ArmFault::TranslationLL + L1,
653 isStage2, ArmFault::LpaeTran);
654 }
655
656 }
657
658 // Perform lookup (ARM ARM issue C B3.6.6)
659 if (start_lookup_level == L1) {
660 n = 5 - tsz;
661 desc_addr = mbits(ttbr, 39, n) |
662 (bits(currState->vaddr, n + 26, 30) << 3);
663 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
664 desc_addr, currState->isSecure ? "s" : "ns");
665 } else {
666 // Skip first-level lookup
667 n = (tsz >= 2 ? 14 - tsz : 12);
668 desc_addr = mbits(ttbr, 39, n) |
669 (bits(currState->vaddr, n + 17, 21) << 3);
670 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
671 desc_addr, currState->isSecure ? "s" : "ns");
672 }
673
674 // Trickbox address check
675 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
676 currState->vaddr, sizeof(uint64_t), currState->isFetch,
677 currState->isWrite, TlbEntry::DomainType::NoAccess,
678 start_lookup_level);
679 if (f) {
680 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
681 if (currState->timing) {
682 pending = false;
683 nextWalk(currState->tc);
684 currState = NULL;
685 } else {
686 currState->tc = NULL;
687 currState->req = NULL;
688 }
689 return f;
690 }
691
692 if (currState->sctlr.c == 0) {
693 flag.set(Request::UNCACHEABLE);
694 }
695
696 if (currState->isSecure)
697 flag.set(Request::SECURE);
698
699 currState->longDesc.lookupLevel = start_lookup_level;
700 currState->longDesc.aarch64 = false;
701 currState->longDesc.grainSize = Grain4KB;
702
703 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
704 : (Event *) &doL2LongDescEvent;
705
706 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
707 sizeof(uint64_t), flag, start_lookup_level,
708 event, &TableWalker::doLongDescriptor);
709 if (!delayed) {
710 f = currState->fault;
711 }
712
713 return f;
714}
715
716unsigned
717TableWalker::adjustTableSizeAArch64(unsigned tsz)
718{
719 if (tsz < 25)
720 return 25;
721 if (tsz > 48)
722 return 48;
723 return tsz;
724}
725
726bool
727TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
728{
729 return (currPhysAddrRange != MaxPhysAddrRange &&
730 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
731}
732
733Fault
734TableWalker::processWalkAArch64()
735{
736 assert(currState->aarch64);
737
738 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
739 currState->vaddr_tainted, currState->tcr);
740
741 static const GrainSize GrainMapDefault[] =
742 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
743 static const GrainSize GrainMap_EL1_tg1[] =
744 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
745
746 statWalkWaitTime.sample(curTick() - currState->startTime);
747
748 // Determine TTBR, table size, granule size and phys. address range
749 Addr ttbr = 0;
750 int tsz = 0, ps = 0;
751 GrainSize tg = Grain4KB; // grain size computed from tg* field
752 bool fault = false;
753 switch (currState->el) {
754 case EL0:
755 case EL1:
756 switch (bits(currState->vaddr, 63,48)) {
757 case 0:
758 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
759 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
760 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
761 tg = GrainMapDefault[currState->tcr.tg0];
762 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
763 currState->tcr.epd0)
764 fault = true;
765 break;
766 case 0xffff:
767 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
768 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
769 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
770 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
771 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
772 currState->tcr.epd1)
773 fault = true;
774 break;
775 default:
776 // top two bytes must be all 0s or all 1s, else invalid addr
777 fault = true;
778 }
779 ps = currState->tcr.ips;
780 break;
781 case EL2:
782 case EL3:
783 switch(bits(currState->vaddr, 63,48)) {
784 case 0:
785 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
786 if (currState->el == EL2)
787 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
788 else
789 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
790 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
791 tg = GrainMapDefault[currState->tcr.tg0];
792 break;
793 default:
794 // invalid addr if top two bytes are not all 0s
795 fault = true;
796 }
797 ps = currState->tcr.ips;
798 break;
799 }
800
801 if (fault) {
802 Fault f;
803 if (currState->isFetch)
804 f = std::make_shared<PrefetchAbort>(
805 currState->vaddr_tainted,
806 ArmFault::TranslationLL + L0, isStage2,
807 ArmFault::LpaeTran);
808 else
809 f = std::make_shared<DataAbort>(
810 currState->vaddr_tainted,
811 TlbEntry::DomainType::NoAccess,
812 currState->isWrite,
813 ArmFault::TranslationLL + L0,
814 isStage2, ArmFault::LpaeTran);
815
816 if (currState->timing) {
817 pending = false;
818 nextWalk(currState->tc);
819 currState = NULL;
820 } else {
821 currState->tc = NULL;
822 currState->req = NULL;
823 }
824 return f;
825
826 }
827
828 if (tg == ReservedGrain) {
829 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
830 "DEFINED behavior takes this to mean 4KB granules\n");
831 tg = Grain4KB;
832 }
833
834 int stride = tg - 3;
835 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
836
837 // Determine starting lookup level
838 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
839 // in ARM DDI 0487A. These table values correspond to the cascading tests
840 // to compute the lookup level and are of the form
841 // (grain_size + N*stride), for N = {1, 2, 3}.
842 // A value of 64 will never succeed and a value of 0 will always succeed.
843 {
844 struct GrainMap {
845 GrainSize grain_size;
846 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
847 };
848 static const GrainMap GM[] = {
849 { Grain4KB, { 39, 30, 0, 0 } },
850 { Grain16KB, { 47, 36, 25, 0 } },
851 { Grain64KB, { 64, 42, 29, 0 } }
852 };
853
854 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
855
856 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
857 if (tg == GM[i].grain_size) {
858 lookup = GM[i].lookup_level_cutoff;
859 break;
860 }
861 }
862 assert(lookup);
863
864 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
865 if (tsz > lookup[L]) {
866 start_lookup_level = (LookupLevel) L;
867 break;
868 }
869 }
870 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
871 "Table walker couldn't find lookup level\n");
872 }
873
874 // Determine table base address
875 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
876 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
877
878 // Determine physical address size and raise an Address Size Fault if
879 // necessary
880 int pa_range = decodePhysAddrRange64(ps);
881 // Clamp to lower limit
882 if (pa_range > physAddrRange)
883 currState->physAddrRange = physAddrRange;
884 else
885 currState->physAddrRange = pa_range;
886 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
887 DPRINTF(TLB, "Address size fault before any lookup\n");
888 Fault f;
889 if (currState->isFetch)
890 f = std::make_shared<PrefetchAbort>(
891 currState->vaddr_tainted,
892 ArmFault::AddressSizeLL + start_lookup_level,
893 isStage2,
894 ArmFault::LpaeTran);
895 else
896 f = std::make_shared<DataAbort>(
897 currState->vaddr_tainted,
898 TlbEntry::DomainType::NoAccess,
899 currState->isWrite,
900 ArmFault::AddressSizeLL + start_lookup_level,
901 isStage2,
902 ArmFault::LpaeTran);
903
904
905 if (currState->timing) {
906 pending = false;
907 nextWalk(currState->tc);
908 currState = NULL;
909 } else {
910 currState->tc = NULL;
911 currState->req = NULL;
912 }
913 return f;
914
915 }
916
917 // Determine descriptor address
918 Addr desc_addr = base_addr |
919 (bits(currState->vaddr, tsz - 1,
920 stride * (3 - start_lookup_level) + tg) << 3);
921
922 // Trickbox address check
923 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
924 currState->vaddr, sizeof(uint64_t), currState->isFetch,
925 currState->isWrite, TlbEntry::DomainType::NoAccess,
926 start_lookup_level);
927 if (f) {
928 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
929 if (currState->timing) {
930 pending = false;
931 nextWalk(currState->tc);
932 currState = NULL;
933 } else {
934 currState->tc = NULL;
935 currState->req = NULL;
936 }
937 return f;
938 }
939
940 Request::Flags flag = Request::PT_WALK;
941 if (currState->sctlr.c == 0) {
942 flag.set(Request::UNCACHEABLE);
943 }
944
945 currState->longDesc.lookupLevel = start_lookup_level;
946 currState->longDesc.aarch64 = true;
947 currState->longDesc.grainSize = tg;
948
949 if (currState->timing) {
950 Event *event;
951 switch (start_lookup_level) {
952 case L0:
953 event = (Event *) &doL0LongDescEvent;
954 break;
955 case L1:
956 event = (Event *) &doL1LongDescEvent;
957 break;
958 case L2:
959 event = (Event *) &doL2LongDescEvent;
960 break;
961 case L3:
962 event = (Event *) &doL3LongDescEvent;
963 break;
964 default:
965 panic("Invalid table lookup level");
966 break;
967 }
968 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
969 event, (uint8_t*) &currState->longDesc.data,
970 currState->tc->getCpuPtr()->clockPeriod(), flag);
971 DPRINTF(TLBVerbose,
972 "Adding to walker fifo: queue size before adding: %d\n",
973 stateQueues[start_lookup_level].size());
974 stateQueues[start_lookup_level].push_back(currState);
975 currState = NULL;
976 } else if (!currState->functional) {
977 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
978 NULL, (uint8_t*) &currState->longDesc.data,
979 currState->tc->getCpuPtr()->clockPeriod(), flag);
980 doLongDescriptor();
981 f = currState->fault;
982 } else {
983 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
984 masterId);
985 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
986 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
987 port->sendFunctional(pkt);
988 doLongDescriptor();
989 delete req;
990 delete pkt;
991 f = currState->fault;
992 }
993
994 return f;
995}
996
997void
998TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
999 uint8_t texcb, bool s)
1000{
1001 // Note: tc and sctlr local variables are hiding tc and sctrl class
1002 // variables
1003 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1004 te.shareable = false; // default value
1005 te.nonCacheable = false;
1006 te.outerShareable = false;
1007 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1008 switch(texcb) {
1009 case 0: // Stongly-ordered
1010 te.nonCacheable = true;
1011 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1012 te.shareable = true;
1013 te.innerAttrs = 1;
1014 te.outerAttrs = 0;
1015 break;
1016 case 1: // Shareable Device
1017 te.nonCacheable = true;
1018 te.mtype = TlbEntry::MemoryType::Device;
1019 te.shareable = true;
1020 te.innerAttrs = 3;
1021 te.outerAttrs = 0;
1022 break;
1023 case 2: // Outer and Inner Write-Through, no Write-Allocate
1024 te.mtype = TlbEntry::MemoryType::Normal;
1025 te.shareable = s;
1026 te.innerAttrs = 6;
1027 te.outerAttrs = bits(texcb, 1, 0);
1028 break;
1029 case 3: // Outer and Inner Write-Back, no Write-Allocate
1030 te.mtype = TlbEntry::MemoryType::Normal;
1031 te.shareable = s;
1032 te.innerAttrs = 7;
1033 te.outerAttrs = bits(texcb, 1, 0);
1034 break;
1035 case 4: // Outer and Inner Non-cacheable
1036 te.nonCacheable = true;
1037 te.mtype = TlbEntry::MemoryType::Normal;
1038 te.shareable = s;
1039 te.innerAttrs = 0;
1040 te.outerAttrs = bits(texcb, 1, 0);
1041 break;
1042 case 5: // Reserved
1043 panic("Reserved texcb value!\n");
1044 break;
1045 case 6: // Implementation Defined
1046 panic("Implementation-defined texcb value!\n");
1047 break;
1048 case 7: // Outer and Inner Write-Back, Write-Allocate
1049 te.mtype = TlbEntry::MemoryType::Normal;
1050 te.shareable = s;
1051 te.innerAttrs = 5;
1052 te.outerAttrs = 1;
1053 break;
1054 case 8: // Non-shareable Device
1055 te.nonCacheable = true;
1056 te.mtype = TlbEntry::MemoryType::Device;
1057 te.shareable = false;
1058 te.innerAttrs = 3;
1059 te.outerAttrs = 0;
1060 break;
1061 case 9 ... 15: // Reserved
1062 panic("Reserved texcb value!\n");
1063 break;
1064 case 16 ... 31: // Cacheable Memory
1065 te.mtype = TlbEntry::MemoryType::Normal;
1066 te.shareable = s;
1067 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1068 te.nonCacheable = true;
1069 te.innerAttrs = bits(texcb, 1, 0);
1070 te.outerAttrs = bits(texcb, 3, 2);
1071 break;
1072 default:
1073 panic("More than 32 states for 5 bits?\n");
1074 }
1075 } else {
1076 assert(tc);
1077 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1078 currState->tc, !currState->isSecure));
1079 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1080 currState->tc, !currState->isSecure));
1081 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1082 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1083 switch(bits(texcb, 2,0)) {
1084 case 0:
1085 curr_tr = prrr.tr0;
1086 curr_ir = nmrr.ir0;
1087 curr_or = nmrr.or0;
1088 te.outerShareable = (prrr.nos0 == 0);
1089 break;
1090 case 1:
1091 curr_tr = prrr.tr1;
1092 curr_ir = nmrr.ir1;
1093 curr_or = nmrr.or1;
1094 te.outerShareable = (prrr.nos1 == 0);
1095 break;
1096 case 2:
1097 curr_tr = prrr.tr2;
1098 curr_ir = nmrr.ir2;
1099 curr_or = nmrr.or2;
1100 te.outerShareable = (prrr.nos2 == 0);
1101 break;
1102 case 3:
1103 curr_tr = prrr.tr3;
1104 curr_ir = nmrr.ir3;
1105 curr_or = nmrr.or3;
1106 te.outerShareable = (prrr.nos3 == 0);
1107 break;
1108 case 4:
1109 curr_tr = prrr.tr4;
1110 curr_ir = nmrr.ir4;
1111 curr_or = nmrr.or4;
1112 te.outerShareable = (prrr.nos4 == 0);
1113 break;
1114 case 5:
1115 curr_tr = prrr.tr5;
1116 curr_ir = nmrr.ir5;
1117 curr_or = nmrr.or5;
1118 te.outerShareable = (prrr.nos5 == 0);
1119 break;
1120 case 6:
1121 panic("Imp defined type\n");
1122 case 7:
1123 curr_tr = prrr.tr7;
1124 curr_ir = nmrr.ir7;
1125 curr_or = nmrr.or7;
1126 te.outerShareable = (prrr.nos7 == 0);
1127 break;
1128 }
1129
1130 switch(curr_tr) {
1131 case 0:
1132 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1133 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1134 te.nonCacheable = true;
1135 te.innerAttrs = 1;
1136 te.outerAttrs = 0;
1137 te.shareable = true;
1138 break;
1139 case 1:
1140 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1141 prrr.ds1, prrr.ds0, s);
1142 te.mtype = TlbEntry::MemoryType::Device;
1143 te.nonCacheable = true;
1144 te.innerAttrs = 3;
1145 te.outerAttrs = 0;
1146 if (prrr.ds1 && s)
1147 te.shareable = true;
1148 if (prrr.ds0 && !s)
1149 te.shareable = true;
1150 break;
1151 case 2:
1152 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1153 prrr.ns1, prrr.ns0, s);
1154 te.mtype = TlbEntry::MemoryType::Normal;
1155 if (prrr.ns1 && s)
1156 te.shareable = true;
1157 if (prrr.ns0 && !s)
1158 te.shareable = true;
1159 break;
1160 case 3:
1161 panic("Reserved type");
1162 }
1163
1164 if (te.mtype == TlbEntry::MemoryType::Normal){
1165 switch(curr_ir) {
1166 case 0:
1167 te.nonCacheable = true;
1168 te.innerAttrs = 0;
1169 break;
1170 case 1:
1171 te.innerAttrs = 5;
1172 break;
1173 case 2:
1174 te.innerAttrs = 6;
1175 break;
1176 case 3:
1177 te.innerAttrs = 7;
1178 break;
1179 }
1180
1181 switch(curr_or) {
1182 case 0:
1183 te.nonCacheable = true;
1184 te.outerAttrs = 0;
1185 break;
1186 case 1:
1187 te.outerAttrs = 1;
1188 break;
1189 case 2:
1190 te.outerAttrs = 2;
1191 break;
1192 case 3:
1193 te.outerAttrs = 3;
1194 break;
1195 }
1196 }
1197 }
1198 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1199 "outerAttrs: %d\n",
1200 te.shareable, te.innerAttrs, te.outerAttrs);
1201 te.setAttributes(false);
1202}
1203
1204void
1205TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1206 LongDescriptor &lDescriptor)
1207{
1208 assert(_haveLPAE);
1209
1210 uint8_t attr;
1211 uint8_t sh = lDescriptor.sh();
1212 // Different format and source of attributes if this is a stage 2
1213 // translation
1214 if (isStage2) {
1215 attr = lDescriptor.memAttr();
1216 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1217 uint8_t attr_1_0 = attr & 0x3;
1218
1219 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1220
1221 if (attr_3_2 == 0) {
1222 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1223 : TlbEntry::MemoryType::Device;
1224 te.outerAttrs = 0;
1225 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1226 te.nonCacheable = true;
1227 } else {
1228 te.mtype = TlbEntry::MemoryType::Normal;
1229 te.outerAttrs = attr_3_2 == 1 ? 0 :
1230 attr_3_2 == 2 ? 2 : 1;
1231 te.innerAttrs = attr_1_0 == 1 ? 0 :
1232 attr_1_0 == 2 ? 6 : 5;
1233 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1234 }
1235 } else {
1236 uint8_t attrIndx = lDescriptor.attrIndx();
1237
1238 // LPAE always uses remapping of memory attributes, irrespective of the
1239 // value of SCTLR.TRE
1240 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1241 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1242 !currState->isSecure);
1243 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1244 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1245 uint8_t attr_7_4 = bits(attr, 7, 4);
1246 uint8_t attr_3_0 = bits(attr, 3, 0);
1247 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1248
1249 // Note: the memory subsystem only cares about the 'cacheable' memory
1250 // attribute. The other attributes are only used to fill the PAR register
1251 // accordingly to provide the illusion of full support
1252 te.nonCacheable = false;
1253
1254 switch (attr_7_4) {
1255 case 0x0:
1256 // Strongly-ordered or Device memory
1257 if (attr_3_0 == 0x0)
1258 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1259 else if (attr_3_0 == 0x4)
1260 te.mtype = TlbEntry::MemoryType::Device;
1261 else
1262 panic("Unpredictable behavior\n");
1263 te.nonCacheable = true;
1264 te.outerAttrs = 0;
1265 break;
1266 case 0x4:
1267 // Normal memory, Outer Non-cacheable
1268 te.mtype = TlbEntry::MemoryType::Normal;
1269 te.outerAttrs = 0;
1270 if (attr_3_0 == 0x4)
1271 // Inner Non-cacheable
1272 te.nonCacheable = true;
1273 else if (attr_3_0 < 0x8)
1274 panic("Unpredictable behavior\n");
1275 break;
1276 case 0x8:
1277 case 0x9:
1278 case 0xa:
1279 case 0xb:
1280 case 0xc:
1281 case 0xd:
1282 case 0xe:
1283 case 0xf:
1284 if (attr_7_4 & 0x4) {
1285 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1286 } else {
1287 te.outerAttrs = 0x2;
1288 }
1289 // Normal memory, Outer Cacheable
1290 te.mtype = TlbEntry::MemoryType::Normal;
1291 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1292 panic("Unpredictable behavior\n");
1293 break;
1294 default:
1295 panic("Unpredictable behavior\n");
1296 break;
1297 }
1298
1299 switch (attr_3_0) {
1300 case 0x0:
1301 te.innerAttrs = 0x1;
1302 break;
1303 case 0x4:
1304 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1305 break;
1306 case 0x8:
1307 case 0x9:
1308 case 0xA:
1309 case 0xB:
1310 te.innerAttrs = 6;
1311 break;
1312 case 0xC:
1313 case 0xD:
1314 case 0xE:
1315 case 0xF:
1316 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1317 break;
1318 default:
1319 panic("Unpredictable behavior\n");
1320 break;
1321 }
1322 }
1323
1324 te.outerShareable = sh == 2;
1325 te.shareable = (sh & 0x2) ? true : false;
1326 te.setAttributes(true);
1327 te.attributes |= (uint64_t) attr << 56;
1328}
1329
1330void
1331TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1332 uint8_t sh)
1333{
1334 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1335
1336 // Select MAIR
1337 uint64_t mair;
1338 switch (currState->el) {
1339 case EL0:
1340 case EL1:
1341 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1342 break;
1343 case EL2:
1344 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1345 break;
1346 case EL3:
1347 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1348 break;
1349 default:
1350 panic("Invalid exception level");
1351 break;
1352 }
1353
1354 // Select attributes
1355 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1356 uint8_t attr_lo = bits(attr, 3, 0);
1357 uint8_t attr_hi = bits(attr, 7, 4);
1358
1359 // Memory type
1360 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1361
1362 // Cacheability
1363 te.nonCacheable = false;
1364 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1365 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1366 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1367 te.nonCacheable = true;
1368 }
1369
1370 te.shareable = sh == 2;
1371 te.outerShareable = (sh & 0x2) ? true : false;
1372 // Attributes formatted according to the 64-bit PAR
1373 te.attributes = ((uint64_t) attr << 56) |
1374 (1 << 11) | // LPAE bit
1375 (te.ns << 9) | // NS bit
1376 (sh << 7);
1377}
1378
1379void
1380TableWalker::doL1Descriptor()
1381{
1382 if (currState->fault != NoFault) {
1383 return;
1384 }
1385
1386 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1387 currState->vaddr_tainted, currState->l1Desc.data);
1388 TlbEntry te;
1389
1390 switch (currState->l1Desc.type()) {
1391 case L1Descriptor::Ignore:
1392 case L1Descriptor::Reserved:
1393 if (!currState->timing) {
1394 currState->tc = NULL;
1395 currState->req = NULL;
1396 }
1397 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1398 if (currState->isFetch)
1399 currState->fault =
1400 std::make_shared<PrefetchAbort>(
1401 currState->vaddr_tainted,
1402 ArmFault::TranslationLL + L1,
1403 isStage2,
1404 ArmFault::VmsaTran);
1405 else
1406 currState->fault =
1407 std::make_shared<DataAbort>(
1408 currState->vaddr_tainted,
1409 TlbEntry::DomainType::NoAccess,
1410 currState->isWrite,
1411 ArmFault::TranslationLL + L1, isStage2,
1412 ArmFault::VmsaTran);
1413 return;
1414 case L1Descriptor::Section:
1415 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1416 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1417 * enabled if set, do l1.Desc.setAp0() instead of generating
1418 * AccessFlag0
1419 */
1420
1421 currState->fault = std::make_shared<DataAbort>(
1422 currState->vaddr_tainted,
1423 currState->l1Desc.domain(),
1424 currState->isWrite,
1425 ArmFault::AccessFlagLL + L1,
1426 isStage2,
1427 ArmFault::VmsaTran);
1428 }
1429 if (currState->l1Desc.supersection()) {
1430 panic("Haven't implemented supersections\n");
1431 }
1432 insertTableEntry(currState->l1Desc, false);
1433 return;
1434 case L1Descriptor::PageTable:
1435 {
1436 Addr l2desc_addr;
1437 l2desc_addr = currState->l1Desc.l2Addr() |
1438 (bits(currState->vaddr, 19, 12) << 2);
1439 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1440 l2desc_addr, currState->isSecure ? "s" : "ns");
1441
1442 // Trickbox address check
1443 currState->fault = tlb->walkTrickBoxCheck(
1444 l2desc_addr, currState->isSecure, currState->vaddr,
1445 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1446 currState->l1Desc.domain(), L2);
1447
1448 if (currState->fault) {
1449 if (!currState->timing) {
1450 currState->tc = NULL;
1451 currState->req = NULL;
1452 }
1453 return;
1454 }
1455
1456 Request::Flags flag = Request::PT_WALK;
1457 if (currState->isSecure)
1458 flag.set(Request::SECURE);
1459
1460 bool delayed;
1461 delayed = fetchDescriptor(l2desc_addr,
1462 (uint8_t*)&currState->l2Desc.data,
1463 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1464 &TableWalker::doL2Descriptor);
1465 if (delayed) {
1466 currState->delayed = true;
1467 }
1468
1469 return;
1470 }
1471 default:
1472 panic("A new type in a 2 bit field?\n");
1473 }
1474}
1475
1476void
1477TableWalker::doLongDescriptor()
1478{
1479 if (currState->fault != NoFault) {
1480 return;
1481 }
1482
1483 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1484 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1485 currState->longDesc.data,
1486 currState->aarch64 ? "AArch64" : "long-desc.");
1487
1488 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1489 (currState->longDesc.type() == LongDescriptor::Page)) {
1490 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1491 "xn: %d, ap: %d, af: %d, type: %d\n",
1492 currState->longDesc.lookupLevel,
1493 currState->longDesc.data,
1494 currState->longDesc.pxn(),
1495 currState->longDesc.xn(),
1496 currState->longDesc.ap(),
1497 currState->longDesc.af(),
1498 currState->longDesc.type());
1499 } else {
1500 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1501 currState->longDesc.lookupLevel,
1502 currState->longDesc.data,
1503 currState->longDesc.type());
1504 }
1505
1506 TlbEntry te;
1507
1508 switch (currState->longDesc.type()) {
1509 case LongDescriptor::Invalid:
1510 if (!currState->timing) {
1511 currState->tc = NULL;
1512 currState->req = NULL;
1513 }
1514
1515 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1516 currState->longDesc.lookupLevel,
1517 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1518 if (currState->isFetch)
1519 currState->fault = std::make_shared<PrefetchAbort>(
1520 currState->vaddr_tainted,
1521 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1522 isStage2,
1523 ArmFault::LpaeTran);
1524 else
1525 currState->fault = std::make_shared<DataAbort>(
1526 currState->vaddr_tainted,
1527 TlbEntry::DomainType::NoAccess,
1528 currState->isWrite,
1529 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1530 isStage2,
1531 ArmFault::LpaeTran);
1532 return;
1533 case LongDescriptor::Block:
1534 case LongDescriptor::Page:
1535 {
1536 bool fault = false;
1537 bool aff = false;
1538 // Check for address size fault
1539 if (checkAddrSizeFaultAArch64(
1540 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1541 currState->longDesc.offsetBits()),
1542 currState->physAddrRange)) {
1543 fault = true;
1544 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1545 currState->longDesc.lookupLevel);
1546 // Check for access fault
1547 } else if (currState->longDesc.af() == 0) {
1548 fault = true;
1549 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1550 currState->longDesc.lookupLevel);
1551 aff = true;
1552 }
1553 if (fault) {
1554 if (currState->isFetch)
1555 currState->fault = std::make_shared<PrefetchAbort>(
1556 currState->vaddr_tainted,
1557 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1558 currState->longDesc.lookupLevel,
1559 isStage2,
1560 ArmFault::LpaeTran);
1561 else
1562 currState->fault = std::make_shared<DataAbort>(
1563 currState->vaddr_tainted,
1564 TlbEntry::DomainType::NoAccess, currState->isWrite,
1565 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1566 currState->longDesc.lookupLevel,
1567 isStage2,
1568 ArmFault::LpaeTran);
1569 } else {
1570 insertTableEntry(currState->longDesc, true);
1571 }
1572 }
1573 return;
1574 case LongDescriptor::Table:
1575 {
1576 // Set hierarchical permission flags
1577 currState->secureLookup = currState->secureLookup &&
1578 currState->longDesc.secureTable();
1579 currState->rwTable = currState->rwTable &&
1580 currState->longDesc.rwTable();
1581 currState->userTable = currState->userTable &&
1582 currState->longDesc.userTable();
1583 currState->xnTable = currState->xnTable ||
1584 currState->longDesc.xnTable();
1585 currState->pxnTable = currState->pxnTable ||
1586 currState->longDesc.pxnTable();
1587
1588 // Set up next level lookup
1589 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1590 currState->vaddr);
1591
1592 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1593 currState->longDesc.lookupLevel,
1594 currState->longDesc.lookupLevel + 1,
1595 next_desc_addr,
1596 currState->secureLookup ? "s" : "ns");
1597
1598 // Check for address size fault
1599 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1600 next_desc_addr, currState->physAddrRange)) {
1601 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1602 currState->longDesc.lookupLevel);
1603 if (currState->isFetch)
1604 currState->fault = std::make_shared<PrefetchAbort>(
1605 currState->vaddr_tainted,
1606 ArmFault::AddressSizeLL
1607 + currState->longDesc.lookupLevel,
1608 isStage2,
1609 ArmFault::LpaeTran);
1610 else
1611 currState->fault = std::make_shared<DataAbort>(
1612 currState->vaddr_tainted,
1613 TlbEntry::DomainType::NoAccess, currState->isWrite,
1614 ArmFault::AddressSizeLL
1615 + currState->longDesc.lookupLevel,
1616 isStage2,
1617 ArmFault::LpaeTran);
1618 return;
1619 }
1620
1621 // Trickbox address check
1622 currState->fault = tlb->walkTrickBoxCheck(
1623 next_desc_addr, currState->vaddr,
1624 currState->vaddr, sizeof(uint64_t),
1625 currState->isFetch, currState->isWrite,
1626 TlbEntry::DomainType::Client,
1627 toLookupLevel(currState->longDesc.lookupLevel +1));
1628
1629 if (currState->fault) {
1630 if (!currState->timing) {
1631 currState->tc = NULL;
1632 currState->req = NULL;
1633 }
1634 return;
1635 }
1636
1637 Request::Flags flag = Request::PT_WALK;
1638 if (currState->secureLookup)
1639 flag.set(Request::SECURE);
1640
1641 currState->longDesc.lookupLevel =
1642 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1643 Event *event = NULL;
1644 switch (currState->longDesc.lookupLevel) {
1645 case L1:
1646 assert(currState->aarch64);
1647 event = &doL1LongDescEvent;
1648 break;
1649 case L2:
1650 event = &doL2LongDescEvent;
1651 break;
1652 case L3:
1653 event = &doL3LongDescEvent;
1654 break;
1655 default:
1656 panic("Wrong lookup level in table walk\n");
1657 break;
1658 }
1659
1660 bool delayed;
1661 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1662 sizeof(uint64_t), flag, -1, event,
1663 &TableWalker::doLongDescriptor);
1664 if (delayed) {
1665 currState->delayed = true;
1666 }
1667 }
1668 return;
1669 default:
1670 panic("A new type in a 2 bit field?\n");
1671 }
1672}
1673
1674void
1675TableWalker::doL2Descriptor()
1676{
1677 if (currState->fault != NoFault) {
1678 return;
1679 }
1680
1681 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1682 currState->vaddr_tainted, currState->l2Desc.data);
1683 TlbEntry te;
1684
1685 if (currState->l2Desc.invalid()) {
1686 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1687 if (!currState->timing) {
1688 currState->tc = NULL;
1689 currState->req = NULL;
1690 }
1691 if (currState->isFetch)
1692 currState->fault = std::make_shared<PrefetchAbort>(
1693 currState->vaddr_tainted,
1694 ArmFault::TranslationLL + L2,
1695 isStage2,
1696 ArmFault::VmsaTran);
1697 else
1698 currState->fault = std::make_shared<DataAbort>(
1699 currState->vaddr_tainted, currState->l1Desc.domain(),
1700 currState->isWrite, ArmFault::TranslationLL + L2,
1701 isStage2,
1702 ArmFault::VmsaTran);
1703 return;
1704 }
1705
1706 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1707 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1708 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1709 */
1710 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1711 currState->sctlr.afe, currState->l2Desc.ap());
1712
1713 currState->fault = std::make_shared<DataAbort>(
1714 currState->vaddr_tainted,
1715 TlbEntry::DomainType::NoAccess, currState->isWrite,
1716 ArmFault::AccessFlagLL + L2, isStage2,
1717 ArmFault::VmsaTran);
1718 }
1719
1720 insertTableEntry(currState->l2Desc, false);
1721}
1722
1723void
1724TableWalker::doL1DescriptorWrapper()
1725{
1726 currState = stateQueues[L1].front();
1727 currState->delayed = false;
1728 // if there's a stage2 translation object we don't need it any more
1729 if (currState->stage2Tran) {
1730 delete currState->stage2Tran;
1731 currState->stage2Tran = NULL;
1732 }
1733
1734
1735 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1736 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1737
1738 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1739 doL1Descriptor();
1740
1741 stateQueues[L1].pop_front();
1742 // Check if fault was generated
1743 if (currState->fault != NoFault) {
1744 currState->transState->finish(currState->fault, currState->req,
1745 currState->tc, currState->mode);
1746 statWalksShortTerminatedAtLevel[0]++;
1747
1748 pending = false;
1749 nextWalk(currState->tc);
1750
1751 currState->req = NULL;
1752 currState->tc = NULL;
1753 currState->delayed = false;
1754 delete currState;
1755 }
1756 else if (!currState->delayed) {
1757 // delay is not set so there is no L2 to do
1758 // Don't finish the translation if a stage 2 look up is underway
1759 if (!currState->doingStage2) {
1760 statWalkServiceTime.sample(curTick() - currState->startTime);
1761 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1762 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1763 currState->transState, currState->mode);
1764 statWalksShortTerminatedAtLevel[0]++;
1765 }
1766
1767 pending = false;
1768 nextWalk(currState->tc);
1769
1770 currState->req = NULL;
1771 currState->tc = NULL;
1772 currState->delayed = false;
1773 delete currState;
1774 } else {
1775 // need to do L2 descriptor
1776 stateQueues[L2].push_back(currState);
1777 }
1778 currState = NULL;
1779}
1780
1781void
1782TableWalker::doL2DescriptorWrapper()
1783{
1784 currState = stateQueues[L2].front();
1785 assert(currState->delayed);
1786 // if there's a stage2 translation object we don't need it any more
1787 if (currState->stage2Tran) {
1788 delete currState->stage2Tran;
1789 currState->stage2Tran = NULL;
1790 }
1791
1792 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1793 currState->vaddr_tainted);
1794 doL2Descriptor();
1795
1796 // Check if fault was generated
1797 if (currState->fault != NoFault) {
1798 currState->transState->finish(currState->fault, currState->req,
1799 currState->tc, currState->mode);
1800 statWalksShortTerminatedAtLevel[1]++;
1801 }
1802 else {
1803 // Don't finish the translation if a stage 2 look up is underway
1804 if (!currState->doingStage2) {
1805 statWalkServiceTime.sample(curTick() - currState->startTime);
1806 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1807 currState->fault = tlb->translateTiming(currState->req,
1808 currState->tc, currState->transState, currState->mode);
1809 statWalksShortTerminatedAtLevel[1]++;
1810 }
1811 }
1812
1813
1814 stateQueues[L2].pop_front();
1815 pending = false;
1816 nextWalk(currState->tc);
1817
1818 currState->req = NULL;
1819 currState->tc = NULL;
1820 currState->delayed = false;
1821
1822 delete currState;
1823 currState = NULL;
1824}
1825
1826void
1827TableWalker::doL0LongDescriptorWrapper()
1828{
1829 doLongDescriptorWrapper(L0);
1830}
1831
1832void
1833TableWalker::doL1LongDescriptorWrapper()
1834{
1835 doLongDescriptorWrapper(L1);
1836}
1837
1838void
1839TableWalker::doL2LongDescriptorWrapper()
1840{
1841 doLongDescriptorWrapper(L2);
1842}
1843
1844void
1845TableWalker::doL3LongDescriptorWrapper()
1846{
1847 doLongDescriptorWrapper(L3);
1848}
1849
1850void
1851TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1852{
1853 currState = stateQueues[curr_lookup_level].front();
1854 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1855 currState->delayed = false;
1856
1857 // if there's a stage2 translation object we don't need it any more
1858 if (currState->stage2Tran) {
1859 delete currState->stage2Tran;
1860 currState->stage2Tran = NULL;
1861 }
1862
1863 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1864 currState->vaddr_tainted);
1865 doLongDescriptor();
1866
1867 stateQueues[curr_lookup_level].pop_front();
1868
1869 if (currState->fault != NoFault) {
1870 // A fault was generated
1871 currState->transState->finish(currState->fault, currState->req,
1872 currState->tc, currState->mode);
1873
1874 pending = false;
1875 nextWalk(currState->tc);
1876
1877 currState->req = NULL;
1878 currState->tc = NULL;
1879 currState->delayed = false;
1880 delete currState;
1881 } else if (!currState->delayed) {
1882 // No additional lookups required
1883 // Don't finish the translation if a stage 2 look up is underway
1884 if (!currState->doingStage2) {
1885 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1886 statWalkServiceTime.sample(curTick() - currState->startTime);
1887 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1888 currState->transState,
1889 currState->mode);
1890 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1891 }
1892
1893 pending = false;
1894 nextWalk(currState->tc);
1895
1896 currState->req = NULL;
1897 currState->tc = NULL;
1898 currState->delayed = false;
1899 delete currState;
1900 } else {
1901 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1902 panic("Max. number of lookups already reached in table walk\n");
1903 // Need to perform additional lookups
1904 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1905 }
1906 currState = NULL;
1907}
1908
1909
1910void
1911TableWalker::nextWalk(ThreadContext *tc)
1912{
1913 if (pendingQueue.size())
1914 schedule(doProcessEvent, clockEdge(Cycles(1)));
1915 else
1916 completeDrain();
1917}
1918
1919bool
1920TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1921 Request::Flags flags, int queueIndex, Event *event,
1922 void (TableWalker::*doDescriptor)())
1923{
1924 bool isTiming = currState->timing;
1925
1926 // do the requests for the page table descriptors have to go through the
1927 // second stage MMU
1928 if (currState->stage2Req) {
1929 Fault fault;
1930 flags = flags | TLB::MustBeOne;
1931
1932 if (isTiming) {
1933 Stage2MMU::Stage2Translation *tran = new
1934 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1935 currState->vaddr);
1936 currState->stage2Tran = tran;
1937 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1938 flags);
1939 fault = tran->fault;
1940 } else {
1941 fault = stage2Mmu->readDataUntimed(currState->tc,
1942 currState->vaddr, descAddr, data, numBytes, flags,
1943 currState->functional);
1944 }
1945
1946 if (fault != NoFault) {
1947 currState->fault = fault;
1948 }
1949 if (isTiming) {
1950 if (queueIndex >= 0) {
1951 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1952 stateQueues[queueIndex].size());
1953 stateQueues[queueIndex].push_back(currState);
1954 currState = NULL;
1955 }
1956 } else {
1957 (this->*doDescriptor)();
1958 }
1959 } else {
1960 if (isTiming) {
1961 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1962 currState->tc->getCpuPtr()->clockPeriod(),flags);
1963 if (queueIndex >= 0) {
1964 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1965 stateQueues[queueIndex].size());
1966 stateQueues[queueIndex].push_back(currState);
1967 currState = NULL;
1968 }
1969 } else if (!currState->functional) {
1970 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1971 currState->tc->getCpuPtr()->clockPeriod(), flags);
1972 (this->*doDescriptor)();
1973 } else {
1974 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1975 req->taskId(ContextSwitchTaskId::DMA);
1976 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1977 pkt->dataStatic(data);
1978 port->sendFunctional(pkt);
1979 (this->*doDescriptor)();
1980 delete req;
1981 delete pkt;
1982 }
1983 }
1984 return (isTiming);
1985}
1986
1987void
1988TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1989{
1990 TlbEntry te;
1991
1992 // Create and fill a new page table entry
1993 te.valid = true;
1994 te.longDescFormat = longDescriptor;
1995 te.isHyp = currState->isHyp;
1996 te.asid = currState->asid;
1997 te.vmid = currState->vmid;
1998 te.N = descriptor.offsetBits();
1999 te.vpn = currState->vaddr >> te.N;
2000 te.size = (1<<te.N) - 1;
2001 te.pfn = descriptor.pfn();
2002 te.domain = descriptor.domain();
2003 te.lookupLevel = descriptor.lookupLevel;
2004 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
2005 te.nstid = !currState->isSecure;
2006 te.xn = descriptor.xn();
2007 if (currState->aarch64)
2008 te.el = currState->el;
2009 else
2010 te.el = 1;
2011
2012 statPageSizes[pageSizeNtoStatBin(te.N)]++;
2013 statRequestOrigin[COMPLETED][currState->isFetch]++;
2014
2015 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2016 // as global
2017 te.global = descriptor.global(currState) || isStage2;
2018 if (longDescriptor) {
2019 LongDescriptor lDescriptor =
2020 dynamic_cast<LongDescriptor &>(descriptor);
2021
2022 te.xn |= currState->xnTable;
2023 te.pxn = currState->pxnTable || lDescriptor.pxn();
2024 if (isStage2) {
2025 // this is actually the HAP field, but its stored in the same bit
2026 // possitions as the AP field in a stage 1 translation.
2027 te.hap = lDescriptor.ap();
2028 } else {
2029 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2030 (currState->userTable && (descriptor.ap() & 0x1));
2031 }
2032 if (currState->aarch64)
2033 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2034 currState->longDesc.sh());
2035 else
2036 memAttrsLPAE(currState->tc, te, lDescriptor);
2037 } else {
2038 te.ap = descriptor.ap();
2039 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2040 descriptor.shareable());
2041 }
2042
2043 // Debug output
2044 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2045 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2046 te.N, te.pfn, te.size, te.global, te.valid);
2047 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2048 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2049 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2050 te.nonCacheable, te.ns);
2051 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2052 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2053 descriptor.getRawData());
2054
2055 // Insert the entry into the TLB
2056 tlb->insert(currState->vaddr, te);
2057 if (!currState->timing) {
2058 currState->tc = NULL;
2059 currState->req = NULL;
2060 }
2061}
2062
2063ArmISA::TableWalker *
2064ArmTableWalkerParams::create()
2065{
2066 return new ArmISA::TableWalker(this);
2067}
2068
2069LookupLevel
2070TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2071{
2072 switch (lookup_level_as_int) {
2073 case L1:
2074 return L1;
2075 case L2:
2076 return L2;
2077 case L3:
2078 return L3;
2079 default:
2080 panic("Invalid lookup level conversion");
2081 }
2082}
2083
2084/* this method keeps track of the table walker queue's residency, so
2085 * needs to be called whenever requests start and complete. */
2086void
2087TableWalker::pendingChange()
2088{
2089 unsigned n = pendingQueue.size();
2090 if ((currState != NULL) && (currState != pendingQueue.front())) {
2091 ++n;
2092 }
2093
2094 if (n != pendingReqs) {
2095 Tick now = curTick();
2096 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2097 pendingReqs = n;
2098 pendingChangeTick = now;
2099 }
2100}
2101
2102uint8_t
2103TableWalker::pageSizeNtoStatBin(uint8_t N)
2104{
2105 /* for statPageSizes */
2106 switch(N) {
2107 case 12: return 0; // 4K
2108 case 14: return 1; // 16K (using 16K granule in v8-64)
2109 case 16: return 2; // 64K
2110 case 20: return 3; // 1M
2111 case 21: return 4; // 2M-LPAE
2112 case 24: return 5; // 16M
2113 case 25: return 6; // 32M (using 16K granule in v8-64)
2114 case 29: return 7; // 512M (using 64K granule in v8-64)
2115 case 30: return 8; // 1G-LPAE
2116 default:
2117 panic("unknown page size");
2118 return 255;
2119 }
2120}
2121
2122void
2123TableWalker::regStats()
2124{
2125 statWalks
2126 .name(name() + ".walks")
2127 .desc("Table walker walks requested")
2128 ;
2129
2130 statWalksShortDescriptor
2131 .name(name() + ".walksShort")
2132 .desc("Table walker walks initiated with short descriptors")
2133 .flags(Stats::nozero)
2134 ;
2135
2136 statWalksLongDescriptor
2137 .name(name() + ".walksLong")
2138 .desc("Table walker walks initiated with long descriptors")
2139 .flags(Stats::nozero)
2140 ;
2141
2142 statWalksShortTerminatedAtLevel
2143 .init(2)
2144 .name(name() + ".walksShortTerminationLevel")
2145 .desc("Level at which table walker walks "
2146 "with short descriptors terminate")
2147 .flags(Stats::nozero)
2148 ;
2149 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2150 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2151
2152 statWalksLongTerminatedAtLevel
2153 .init(4)
2154 .name(name() + ".walksLongTerminationLevel")
2155 .desc("Level at which table walker walks "
2156 "with long descriptors terminate")
2157 .flags(Stats::nozero)
2158 ;
2159 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2160 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2161 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2162 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2163
2164 statSquashedBefore
2165 .name(name() + ".walksSquashedBefore")
2166 .desc("Table walks squashed before starting")
2167 .flags(Stats::nozero)
2168 ;
2169
2170 statSquashedAfter
2171 .name(name() + ".walksSquashedAfter")
2172 .desc("Table walks squashed after completion")
2173 .flags(Stats::nozero)
2174 ;
2175
2176 statWalkWaitTime
2177 .init(16)
2178 .name(name() + ".walkWaitTime")
2179 .desc("Table walker wait (enqueue to first request) latency")
2180 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2181 ;
2182
2183 statWalkServiceTime
2184 .init(16)
2185 .name(name() + ".walkCompletionTime")
2186 .desc("Table walker service (enqueue to completion) latency")
2187 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2188 ;
2189
2190 statPendingWalks
2191 .init(16)
2192 .name(name() + ".walksPending")
2193 .desc("Table walker pending requests distribution")
2194 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2195 ;
2196
2197 statPageSizes // see DDI 0487A D4-1661
2198 .init(9)
2199 .name(name() + ".walkPageSizes")
2200 .desc("Table walker page sizes translated")
2201 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2202 ;
2203 statPageSizes.subname(0, "4K");
2204 statPageSizes.subname(1, "16K");
2205 statPageSizes.subname(2, "64K");
2206 statPageSizes.subname(3, "1M");
2207 statPageSizes.subname(4, "2M");
2208 statPageSizes.subname(5, "16M");
2209 statPageSizes.subname(6, "32M");
2210 statPageSizes.subname(7, "512M");
2211 statPageSizes.subname(8, "1G");
2212
2213 statRequestOrigin
2214 .init(2,2) // Instruction/Data, requests/completed
2215 .name(name() + ".walkRequestOrigin")
2216 .desc("Table walker requests started/completed, data/inst")
2217 .flags(Stats::total)
2218 ;
2219 statRequestOrigin.subname(0,"Requested");
2220 statRequestOrigin.subname(1,"Completed");
2221 statRequestOrigin.ysubname(0,"Data");
2222 statRequestOrigin.ysubname(1,"Inst");
2223}
|