54#include "sim/system.hh"
55
56using namespace ArmISA;
57
58TableWalker::TableWalker(const Params *p)
59 : MemObject(p), drainManager(NULL),
60 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId),
61 isStage2(p->is_stage2), tlb(NULL),
62 currState(NULL), pending(false),
63 numSquashable(p->num_squash_per_cycle),
64 pendingReqs(0),
65 pendingChangeTick(curTick()),
66 doL1DescEvent(this), doL2DescEvent(this),
67 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
68 doL3LongDescEvent(this),
69 doProcessEvent(this)
70{
71 sctlr = 0;
72
73 // Cache system-level properties
74 if (FullSystem) {
75 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys);
76 assert(armSys);
77 haveSecurity = armSys->haveSecurity();
78 _haveLPAE = armSys->haveLPAE();
79 _haveVirtualization = armSys->haveVirtualization();
80 physAddrRange = armSys->physAddrRange();
81 _haveLargeAsid64 = armSys->haveLargeAsid64();
82 } else {
83 haveSecurity = _haveLPAE = _haveVirtualization = false;
84 _haveLargeAsid64 = false;
85 physAddrRange = 32;
86 }
87
88}
89
90TableWalker::~TableWalker()
91{
92 ;
93}
94
95void
96TableWalker::setMMU(Stage2MMU *m, MasterID master_id)
97{
98 stage2Mmu = m;
99 port = &m->getPort();
100 masterId = master_id;
101}
102
103void
104TableWalker::init()
105{
106 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n");
107 fatal_if(!port, "Table walker must have a valid port\n");
108 fatal_if(!tlb, "Table walker must have a valid TLB\n");
109}
110
111BaseMasterPort&
112TableWalker::getMasterPort(const std::string &if_name, PortID idx)
113{
114 if (if_name == "port") {
115 if (!isStage2) {
116 return *port;
117 } else {
118 fatal("Cannot access table walker port through stage-two walker\n");
119 }
120 }
121 return MemObject::getMasterPort(if_name, idx);
122}
123
124TableWalker::WalkerState::WalkerState() :
125 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
126 asid(0), vmid(0), isHyp(false), transState(nullptr),
127 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
128 secureLookup(false), rwTable(false), userTable(false), xnTable(false),
129 pxnTable(false), stage2Req(false), doingStage2(false),
130 stage2Tran(nullptr), timing(false), functional(false),
131 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
132 delayed(false), tableWalker(nullptr)
133{
134}
135
136void
137TableWalker::completeDrain()
138{
139 if (drainManager && stateQueues[L1].empty() && stateQueues[L2].empty() &&
140 pendingQueue.empty()) {
141 setDrainState(Drainable::Drained);
142 DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
143 drainManager->signalDrainDone();
144 drainManager = NULL;
145 }
146}
147
148unsigned int
149TableWalker::drain(DrainManager *dm)
150{
151 bool state_queues_not_empty = false;
152
153 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
154 if (!stateQueues[i].empty()) {
155 state_queues_not_empty = true;
156 break;
157 }
158 }
159
160 if (state_queues_not_empty || pendingQueue.size()) {
161 drainManager = dm;
162 setDrainState(Drainable::Draining);
163 DPRINTF(Drain, "TableWalker not drained\n");
164
165 // return port drain count plus the table walker itself needs to drain
166 return 1;
167 } else {
168 setDrainState(Drainable::Drained);
169 DPRINTF(Drain, "TableWalker free, no need to drain\n");
170
171 // table walker is drained, but its ports may still need to be drained
172 return 0;
173 }
174}
175
176void
177TableWalker::drainResume()
178{
179 Drainable::drainResume();
180 if (params()->sys->isTimingMode() && currState) {
181 delete currState;
182 currState = NULL;
183 pendingChange();
184 }
185}
186
187Fault
188TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
189 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
190 TLB::Translation *_trans, bool _timing, bool _functional,
191 bool secure, TLB::ArmTranslationType tranType)
192{
193 assert(!(_functional && _timing));
194 ++statWalks;
195
196 WalkerState *savedCurrState = NULL;
197
198 if (!currState && !_functional) {
199 // For atomic mode, a new WalkerState instance should be only created
200 // once per TLB. For timing mode, a new instance is generated for every
201 // TLB miss.
202 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
203
204 currState = new WalkerState();
205 currState->tableWalker = this;
206 } else if (_functional) {
207 // If we are mixing functional mode with timing (or even
208 // atomic), we need to to be careful and clean up after
209 // ourselves to not risk getting into an inconsistent state.
210 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
211 savedCurrState = currState;
212 currState = new WalkerState();
213 currState->tableWalker = this;
214 } else if (_timing) {
215 // This is a translation that was completed and then faulted again
216 // because some underlying parameters that affect the translation
217 // changed out from under us (e.g. asid). It will either be a
218 // misprediction, in which case nothing will happen or we'll use
219 // this fault to re-execute the faulting instruction which should clean
220 // up everything.
221 if (currState->vaddr_tainted == _req->getVaddr()) {
222 ++statSquashedBefore;
223 return std::make_shared<ReExec>();
224 }
225 }
226 pendingChange();
227
228 currState->startTime = curTick();
229 currState->tc = _tc;
230 currState->aarch64 = opModeIs64(currOpMode(_tc));
231 currState->el = currEL(_tc);
232 currState->transState = _trans;
233 currState->req = _req;
234 currState->fault = NoFault;
235 currState->asid = _asid;
236 currState->vmid = _vmid;
237 currState->isHyp = _isHyp;
238 currState->timing = _timing;
239 currState->functional = _functional;
240 currState->mode = _mode;
241 currState->tranType = tranType;
242 currState->isSecure = secure;
243 currState->physAddrRange = physAddrRange;
244
245 /** @todo These should be cached or grabbed from cached copies in
246 the TLB, all these miscreg reads are expensive */
247 currState->vaddr_tainted = currState->req->getVaddr();
248 if (currState->aarch64)
249 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
250 currState->tc, currState->el);
251 else
252 currState->vaddr = currState->vaddr_tainted;
253
254 if (currState->aarch64) {
255 switch (currState->el) {
256 case EL0:
257 case EL1:
258 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
259 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
260 break;
261 // @todo: uncomment this to enable Virtualization
262 // case EL2:
263 // assert(haveVirtualization);
264 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
265 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
266 // break;
267 case EL3:
268 assert(haveSecurity);
269 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
270 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
271 break;
272 default:
273 panic("Invalid exception level");
274 break;
275 }
276 } else {
277 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
278 MISCREG_SCTLR, currState->tc, !currState->isSecure));
279 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
280 MISCREG_TTBCR, currState->tc, !currState->isSecure));
281 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
282 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
283 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
284 }
285 sctlr = currState->sctlr;
286
287 currState->isFetch = (currState->mode == TLB::Execute);
288 currState->isWrite = (currState->mode == TLB::Write);
289
290 statRequestOrigin[REQUESTED][currState->isFetch]++;
291
292 // We only do a second stage of translation if we're not secure, or in
293 // hyp mode, the second stage MMU is enabled, and this table walker
294 // instance is the first stage.
295 currState->doingStage2 = false;
296 // @todo: for now disable this in AArch64 (HCR is not set)
297 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
298 !isStage2 && !currState->isSecure && !currState->isHyp;
299
300 bool long_desc_format = currState->aarch64 ||
301 (_haveLPAE && currState->ttbcr.eae) ||
302 _isHyp || isStage2;
303
304 if (long_desc_format) {
305 // Helper variables used for hierarchical permissions
306 currState->secureLookup = currState->isSecure;
307 currState->rwTable = true;
308 currState->userTable = true;
309 currState->xnTable = false;
310 currState->pxnTable = false;
311
312 ++statWalksLongDescriptor;
313 } else {
314 ++statWalksShortDescriptor;
315 }
316
317 if (!currState->timing) {
318 Fault fault = NoFault;
319 if (currState->aarch64)
320 fault = processWalkAArch64();
321 else if (long_desc_format)
322 fault = processWalkLPAE();
323 else
324 fault = processWalk();
325
326 // If this was a functional non-timing access restore state to
327 // how we found it.
328 if (currState->functional) {
329 delete currState;
330 currState = savedCurrState;
331 }
332 return fault;
333 }
334
335 if (pending || pendingQueue.size()) {
336 pendingQueue.push_back(currState);
337 currState = NULL;
338 pendingChange();
339 } else {
340 pending = true;
341 pendingChange();
342 if (currState->aarch64)
343 return processWalkAArch64();
344 else if (long_desc_format)
345 return processWalkLPAE();
346 else
347 return processWalk();
348 }
349
350 return NoFault;
351}
352
353void
354TableWalker::processWalkWrapper()
355{
356 assert(!currState);
357 assert(pendingQueue.size());
358 pendingChange();
359 currState = pendingQueue.front();
360
361 ExceptionLevel target_el = EL0;
362 if (currState->aarch64)
363 target_el = currEL(currState->tc);
364 else
365 target_el = EL1;
366
367 // Check if a previous walk filled this request already
368 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
369 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
370 currState->vmid, currState->isHyp, currState->isSecure, true, false,
371 target_el);
372
373 // Check if we still need to have a walk for this request. If the requesting
374 // instruction has been squashed, or a previous walk has filled the TLB with
375 // a match, we just want to get rid of the walk. The latter could happen
376 // when there are multiple outstanding misses to a single page and a
377 // previous request has been successfully translated.
378 if (!currState->transState->squashed() && !te) {
379 // We've got a valid request, lets process it
380 pending = true;
381 pendingQueue.pop_front();
382 // Keep currState in case one of the processWalk... calls NULLs it
383 WalkerState *curr_state_copy = currState;
384 Fault f;
385 if (currState->aarch64)
386 f = processWalkAArch64();
387 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
388 f = processWalkLPAE();
389 else
390 f = processWalk();
391
392 if (f != NoFault) {
393 curr_state_copy->transState->finish(f, curr_state_copy->req,
394 curr_state_copy->tc, curr_state_copy->mode);
395
396 delete curr_state_copy;
397 }
398 return;
399 }
400
401
402 // If the instruction that we were translating for has been
403 // squashed we shouldn't bother.
404 unsigned num_squashed = 0;
405 ThreadContext *tc = currState->tc;
406 while ((num_squashed < numSquashable) && currState &&
407 (currState->transState->squashed() || te)) {
408 pendingQueue.pop_front();
409 num_squashed++;
410 statSquashedBefore++;
411
412 DPRINTF(TLB, "Squashing table walk for address %#x\n",
413 currState->vaddr_tainted);
414
415 if (currState->transState->squashed()) {
416 // finish the translation which will delete the translation object
417 currState->transState->finish(
418 std::make_shared<UnimpFault>("Squashed Inst"),
419 currState->req, currState->tc, currState->mode);
420 } else {
421 // translate the request now that we know it will work
422 statWalkServiceTime.sample(curTick() - currState->startTime);
423 tlb->translateTiming(currState->req, currState->tc,
424 currState->transState, currState->mode);
425
426 }
427
428 // delete the current request
429 delete currState;
430
431 // peak at the next one
432 if (pendingQueue.size()) {
433 currState = pendingQueue.front();
434 te = tlb->lookup(currState->vaddr, currState->asid,
435 currState->vmid, currState->isHyp, currState->isSecure, true,
436 false, target_el);
437 } else {
438 // Terminate the loop, nothing more to do
439 currState = NULL;
440 }
441 }
442 pendingChange();
443
444 // if we still have pending translations, schedule more work
445 nextWalk(tc);
446 currState = NULL;
447}
448
449Fault
450TableWalker::processWalk()
451{
452 Addr ttbr = 0;
453
454 // If translation isn't enabled, we shouldn't be here
455 assert(currState->sctlr.m || isStage2);
456
457 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
458 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
459 32 - currState->ttbcr.n));
460
461 statWalkWaitTime.sample(curTick() - currState->startTime);
462
463 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
464 32 - currState->ttbcr.n)) {
465 DPRINTF(TLB, " - Selecting TTBR0\n");
466 // Check if table walk is allowed when Security Extensions are enabled
467 if (haveSecurity && currState->ttbcr.pd0) {
468 if (currState->isFetch)
469 return std::make_shared<PrefetchAbort>(
470 currState->vaddr_tainted,
471 ArmFault::TranslationLL + L1,
472 isStage2,
473 ArmFault::VmsaTran);
474 else
475 return std::make_shared<DataAbort>(
476 currState->vaddr_tainted,
477 TlbEntry::DomainType::NoAccess, currState->isWrite,
478 ArmFault::TranslationLL + L1, isStage2,
479 ArmFault::VmsaTran);
480 }
481 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
482 MISCREG_TTBR0, currState->tc, !currState->isSecure));
483 } else {
484 DPRINTF(TLB, " - Selecting TTBR1\n");
485 // Check if table walk is allowed when Security Extensions are enabled
486 if (haveSecurity && currState->ttbcr.pd1) {
487 if (currState->isFetch)
488 return std::make_shared<PrefetchAbort>(
489 currState->vaddr_tainted,
490 ArmFault::TranslationLL + L1,
491 isStage2,
492 ArmFault::VmsaTran);
493 else
494 return std::make_shared<DataAbort>(
495 currState->vaddr_tainted,
496 TlbEntry::DomainType::NoAccess, currState->isWrite,
497 ArmFault::TranslationLL + L1, isStage2,
498 ArmFault::VmsaTran);
499 }
500 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
501 MISCREG_TTBR1, currState->tc, !currState->isSecure));
502 currState->ttbcr.n = 0;
503 }
504
505 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
506 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
507 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
508 currState->isSecure ? "s" : "ns");
509
510 // Trickbox address check
511 Fault f;
512 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
513 currState->vaddr, sizeof(uint32_t), currState->isFetch,
514 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
515 if (f) {
516 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
517 if (currState->timing) {
518 pending = false;
519 nextWalk(currState->tc);
520 currState = NULL;
521 } else {
522 currState->tc = NULL;
523 currState->req = NULL;
524 }
525 return f;
526 }
527
528 Request::Flags flag = Request::PT_WALK;
529 if (currState->sctlr.c == 0) {
530 flag.set(Request::UNCACHEABLE);
531 }
532
533 bool delayed;
534 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
535 sizeof(uint32_t), flag, L1, &doL1DescEvent,
536 &TableWalker::doL1Descriptor);
537 if (!delayed) {
538 f = currState->fault;
539 }
540
541 return f;
542}
543
544Fault
545TableWalker::processWalkLPAE()
546{
547 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
548 int tsz, n;
549 LookupLevel start_lookup_level = L1;
550
551 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
552 currState->vaddr_tainted, currState->ttbcr);
553
554 statWalkWaitTime.sample(curTick() - currState->startTime);
555
556 Request::Flags flag = Request::PT_WALK;
557 if (currState->isSecure)
558 flag.set(Request::SECURE);
559
560 // work out which base address register to use, if in hyp mode we always
561 // use HTTBR
562 if (isStage2) {
563 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
564 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
565 tsz = sext<4>(currState->vtcr.t0sz);
566 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
567 } else if (currState->isHyp) {
568 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
569 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
570 tsz = currState->htcr.t0sz;
571 } else {
572 assert(_haveLPAE && currState->ttbcr.eae);
573
574 // Determine boundaries of TTBR0/1 regions
575 if (currState->ttbcr.t0sz)
576 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
577 else if (currState->ttbcr.t1sz)
578 ttbr0_max = (1ULL << 32) -
579 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
580 else
581 ttbr0_max = (1ULL << 32) - 1;
582 if (currState->ttbcr.t1sz)
583 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
584 else
585 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
586
587 // The following code snippet selects the appropriate translation table base
588 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
589 // depending on the address range supported by the translation table (ARM
590 // ARM issue C B3.6.4)
591 if (currState->vaddr <= ttbr0_max) {
592 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
593 // Check if table walk is allowed
594 if (currState->ttbcr.epd0) {
595 if (currState->isFetch)
596 return std::make_shared<PrefetchAbort>(
597 currState->vaddr_tainted,
598 ArmFault::TranslationLL + L1,
599 isStage2,
600 ArmFault::LpaeTran);
601 else
602 return std::make_shared<DataAbort>(
603 currState->vaddr_tainted,
604 TlbEntry::DomainType::NoAccess,
605 currState->isWrite,
606 ArmFault::TranslationLL + L1,
607 isStage2,
608 ArmFault::LpaeTran);
609 }
610 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
611 MISCREG_TTBR0, currState->tc, !currState->isSecure));
612 tsz = currState->ttbcr.t0sz;
613 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
614 start_lookup_level = L2;
615 } else if (currState->vaddr >= ttbr1_min) {
616 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
617 // Check if table walk is allowed
618 if (currState->ttbcr.epd1) {
619 if (currState->isFetch)
620 return std::make_shared<PrefetchAbort>(
621 currState->vaddr_tainted,
622 ArmFault::TranslationLL + L1,
623 isStage2,
624 ArmFault::LpaeTran);
625 else
626 return std::make_shared<DataAbort>(
627 currState->vaddr_tainted,
628 TlbEntry::DomainType::NoAccess,
629 currState->isWrite,
630 ArmFault::TranslationLL + L1,
631 isStage2,
632 ArmFault::LpaeTran);
633 }
634 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
635 MISCREG_TTBR1, currState->tc, !currState->isSecure));
636 tsz = currState->ttbcr.t1sz;
637 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
638 start_lookup_level = L2;
639 } else {
640 // Out of boundaries -> translation fault
641 if (currState->isFetch)
642 return std::make_shared<PrefetchAbort>(
643 currState->vaddr_tainted,
644 ArmFault::TranslationLL + L1,
645 isStage2,
646 ArmFault::LpaeTran);
647 else
648 return std::make_shared<DataAbort>(
649 currState->vaddr_tainted,
650 TlbEntry::DomainType::NoAccess,
651 currState->isWrite, ArmFault::TranslationLL + L1,
652 isStage2, ArmFault::LpaeTran);
653 }
654
655 }
656
657 // Perform lookup (ARM ARM issue C B3.6.6)
658 if (start_lookup_level == L1) {
659 n = 5 - tsz;
660 desc_addr = mbits(ttbr, 39, n) |
661 (bits(currState->vaddr, n + 26, 30) << 3);
662 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
663 desc_addr, currState->isSecure ? "s" : "ns");
664 } else {
665 // Skip first-level lookup
666 n = (tsz >= 2 ? 14 - tsz : 12);
667 desc_addr = mbits(ttbr, 39, n) |
668 (bits(currState->vaddr, n + 17, 21) << 3);
669 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
670 desc_addr, currState->isSecure ? "s" : "ns");
671 }
672
673 // Trickbox address check
674 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
675 currState->vaddr, sizeof(uint64_t), currState->isFetch,
676 currState->isWrite, TlbEntry::DomainType::NoAccess,
677 start_lookup_level);
678 if (f) {
679 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
680 if (currState->timing) {
681 pending = false;
682 nextWalk(currState->tc);
683 currState = NULL;
684 } else {
685 currState->tc = NULL;
686 currState->req = NULL;
687 }
688 return f;
689 }
690
691 if (currState->sctlr.c == 0) {
692 flag.set(Request::UNCACHEABLE);
693 }
694
695 if (currState->isSecure)
696 flag.set(Request::SECURE);
697
698 currState->longDesc.lookupLevel = start_lookup_level;
699 currState->longDesc.aarch64 = false;
700 currState->longDesc.grainSize = Grain4KB;
701
702 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
703 : (Event *) &doL2LongDescEvent;
704
705 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
706 sizeof(uint64_t), flag, start_lookup_level,
707 event, &TableWalker::doLongDescriptor);
708 if (!delayed) {
709 f = currState->fault;
710 }
711
712 return f;
713}
714
715unsigned
716TableWalker::adjustTableSizeAArch64(unsigned tsz)
717{
718 if (tsz < 25)
719 return 25;
720 if (tsz > 48)
721 return 48;
722 return tsz;
723}
724
725bool
726TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
727{
728 return (currPhysAddrRange != MaxPhysAddrRange &&
729 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
730}
731
732Fault
733TableWalker::processWalkAArch64()
734{
735 assert(currState->aarch64);
736
737 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
738 currState->vaddr_tainted, currState->tcr);
739
740 static const GrainSize GrainMapDefault[] =
741 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
742 static const GrainSize GrainMap_EL1_tg1[] =
743 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
744
745 statWalkWaitTime.sample(curTick() - currState->startTime);
746
747 // Determine TTBR, table size, granule size and phys. address range
748 Addr ttbr = 0;
749 int tsz = 0, ps = 0;
750 GrainSize tg = Grain4KB; // grain size computed from tg* field
751 bool fault = false;
752 switch (currState->el) {
753 case EL0:
754 case EL1:
755 switch (bits(currState->vaddr, 63,48)) {
756 case 0:
757 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
758 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
759 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
760 tg = GrainMapDefault[currState->tcr.tg0];
761 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
762 currState->tcr.epd0)
763 fault = true;
764 break;
765 case 0xffff:
766 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
767 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
768 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
769 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
770 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
771 currState->tcr.epd1)
772 fault = true;
773 break;
774 default:
775 // top two bytes must be all 0s or all 1s, else invalid addr
776 fault = true;
777 }
778 ps = currState->tcr.ips;
779 break;
780 case EL2:
781 case EL3:
782 switch(bits(currState->vaddr, 63,48)) {
783 case 0:
784 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
785 if (currState->el == EL2)
786 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
787 else
788 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
789 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
790 tg = GrainMapDefault[currState->tcr.tg0];
791 break;
792 default:
793 // invalid addr if top two bytes are not all 0s
794 fault = true;
795 }
796 ps = currState->tcr.ips;
797 break;
798 }
799
800 if (fault) {
801 Fault f;
802 if (currState->isFetch)
803 f = std::make_shared<PrefetchAbort>(
804 currState->vaddr_tainted,
805 ArmFault::TranslationLL + L0, isStage2,
806 ArmFault::LpaeTran);
807 else
808 f = std::make_shared<DataAbort>(
809 currState->vaddr_tainted,
810 TlbEntry::DomainType::NoAccess,
811 currState->isWrite,
812 ArmFault::TranslationLL + L0,
813 isStage2, ArmFault::LpaeTran);
814
815 if (currState->timing) {
816 pending = false;
817 nextWalk(currState->tc);
818 currState = NULL;
819 } else {
820 currState->tc = NULL;
821 currState->req = NULL;
822 }
823 return f;
824
825 }
826
827 if (tg == ReservedGrain) {
828 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
829 "DEFINED behavior takes this to mean 4KB granules\n");
830 tg = Grain4KB;
831 }
832
833 int stride = tg - 3;
834 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
835
836 // Determine starting lookup level
837 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
838 // in ARM DDI 0487A. These table values correspond to the cascading tests
839 // to compute the lookup level and are of the form
840 // (grain_size + N*stride), for N = {1, 2, 3}.
841 // A value of 64 will never succeed and a value of 0 will always succeed.
842 {
843 struct GrainMap {
844 GrainSize grain_size;
845 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
846 };
847 static const GrainMap GM[] = {
848 { Grain4KB, { 39, 30, 0, 0 } },
849 { Grain16KB, { 47, 36, 25, 0 } },
850 { Grain64KB, { 64, 42, 29, 0 } }
851 };
852
853 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
854
855 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
856 if (tg == GM[i].grain_size) {
857 lookup = GM[i].lookup_level_cutoff;
858 break;
859 }
860 }
861 assert(lookup);
862
863 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
864 if (tsz > lookup[L]) {
865 start_lookup_level = (LookupLevel) L;
866 break;
867 }
868 }
869 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
870 "Table walker couldn't find lookup level\n");
871 }
872
873 // Determine table base address
874 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
875 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
876
877 // Determine physical address size and raise an Address Size Fault if
878 // necessary
879 int pa_range = decodePhysAddrRange64(ps);
880 // Clamp to lower limit
881 if (pa_range > physAddrRange)
882 currState->physAddrRange = physAddrRange;
883 else
884 currState->physAddrRange = pa_range;
885 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
886 DPRINTF(TLB, "Address size fault before any lookup\n");
887 Fault f;
888 if (currState->isFetch)
889 f = std::make_shared<PrefetchAbort>(
890 currState->vaddr_tainted,
891 ArmFault::AddressSizeLL + start_lookup_level,
892 isStage2,
893 ArmFault::LpaeTran);
894 else
895 f = std::make_shared<DataAbort>(
896 currState->vaddr_tainted,
897 TlbEntry::DomainType::NoAccess,
898 currState->isWrite,
899 ArmFault::AddressSizeLL + start_lookup_level,
900 isStage2,
901 ArmFault::LpaeTran);
902
903
904 if (currState->timing) {
905 pending = false;
906 nextWalk(currState->tc);
907 currState = NULL;
908 } else {
909 currState->tc = NULL;
910 currState->req = NULL;
911 }
912 return f;
913
914 }
915
916 // Determine descriptor address
917 Addr desc_addr = base_addr |
918 (bits(currState->vaddr, tsz - 1,
919 stride * (3 - start_lookup_level) + tg) << 3);
920
921 // Trickbox address check
922 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
923 currState->vaddr, sizeof(uint64_t), currState->isFetch,
924 currState->isWrite, TlbEntry::DomainType::NoAccess,
925 start_lookup_level);
926 if (f) {
927 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
928 if (currState->timing) {
929 pending = false;
930 nextWalk(currState->tc);
931 currState = NULL;
932 } else {
933 currState->tc = NULL;
934 currState->req = NULL;
935 }
936 return f;
937 }
938
939 Request::Flags flag = Request::PT_WALK;
940 if (currState->sctlr.c == 0) {
941 flag.set(Request::UNCACHEABLE);
942 }
943
944 currState->longDesc.lookupLevel = start_lookup_level;
945 currState->longDesc.aarch64 = true;
946 currState->longDesc.grainSize = tg;
947
948 if (currState->timing) {
949 Event *event;
950 switch (start_lookup_level) {
951 case L0:
952 event = (Event *) &doL0LongDescEvent;
953 break;
954 case L1:
955 event = (Event *) &doL1LongDescEvent;
956 break;
957 case L2:
958 event = (Event *) &doL2LongDescEvent;
959 break;
960 case L3:
961 event = (Event *) &doL3LongDescEvent;
962 break;
963 default:
964 panic("Invalid table lookup level");
965 break;
966 }
967 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
968 event, (uint8_t*) &currState->longDesc.data,
969 currState->tc->getCpuPtr()->clockPeriod(), flag);
970 DPRINTF(TLBVerbose,
971 "Adding to walker fifo: queue size before adding: %d\n",
972 stateQueues[start_lookup_level].size());
973 stateQueues[start_lookup_level].push_back(currState);
974 currState = NULL;
975 } else if (!currState->functional) {
976 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
977 NULL, (uint8_t*) &currState->longDesc.data,
978 currState->tc->getCpuPtr()->clockPeriod(), flag);
979 doLongDescriptor();
980 f = currState->fault;
981 } else {
982 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
983 masterId);
984 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
985 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
986 port->sendFunctional(pkt);
987 doLongDescriptor();
988 delete req;
989 delete pkt;
990 f = currState->fault;
991 }
992
993 return f;
994}
995
996void
997TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
998 uint8_t texcb, bool s)
999{
1000 // Note: tc and sctlr local variables are hiding tc and sctrl class
1001 // variables
1002 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1003 te.shareable = false; // default value
1004 te.nonCacheable = false;
1005 te.outerShareable = false;
1006 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1007 switch(texcb) {
1008 case 0: // Stongly-ordered
1009 te.nonCacheable = true;
1010 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1011 te.shareable = true;
1012 te.innerAttrs = 1;
1013 te.outerAttrs = 0;
1014 break;
1015 case 1: // Shareable Device
1016 te.nonCacheable = true;
1017 te.mtype = TlbEntry::MemoryType::Device;
1018 te.shareable = true;
1019 te.innerAttrs = 3;
1020 te.outerAttrs = 0;
1021 break;
1022 case 2: // Outer and Inner Write-Through, no Write-Allocate
1023 te.mtype = TlbEntry::MemoryType::Normal;
1024 te.shareable = s;
1025 te.innerAttrs = 6;
1026 te.outerAttrs = bits(texcb, 1, 0);
1027 break;
1028 case 3: // Outer and Inner Write-Back, no Write-Allocate
1029 te.mtype = TlbEntry::MemoryType::Normal;
1030 te.shareable = s;
1031 te.innerAttrs = 7;
1032 te.outerAttrs = bits(texcb, 1, 0);
1033 break;
1034 case 4: // Outer and Inner Non-cacheable
1035 te.nonCacheable = true;
1036 te.mtype = TlbEntry::MemoryType::Normal;
1037 te.shareable = s;
1038 te.innerAttrs = 0;
1039 te.outerAttrs = bits(texcb, 1, 0);
1040 break;
1041 case 5: // Reserved
1042 panic("Reserved texcb value!\n");
1043 break;
1044 case 6: // Implementation Defined
1045 panic("Implementation-defined texcb value!\n");
1046 break;
1047 case 7: // Outer and Inner Write-Back, Write-Allocate
1048 te.mtype = TlbEntry::MemoryType::Normal;
1049 te.shareable = s;
1050 te.innerAttrs = 5;
1051 te.outerAttrs = 1;
1052 break;
1053 case 8: // Non-shareable Device
1054 te.nonCacheable = true;
1055 te.mtype = TlbEntry::MemoryType::Device;
1056 te.shareable = false;
1057 te.innerAttrs = 3;
1058 te.outerAttrs = 0;
1059 break;
1060 case 9 ... 15: // Reserved
1061 panic("Reserved texcb value!\n");
1062 break;
1063 case 16 ... 31: // Cacheable Memory
1064 te.mtype = TlbEntry::MemoryType::Normal;
1065 te.shareable = s;
1066 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1067 te.nonCacheable = true;
1068 te.innerAttrs = bits(texcb, 1, 0);
1069 te.outerAttrs = bits(texcb, 3, 2);
1070 break;
1071 default:
1072 panic("More than 32 states for 5 bits?\n");
1073 }
1074 } else {
1075 assert(tc);
1076 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1077 currState->tc, !currState->isSecure));
1078 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1079 currState->tc, !currState->isSecure));
1080 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1081 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1082 switch(bits(texcb, 2,0)) {
1083 case 0:
1084 curr_tr = prrr.tr0;
1085 curr_ir = nmrr.ir0;
1086 curr_or = nmrr.or0;
1087 te.outerShareable = (prrr.nos0 == 0);
1088 break;
1089 case 1:
1090 curr_tr = prrr.tr1;
1091 curr_ir = nmrr.ir1;
1092 curr_or = nmrr.or1;
1093 te.outerShareable = (prrr.nos1 == 0);
1094 break;
1095 case 2:
1096 curr_tr = prrr.tr2;
1097 curr_ir = nmrr.ir2;
1098 curr_or = nmrr.or2;
1099 te.outerShareable = (prrr.nos2 == 0);
1100 break;
1101 case 3:
1102 curr_tr = prrr.tr3;
1103 curr_ir = nmrr.ir3;
1104 curr_or = nmrr.or3;
1105 te.outerShareable = (prrr.nos3 == 0);
1106 break;
1107 case 4:
1108 curr_tr = prrr.tr4;
1109 curr_ir = nmrr.ir4;
1110 curr_or = nmrr.or4;
1111 te.outerShareable = (prrr.nos4 == 0);
1112 break;
1113 case 5:
1114 curr_tr = prrr.tr5;
1115 curr_ir = nmrr.ir5;
1116 curr_or = nmrr.or5;
1117 te.outerShareable = (prrr.nos5 == 0);
1118 break;
1119 case 6:
1120 panic("Imp defined type\n");
1121 case 7:
1122 curr_tr = prrr.tr7;
1123 curr_ir = nmrr.ir7;
1124 curr_or = nmrr.or7;
1125 te.outerShareable = (prrr.nos7 == 0);
1126 break;
1127 }
1128
1129 switch(curr_tr) {
1130 case 0:
1131 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1132 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1133 te.nonCacheable = true;
1134 te.innerAttrs = 1;
1135 te.outerAttrs = 0;
1136 te.shareable = true;
1137 break;
1138 case 1:
1139 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1140 prrr.ds1, prrr.ds0, s);
1141 te.mtype = TlbEntry::MemoryType::Device;
1142 te.nonCacheable = true;
1143 te.innerAttrs = 3;
1144 te.outerAttrs = 0;
1145 if (prrr.ds1 && s)
1146 te.shareable = true;
1147 if (prrr.ds0 && !s)
1148 te.shareable = true;
1149 break;
1150 case 2:
1151 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1152 prrr.ns1, prrr.ns0, s);
1153 te.mtype = TlbEntry::MemoryType::Normal;
1154 if (prrr.ns1 && s)
1155 te.shareable = true;
1156 if (prrr.ns0 && !s)
1157 te.shareable = true;
1158 break;
1159 case 3:
1160 panic("Reserved type");
1161 }
1162
1163 if (te.mtype == TlbEntry::MemoryType::Normal){
1164 switch(curr_ir) {
1165 case 0:
1166 te.nonCacheable = true;
1167 te.innerAttrs = 0;
1168 break;
1169 case 1:
1170 te.innerAttrs = 5;
1171 break;
1172 case 2:
1173 te.innerAttrs = 6;
1174 break;
1175 case 3:
1176 te.innerAttrs = 7;
1177 break;
1178 }
1179
1180 switch(curr_or) {
1181 case 0:
1182 te.nonCacheable = true;
1183 te.outerAttrs = 0;
1184 break;
1185 case 1:
1186 te.outerAttrs = 1;
1187 break;
1188 case 2:
1189 te.outerAttrs = 2;
1190 break;
1191 case 3:
1192 te.outerAttrs = 3;
1193 break;
1194 }
1195 }
1196 }
1197 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1198 "outerAttrs: %d\n",
1199 te.shareable, te.innerAttrs, te.outerAttrs);
1200 te.setAttributes(false);
1201}
1202
1203void
1204TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1205 LongDescriptor &lDescriptor)
1206{
1207 assert(_haveLPAE);
1208
1209 uint8_t attr;
1210 uint8_t sh = lDescriptor.sh();
1211 // Different format and source of attributes if this is a stage 2
1212 // translation
1213 if (isStage2) {
1214 attr = lDescriptor.memAttr();
1215 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1216 uint8_t attr_1_0 = attr & 0x3;
1217
1218 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1219
1220 if (attr_3_2 == 0) {
1221 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1222 : TlbEntry::MemoryType::Device;
1223 te.outerAttrs = 0;
1224 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1225 te.nonCacheable = true;
1226 } else {
1227 te.mtype = TlbEntry::MemoryType::Normal;
1228 te.outerAttrs = attr_3_2 == 1 ? 0 :
1229 attr_3_2 == 2 ? 2 : 1;
1230 te.innerAttrs = attr_1_0 == 1 ? 0 :
1231 attr_1_0 == 2 ? 6 : 5;
1232 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1233 }
1234 } else {
1235 uint8_t attrIndx = lDescriptor.attrIndx();
1236
1237 // LPAE always uses remapping of memory attributes, irrespective of the
1238 // value of SCTLR.TRE
1239 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1240 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1241 !currState->isSecure);
1242 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1243 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1244 uint8_t attr_7_4 = bits(attr, 7, 4);
1245 uint8_t attr_3_0 = bits(attr, 3, 0);
1246 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1247
1248 // Note: the memory subsystem only cares about the 'cacheable' memory
1249 // attribute. The other attributes are only used to fill the PAR register
1250 // accordingly to provide the illusion of full support
1251 te.nonCacheable = false;
1252
1253 switch (attr_7_4) {
1254 case 0x0:
1255 // Strongly-ordered or Device memory
1256 if (attr_3_0 == 0x0)
1257 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1258 else if (attr_3_0 == 0x4)
1259 te.mtype = TlbEntry::MemoryType::Device;
1260 else
1261 panic("Unpredictable behavior\n");
1262 te.nonCacheable = true;
1263 te.outerAttrs = 0;
1264 break;
1265 case 0x4:
1266 // Normal memory, Outer Non-cacheable
1267 te.mtype = TlbEntry::MemoryType::Normal;
1268 te.outerAttrs = 0;
1269 if (attr_3_0 == 0x4)
1270 // Inner Non-cacheable
1271 te.nonCacheable = true;
1272 else if (attr_3_0 < 0x8)
1273 panic("Unpredictable behavior\n");
1274 break;
1275 case 0x8:
1276 case 0x9:
1277 case 0xa:
1278 case 0xb:
1279 case 0xc:
1280 case 0xd:
1281 case 0xe:
1282 case 0xf:
1283 if (attr_7_4 & 0x4) {
1284 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1285 } else {
1286 te.outerAttrs = 0x2;
1287 }
1288 // Normal memory, Outer Cacheable
1289 te.mtype = TlbEntry::MemoryType::Normal;
1290 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1291 panic("Unpredictable behavior\n");
1292 break;
1293 default:
1294 panic("Unpredictable behavior\n");
1295 break;
1296 }
1297
1298 switch (attr_3_0) {
1299 case 0x0:
1300 te.innerAttrs = 0x1;
1301 break;
1302 case 0x4:
1303 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1304 break;
1305 case 0x8:
1306 case 0x9:
1307 case 0xA:
1308 case 0xB:
1309 te.innerAttrs = 6;
1310 break;
1311 case 0xC:
1312 case 0xD:
1313 case 0xE:
1314 case 0xF:
1315 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1316 break;
1317 default:
1318 panic("Unpredictable behavior\n");
1319 break;
1320 }
1321 }
1322
1323 te.outerShareable = sh == 2;
1324 te.shareable = (sh & 0x2) ? true : false;
1325 te.setAttributes(true);
1326 te.attributes |= (uint64_t) attr << 56;
1327}
1328
1329void
1330TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1331 uint8_t sh)
1332{
1333 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1334
1335 // Select MAIR
1336 uint64_t mair;
1337 switch (currState->el) {
1338 case EL0:
1339 case EL1:
1340 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1341 break;
1342 case EL2:
1343 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1344 break;
1345 case EL3:
1346 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1347 break;
1348 default:
1349 panic("Invalid exception level");
1350 break;
1351 }
1352
1353 // Select attributes
1354 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1355 uint8_t attr_lo = bits(attr, 3, 0);
1356 uint8_t attr_hi = bits(attr, 7, 4);
1357
1358 // Memory type
1359 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1360
1361 // Cacheability
1362 te.nonCacheable = false;
1363 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1364 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1365 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1366 te.nonCacheable = true;
1367 }
1368
1369 te.shareable = sh == 2;
1370 te.outerShareable = (sh & 0x2) ? true : false;
1371 // Attributes formatted according to the 64-bit PAR
1372 te.attributes = ((uint64_t) attr << 56) |
1373 (1 << 11) | // LPAE bit
1374 (te.ns << 9) | // NS bit
1375 (sh << 7);
1376}
1377
1378void
1379TableWalker::doL1Descriptor()
1380{
1381 if (currState->fault != NoFault) {
1382 return;
1383 }
1384
1385 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1386 currState->vaddr_tainted, currState->l1Desc.data);
1387 TlbEntry te;
1388
1389 switch (currState->l1Desc.type()) {
1390 case L1Descriptor::Ignore:
1391 case L1Descriptor::Reserved:
1392 if (!currState->timing) {
1393 currState->tc = NULL;
1394 currState->req = NULL;
1395 }
1396 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1397 if (currState->isFetch)
1398 currState->fault =
1399 std::make_shared<PrefetchAbort>(
1400 currState->vaddr_tainted,
1401 ArmFault::TranslationLL + L1,
1402 isStage2,
1403 ArmFault::VmsaTran);
1404 else
1405 currState->fault =
1406 std::make_shared<DataAbort>(
1407 currState->vaddr_tainted,
1408 TlbEntry::DomainType::NoAccess,
1409 currState->isWrite,
1410 ArmFault::TranslationLL + L1, isStage2,
1411 ArmFault::VmsaTran);
1412 return;
1413 case L1Descriptor::Section:
1414 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1415 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1416 * enabled if set, do l1.Desc.setAp0() instead of generating
1417 * AccessFlag0
1418 */
1419
1420 currState->fault = std::make_shared<DataAbort>(
1421 currState->vaddr_tainted,
1422 currState->l1Desc.domain(),
1423 currState->isWrite,
1424 ArmFault::AccessFlagLL + L1,
1425 isStage2,
1426 ArmFault::VmsaTran);
1427 }
1428 if (currState->l1Desc.supersection()) {
1429 panic("Haven't implemented supersections\n");
1430 }
1431 insertTableEntry(currState->l1Desc, false);
1432 return;
1433 case L1Descriptor::PageTable:
1434 {
1435 Addr l2desc_addr;
1436 l2desc_addr = currState->l1Desc.l2Addr() |
1437 (bits(currState->vaddr, 19, 12) << 2);
1438 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1439 l2desc_addr, currState->isSecure ? "s" : "ns");
1440
1441 // Trickbox address check
1442 currState->fault = tlb->walkTrickBoxCheck(
1443 l2desc_addr, currState->isSecure, currState->vaddr,
1444 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1445 currState->l1Desc.domain(), L2);
1446
1447 if (currState->fault) {
1448 if (!currState->timing) {
1449 currState->tc = NULL;
1450 currState->req = NULL;
1451 }
1452 return;
1453 }
1454
1455 Request::Flags flag = Request::PT_WALK;
1456 if (currState->isSecure)
1457 flag.set(Request::SECURE);
1458
1459 bool delayed;
1460 delayed = fetchDescriptor(l2desc_addr,
1461 (uint8_t*)&currState->l2Desc.data,
1462 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1463 &TableWalker::doL2Descriptor);
1464 if (delayed) {
1465 currState->delayed = true;
1466 }
1467
1468 return;
1469 }
1470 default:
1471 panic("A new type in a 2 bit field?\n");
1472 }
1473}
1474
1475void
1476TableWalker::doLongDescriptor()
1477{
1478 if (currState->fault != NoFault) {
1479 return;
1480 }
1481
1482 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1483 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1484 currState->longDesc.data,
1485 currState->aarch64 ? "AArch64" : "long-desc.");
1486
1487 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1488 (currState->longDesc.type() == LongDescriptor::Page)) {
1489 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1490 "xn: %d, ap: %d, af: %d, type: %d\n",
1491 currState->longDesc.lookupLevel,
1492 currState->longDesc.data,
1493 currState->longDesc.pxn(),
1494 currState->longDesc.xn(),
1495 currState->longDesc.ap(),
1496 currState->longDesc.af(),
1497 currState->longDesc.type());
1498 } else {
1499 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1500 currState->longDesc.lookupLevel,
1501 currState->longDesc.data,
1502 currState->longDesc.type());
1503 }
1504
1505 TlbEntry te;
1506
1507 switch (currState->longDesc.type()) {
1508 case LongDescriptor::Invalid:
1509 if (!currState->timing) {
1510 currState->tc = NULL;
1511 currState->req = NULL;
1512 }
1513
1514 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1515 currState->longDesc.lookupLevel,
1516 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1517 if (currState->isFetch)
1518 currState->fault = std::make_shared<PrefetchAbort>(
1519 currState->vaddr_tainted,
1520 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1521 isStage2,
1522 ArmFault::LpaeTran);
1523 else
1524 currState->fault = std::make_shared<DataAbort>(
1525 currState->vaddr_tainted,
1526 TlbEntry::DomainType::NoAccess,
1527 currState->isWrite,
1528 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1529 isStage2,
1530 ArmFault::LpaeTran);
1531 return;
1532 case LongDescriptor::Block:
1533 case LongDescriptor::Page:
1534 {
1535 bool fault = false;
1536 bool aff = false;
1537 // Check for address size fault
1538 if (checkAddrSizeFaultAArch64(
1539 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1540 currState->longDesc.offsetBits()),
1541 currState->physAddrRange)) {
1542 fault = true;
1543 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1544 currState->longDesc.lookupLevel);
1545 // Check for access fault
1546 } else if (currState->longDesc.af() == 0) {
1547 fault = true;
1548 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1549 currState->longDesc.lookupLevel);
1550 aff = true;
1551 }
1552 if (fault) {
1553 if (currState->isFetch)
1554 currState->fault = std::make_shared<PrefetchAbort>(
1555 currState->vaddr_tainted,
1556 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1557 currState->longDesc.lookupLevel,
1558 isStage2,
1559 ArmFault::LpaeTran);
1560 else
1561 currState->fault = std::make_shared<DataAbort>(
1562 currState->vaddr_tainted,
1563 TlbEntry::DomainType::NoAccess, currState->isWrite,
1564 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1565 currState->longDesc.lookupLevel,
1566 isStage2,
1567 ArmFault::LpaeTran);
1568 } else {
1569 insertTableEntry(currState->longDesc, true);
1570 }
1571 }
1572 return;
1573 case LongDescriptor::Table:
1574 {
1575 // Set hierarchical permission flags
1576 currState->secureLookup = currState->secureLookup &&
1577 currState->longDesc.secureTable();
1578 currState->rwTable = currState->rwTable &&
1579 currState->longDesc.rwTable();
1580 currState->userTable = currState->userTable &&
1581 currState->longDesc.userTable();
1582 currState->xnTable = currState->xnTable ||
1583 currState->longDesc.xnTable();
1584 currState->pxnTable = currState->pxnTable ||
1585 currState->longDesc.pxnTable();
1586
1587 // Set up next level lookup
1588 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1589 currState->vaddr);
1590
1591 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1592 currState->longDesc.lookupLevel,
1593 currState->longDesc.lookupLevel + 1,
1594 next_desc_addr,
1595 currState->secureLookup ? "s" : "ns");
1596
1597 // Check for address size fault
1598 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1599 next_desc_addr, currState->physAddrRange)) {
1600 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1601 currState->longDesc.lookupLevel);
1602 if (currState->isFetch)
1603 currState->fault = std::make_shared<PrefetchAbort>(
1604 currState->vaddr_tainted,
1605 ArmFault::AddressSizeLL
1606 + currState->longDesc.lookupLevel,
1607 isStage2,
1608 ArmFault::LpaeTran);
1609 else
1610 currState->fault = std::make_shared<DataAbort>(
1611 currState->vaddr_tainted,
1612 TlbEntry::DomainType::NoAccess, currState->isWrite,
1613 ArmFault::AddressSizeLL
1614 + currState->longDesc.lookupLevel,
1615 isStage2,
1616 ArmFault::LpaeTran);
1617 return;
1618 }
1619
1620 // Trickbox address check
1621 currState->fault = tlb->walkTrickBoxCheck(
1622 next_desc_addr, currState->vaddr,
1623 currState->vaddr, sizeof(uint64_t),
1624 currState->isFetch, currState->isWrite,
1625 TlbEntry::DomainType::Client,
1626 toLookupLevel(currState->longDesc.lookupLevel +1));
1627
1628 if (currState->fault) {
1629 if (!currState->timing) {
1630 currState->tc = NULL;
1631 currState->req = NULL;
1632 }
1633 return;
1634 }
1635
1636 Request::Flags flag = Request::PT_WALK;
1637 if (currState->secureLookup)
1638 flag.set(Request::SECURE);
1639
1640 currState->longDesc.lookupLevel =
1641 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1642 Event *event = NULL;
1643 switch (currState->longDesc.lookupLevel) {
1644 case L1:
1645 assert(currState->aarch64);
1646 event = &doL1LongDescEvent;
1647 break;
1648 case L2:
1649 event = &doL2LongDescEvent;
1650 break;
1651 case L3:
1652 event = &doL3LongDescEvent;
1653 break;
1654 default:
1655 panic("Wrong lookup level in table walk\n");
1656 break;
1657 }
1658
1659 bool delayed;
1660 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1661 sizeof(uint64_t), flag, -1, event,
1662 &TableWalker::doLongDescriptor);
1663 if (delayed) {
1664 currState->delayed = true;
1665 }
1666 }
1667 return;
1668 default:
1669 panic("A new type in a 2 bit field?\n");
1670 }
1671}
1672
1673void
1674TableWalker::doL2Descriptor()
1675{
1676 if (currState->fault != NoFault) {
1677 return;
1678 }
1679
1680 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1681 currState->vaddr_tainted, currState->l2Desc.data);
1682 TlbEntry te;
1683
1684 if (currState->l2Desc.invalid()) {
1685 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1686 if (!currState->timing) {
1687 currState->tc = NULL;
1688 currState->req = NULL;
1689 }
1690 if (currState->isFetch)
1691 currState->fault = std::make_shared<PrefetchAbort>(
1692 currState->vaddr_tainted,
1693 ArmFault::TranslationLL + L2,
1694 isStage2,
1695 ArmFault::VmsaTran);
1696 else
1697 currState->fault = std::make_shared<DataAbort>(
1698 currState->vaddr_tainted, currState->l1Desc.domain(),
1699 currState->isWrite, ArmFault::TranslationLL + L2,
1700 isStage2,
1701 ArmFault::VmsaTran);
1702 return;
1703 }
1704
1705 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1706 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1707 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1708 */
1709 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1710 currState->sctlr.afe, currState->l2Desc.ap());
1711
1712 currState->fault = std::make_shared<DataAbort>(
1713 currState->vaddr_tainted,
1714 TlbEntry::DomainType::NoAccess, currState->isWrite,
1715 ArmFault::AccessFlagLL + L2, isStage2,
1716 ArmFault::VmsaTran);
1717 }
1718
1719 insertTableEntry(currState->l2Desc, false);
1720}
1721
1722void
1723TableWalker::doL1DescriptorWrapper()
1724{
1725 currState = stateQueues[L1].front();
1726 currState->delayed = false;
1727 // if there's a stage2 translation object we don't need it any more
1728 if (currState->stage2Tran) {
1729 delete currState->stage2Tran;
1730 currState->stage2Tran = NULL;
1731 }
1732
1733
1734 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1735 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1736
1737 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1738 doL1Descriptor();
1739
1740 stateQueues[L1].pop_front();
1741 // Check if fault was generated
1742 if (currState->fault != NoFault) {
1743 currState->transState->finish(currState->fault, currState->req,
1744 currState->tc, currState->mode);
1745 statWalksShortTerminatedAtLevel[0]++;
1746
1747 pending = false;
1748 nextWalk(currState->tc);
1749
1750 currState->req = NULL;
1751 currState->tc = NULL;
1752 currState->delayed = false;
1753 delete currState;
1754 }
1755 else if (!currState->delayed) {
1756 // delay is not set so there is no L2 to do
1757 // Don't finish the translation if a stage 2 look up is underway
1758 if (!currState->doingStage2) {
1759 statWalkServiceTime.sample(curTick() - currState->startTime);
1760 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1761 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1762 currState->transState, currState->mode);
1763 statWalksShortTerminatedAtLevel[0]++;
1764 }
1765
1766 pending = false;
1767 nextWalk(currState->tc);
1768
1769 currState->req = NULL;
1770 currState->tc = NULL;
1771 currState->delayed = false;
1772 delete currState;
1773 } else {
1774 // need to do L2 descriptor
1775 stateQueues[L2].push_back(currState);
1776 }
1777 currState = NULL;
1778}
1779
1780void
1781TableWalker::doL2DescriptorWrapper()
1782{
1783 currState = stateQueues[L2].front();
1784 assert(currState->delayed);
1785 // if there's a stage2 translation object we don't need it any more
1786 if (currState->stage2Tran) {
1787 delete currState->stage2Tran;
1788 currState->stage2Tran = NULL;
1789 }
1790
1791 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1792 currState->vaddr_tainted);
1793 doL2Descriptor();
1794
1795 // Check if fault was generated
1796 if (currState->fault != NoFault) {
1797 currState->transState->finish(currState->fault, currState->req,
1798 currState->tc, currState->mode);
1799 statWalksShortTerminatedAtLevel[1]++;
1800 }
1801 else {
1802 // Don't finish the translation if a stage 2 look up is underway
1803 if (!currState->doingStage2) {
1804 statWalkServiceTime.sample(curTick() - currState->startTime);
1805 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1806 currState->fault = tlb->translateTiming(currState->req,
1807 currState->tc, currState->transState, currState->mode);
1808 statWalksShortTerminatedAtLevel[1]++;
1809 }
1810 }
1811
1812
1813 stateQueues[L2].pop_front();
1814 pending = false;
1815 nextWalk(currState->tc);
1816
1817 currState->req = NULL;
1818 currState->tc = NULL;
1819 currState->delayed = false;
1820
1821 delete currState;
1822 currState = NULL;
1823}
1824
1825void
1826TableWalker::doL0LongDescriptorWrapper()
1827{
1828 doLongDescriptorWrapper(L0);
1829}
1830
1831void
1832TableWalker::doL1LongDescriptorWrapper()
1833{
1834 doLongDescriptorWrapper(L1);
1835}
1836
1837void
1838TableWalker::doL2LongDescriptorWrapper()
1839{
1840 doLongDescriptorWrapper(L2);
1841}
1842
1843void
1844TableWalker::doL3LongDescriptorWrapper()
1845{
1846 doLongDescriptorWrapper(L3);
1847}
1848
1849void
1850TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1851{
1852 currState = stateQueues[curr_lookup_level].front();
1853 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1854 currState->delayed = false;
1855
1856 // if there's a stage2 translation object we don't need it any more
1857 if (currState->stage2Tran) {
1858 delete currState->stage2Tran;
1859 currState->stage2Tran = NULL;
1860 }
1861
1862 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1863 currState->vaddr_tainted);
1864 doLongDescriptor();
1865
1866 stateQueues[curr_lookup_level].pop_front();
1867
1868 if (currState->fault != NoFault) {
1869 // A fault was generated
1870 currState->transState->finish(currState->fault, currState->req,
1871 currState->tc, currState->mode);
1872
1873 pending = false;
1874 nextWalk(currState->tc);
1875
1876 currState->req = NULL;
1877 currState->tc = NULL;
1878 currState->delayed = false;
1879 delete currState;
1880 } else if (!currState->delayed) {
1881 // No additional lookups required
1882 // Don't finish the translation if a stage 2 look up is underway
1883 if (!currState->doingStage2) {
1884 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1885 statWalkServiceTime.sample(curTick() - currState->startTime);
1886 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1887 currState->transState,
1888 currState->mode);
1889 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1890 }
1891
1892 pending = false;
1893 nextWalk(currState->tc);
1894
1895 currState->req = NULL;
1896 currState->tc = NULL;
1897 currState->delayed = false;
1898 delete currState;
1899 } else {
1900 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1901 panic("Max. number of lookups already reached in table walk\n");
1902 // Need to perform additional lookups
1903 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1904 }
1905 currState = NULL;
1906}
1907
1908
1909void
1910TableWalker::nextWalk(ThreadContext *tc)
1911{
1912 if (pendingQueue.size())
1913 schedule(doProcessEvent, clockEdge(Cycles(1)));
1914 else
1915 completeDrain();
1916}
1917
1918bool
1919TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1920 Request::Flags flags, int queueIndex, Event *event,
1921 void (TableWalker::*doDescriptor)())
1922{
1923 bool isTiming = currState->timing;
1924
1925 // do the requests for the page table descriptors have to go through the
1926 // second stage MMU
1927 if (currState->stage2Req) {
1928 Fault fault;
1929 flags = flags | TLB::MustBeOne;
1930
1931 if (isTiming) {
1932 Stage2MMU::Stage2Translation *tran = new
1933 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1934 currState->vaddr);
1935 currState->stage2Tran = tran;
1936 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1937 flags);
1938 fault = tran->fault;
1939 } else {
1940 fault = stage2Mmu->readDataUntimed(currState->tc,
1941 currState->vaddr, descAddr, data, numBytes, flags,
1942 currState->functional);
1943 }
1944
1945 if (fault != NoFault) {
1946 currState->fault = fault;
1947 }
1948 if (isTiming) {
1949 if (queueIndex >= 0) {
1950 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1951 stateQueues[queueIndex].size());
1952 stateQueues[queueIndex].push_back(currState);
1953 currState = NULL;
1954 }
1955 } else {
1956 (this->*doDescriptor)();
1957 }
1958 } else {
1959 if (isTiming) {
1960 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1961 currState->tc->getCpuPtr()->clockPeriod(),flags);
1962 if (queueIndex >= 0) {
1963 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1964 stateQueues[queueIndex].size());
1965 stateQueues[queueIndex].push_back(currState);
1966 currState = NULL;
1967 }
1968 } else if (!currState->functional) {
1969 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1970 currState->tc->getCpuPtr()->clockPeriod(), flags);
1971 (this->*doDescriptor)();
1972 } else {
1973 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1974 req->taskId(ContextSwitchTaskId::DMA);
1975 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1976 pkt->dataStatic(data);
1977 port->sendFunctional(pkt);
1978 (this->*doDescriptor)();
1979 delete req;
1980 delete pkt;
1981 }
1982 }
1983 return (isTiming);
1984}
1985
1986void
1987TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1988{
1989 TlbEntry te;
1990
1991 // Create and fill a new page table entry
1992 te.valid = true;
1993 te.longDescFormat = longDescriptor;
1994 te.isHyp = currState->isHyp;
1995 te.asid = currState->asid;
1996 te.vmid = currState->vmid;
1997 te.N = descriptor.offsetBits();
1998 te.vpn = currState->vaddr >> te.N;
1999 te.size = (1<<te.N) - 1;
2000 te.pfn = descriptor.pfn();
2001 te.domain = descriptor.domain();
2002 te.lookupLevel = descriptor.lookupLevel;
2003 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
2004 te.nstid = !currState->isSecure;
2005 te.xn = descriptor.xn();
2006 if (currState->aarch64)
2007 te.el = currState->el;
2008 else
2009 te.el = 1;
2010
2011 statPageSizes[pageSizeNtoStatBin(te.N)]++;
2012 statRequestOrigin[COMPLETED][currState->isFetch]++;
2013
2014 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2015 // as global
2016 te.global = descriptor.global(currState) || isStage2;
2017 if (longDescriptor) {
2018 LongDescriptor lDescriptor =
2019 dynamic_cast<LongDescriptor &>(descriptor);
2020
2021 te.xn |= currState->xnTable;
2022 te.pxn = currState->pxnTable || lDescriptor.pxn();
2023 if (isStage2) {
2024 // this is actually the HAP field, but its stored in the same bit
2025 // possitions as the AP field in a stage 1 translation.
2026 te.hap = lDescriptor.ap();
2027 } else {
2028 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2029 (currState->userTable && (descriptor.ap() & 0x1));
2030 }
2031 if (currState->aarch64)
2032 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2033 currState->longDesc.sh());
2034 else
2035 memAttrsLPAE(currState->tc, te, lDescriptor);
2036 } else {
2037 te.ap = descriptor.ap();
2038 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2039 descriptor.shareable());
2040 }
2041
2042 // Debug output
2043 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2044 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2045 te.N, te.pfn, te.size, te.global, te.valid);
2046 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2047 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2048 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2049 te.nonCacheable, te.ns);
2050 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2051 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2052 descriptor.getRawData());
2053
2054 // Insert the entry into the TLB
2055 tlb->insert(currState->vaddr, te);
2056 if (!currState->timing) {
2057 currState->tc = NULL;
2058 currState->req = NULL;
2059 }
2060}
2061
2062ArmISA::TableWalker *
2063ArmTableWalkerParams::create()
2064{
2065 return new ArmISA::TableWalker(this);
2066}
2067
2068LookupLevel
2069TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2070{
2071 switch (lookup_level_as_int) {
2072 case L1:
2073 return L1;
2074 case L2:
2075 return L2;
2076 case L3:
2077 return L3;
2078 default:
2079 panic("Invalid lookup level conversion");
2080 }
2081}
2082
2083/* this method keeps track of the table walker queue's residency, so
2084 * needs to be called whenever requests start and complete. */
2085void
2086TableWalker::pendingChange()
2087{
2088 unsigned n = pendingQueue.size();
2089 if ((currState != NULL) && (currState != pendingQueue.front())) {
2090 ++n;
2091 }
2092
2093 if (n != pendingReqs) {
2094 Tick now = curTick();
2095 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2096 pendingReqs = n;
2097 pendingChangeTick = now;
2098 }
2099}
2100
2101uint8_t
2102TableWalker::pageSizeNtoStatBin(uint8_t N)
2103{
2104 /* for statPageSizes */
2105 switch(N) {
2106 case 12: return 0; // 4K
2107 case 14: return 1; // 16K (using 16K granule in v8-64)
2108 case 16: return 2; // 64K
2109 case 20: return 3; // 1M
2110 case 21: return 4; // 2M-LPAE
2111 case 24: return 5; // 16M
2112 case 25: return 6; // 32M (using 16K granule in v8-64)
2113 case 29: return 7; // 512M (using 64K granule in v8-64)
2114 case 30: return 8; // 1G-LPAE
2115 default:
2116 panic("unknown page size");
2117 return 255;
2118 }
2119}
2120
2121void
2122TableWalker::regStats()
2123{
2124 statWalks
2125 .name(name() + ".walks")
2126 .desc("Table walker walks requested")
2127 ;
2128
2129 statWalksShortDescriptor
2130 .name(name() + ".walksShort")
2131 .desc("Table walker walks initiated with short descriptors")
2132 .flags(Stats::nozero)
2133 ;
2134
2135 statWalksLongDescriptor
2136 .name(name() + ".walksLong")
2137 .desc("Table walker walks initiated with long descriptors")
2138 .flags(Stats::nozero)
2139 ;
2140
2141 statWalksShortTerminatedAtLevel
2142 .init(2)
2143 .name(name() + ".walksShortTerminationLevel")
2144 .desc("Level at which table walker walks "
2145 "with short descriptors terminate")
2146 .flags(Stats::nozero)
2147 ;
2148 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2149 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2150
2151 statWalksLongTerminatedAtLevel
2152 .init(4)
2153 .name(name() + ".walksLongTerminationLevel")
2154 .desc("Level at which table walker walks "
2155 "with long descriptors terminate")
2156 .flags(Stats::nozero)
2157 ;
2158 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2159 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2160 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2161 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2162
2163 statSquashedBefore
2164 .name(name() + ".walksSquashedBefore")
2165 .desc("Table walks squashed before starting")
2166 .flags(Stats::nozero)
2167 ;
2168
2169 statSquashedAfter
2170 .name(name() + ".walksSquashedAfter")
2171 .desc("Table walks squashed after completion")
2172 .flags(Stats::nozero)
2173 ;
2174
2175 statWalkWaitTime
2176 .init(16)
2177 .name(name() + ".walkWaitTime")
2178 .desc("Table walker wait (enqueue to first request) latency")
2179 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2180 ;
2181
2182 statWalkServiceTime
2183 .init(16)
2184 .name(name() + ".walkCompletionTime")
2185 .desc("Table walker service (enqueue to completion) latency")
2186 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2187 ;
2188
2189 statPendingWalks
2190 .init(16)
2191 .name(name() + ".walksPending")
2192 .desc("Table walker pending requests distribution")
2193 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2194 ;
2195
2196 statPageSizes // see DDI 0487A D4-1661
2197 .init(9)
2198 .name(name() + ".walkPageSizes")
2199 .desc("Table walker page sizes translated")
2200 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2201 ;
2202 statPageSizes.subname(0, "4K");
2203 statPageSizes.subname(1, "16K");
2204 statPageSizes.subname(2, "64K");
2205 statPageSizes.subname(3, "1M");
2206 statPageSizes.subname(4, "2M");
2207 statPageSizes.subname(5, "16M");
2208 statPageSizes.subname(6, "32M");
2209 statPageSizes.subname(7, "512M");
2210 statPageSizes.subname(8, "1G");
2211
2212 statRequestOrigin
2213 .init(2,2) // Instruction/Data, requests/completed
2214 .name(name() + ".walkRequestOrigin")
2215 .desc("Table walker requests started/completed, data/inst")
2216 .flags(Stats::total)
2217 ;
2218 statRequestOrigin.subname(0,"Requested");
2219 statRequestOrigin.subname(1,"Completed");
2220 statRequestOrigin.ysubname(0,"Data");
2221 statRequestOrigin.ysubname(1,"Inst");
2222}
| 55#include "sim/system.hh"
56
57using namespace ArmISA;
58
59TableWalker::TableWalker(const Params *p)
60 : MemObject(p), drainManager(NULL),
61 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId),
62 isStage2(p->is_stage2), tlb(NULL),
63 currState(NULL), pending(false),
64 numSquashable(p->num_squash_per_cycle),
65 pendingReqs(0),
66 pendingChangeTick(curTick()),
67 doL1DescEvent(this), doL2DescEvent(this),
68 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
69 doL3LongDescEvent(this),
70 doProcessEvent(this)
71{
72 sctlr = 0;
73
74 // Cache system-level properties
75 if (FullSystem) {
76 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys);
77 assert(armSys);
78 haveSecurity = armSys->haveSecurity();
79 _haveLPAE = armSys->haveLPAE();
80 _haveVirtualization = armSys->haveVirtualization();
81 physAddrRange = armSys->physAddrRange();
82 _haveLargeAsid64 = armSys->haveLargeAsid64();
83 } else {
84 haveSecurity = _haveLPAE = _haveVirtualization = false;
85 _haveLargeAsid64 = false;
86 physAddrRange = 32;
87 }
88
89}
90
91TableWalker::~TableWalker()
92{
93 ;
94}
95
96void
97TableWalker::setMMU(Stage2MMU *m, MasterID master_id)
98{
99 stage2Mmu = m;
100 port = &m->getPort();
101 masterId = master_id;
102}
103
104void
105TableWalker::init()
106{
107 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n");
108 fatal_if(!port, "Table walker must have a valid port\n");
109 fatal_if(!tlb, "Table walker must have a valid TLB\n");
110}
111
112BaseMasterPort&
113TableWalker::getMasterPort(const std::string &if_name, PortID idx)
114{
115 if (if_name == "port") {
116 if (!isStage2) {
117 return *port;
118 } else {
119 fatal("Cannot access table walker port through stage-two walker\n");
120 }
121 }
122 return MemObject::getMasterPort(if_name, idx);
123}
124
125TableWalker::WalkerState::WalkerState() :
126 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
127 asid(0), vmid(0), isHyp(false), transState(nullptr),
128 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
129 secureLookup(false), rwTable(false), userTable(false), xnTable(false),
130 pxnTable(false), stage2Req(false), doingStage2(false),
131 stage2Tran(nullptr), timing(false), functional(false),
132 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
133 delayed(false), tableWalker(nullptr)
134{
135}
136
137void
138TableWalker::completeDrain()
139{
140 if (drainManager && stateQueues[L1].empty() && stateQueues[L2].empty() &&
141 pendingQueue.empty()) {
142 setDrainState(Drainable::Drained);
143 DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
144 drainManager->signalDrainDone();
145 drainManager = NULL;
146 }
147}
148
149unsigned int
150TableWalker::drain(DrainManager *dm)
151{
152 bool state_queues_not_empty = false;
153
154 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
155 if (!stateQueues[i].empty()) {
156 state_queues_not_empty = true;
157 break;
158 }
159 }
160
161 if (state_queues_not_empty || pendingQueue.size()) {
162 drainManager = dm;
163 setDrainState(Drainable::Draining);
164 DPRINTF(Drain, "TableWalker not drained\n");
165
166 // return port drain count plus the table walker itself needs to drain
167 return 1;
168 } else {
169 setDrainState(Drainable::Drained);
170 DPRINTF(Drain, "TableWalker free, no need to drain\n");
171
172 // table walker is drained, but its ports may still need to be drained
173 return 0;
174 }
175}
176
177void
178TableWalker::drainResume()
179{
180 Drainable::drainResume();
181 if (params()->sys->isTimingMode() && currState) {
182 delete currState;
183 currState = NULL;
184 pendingChange();
185 }
186}
187
188Fault
189TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
190 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
191 TLB::Translation *_trans, bool _timing, bool _functional,
192 bool secure, TLB::ArmTranslationType tranType)
193{
194 assert(!(_functional && _timing));
195 ++statWalks;
196
197 WalkerState *savedCurrState = NULL;
198
199 if (!currState && !_functional) {
200 // For atomic mode, a new WalkerState instance should be only created
201 // once per TLB. For timing mode, a new instance is generated for every
202 // TLB miss.
203 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
204
205 currState = new WalkerState();
206 currState->tableWalker = this;
207 } else if (_functional) {
208 // If we are mixing functional mode with timing (or even
209 // atomic), we need to to be careful and clean up after
210 // ourselves to not risk getting into an inconsistent state.
211 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
212 savedCurrState = currState;
213 currState = new WalkerState();
214 currState->tableWalker = this;
215 } else if (_timing) {
216 // This is a translation that was completed and then faulted again
217 // because some underlying parameters that affect the translation
218 // changed out from under us (e.g. asid). It will either be a
219 // misprediction, in which case nothing will happen or we'll use
220 // this fault to re-execute the faulting instruction which should clean
221 // up everything.
222 if (currState->vaddr_tainted == _req->getVaddr()) {
223 ++statSquashedBefore;
224 return std::make_shared<ReExec>();
225 }
226 }
227 pendingChange();
228
229 currState->startTime = curTick();
230 currState->tc = _tc;
231 currState->aarch64 = opModeIs64(currOpMode(_tc));
232 currState->el = currEL(_tc);
233 currState->transState = _trans;
234 currState->req = _req;
235 currState->fault = NoFault;
236 currState->asid = _asid;
237 currState->vmid = _vmid;
238 currState->isHyp = _isHyp;
239 currState->timing = _timing;
240 currState->functional = _functional;
241 currState->mode = _mode;
242 currState->tranType = tranType;
243 currState->isSecure = secure;
244 currState->physAddrRange = physAddrRange;
245
246 /** @todo These should be cached or grabbed from cached copies in
247 the TLB, all these miscreg reads are expensive */
248 currState->vaddr_tainted = currState->req->getVaddr();
249 if (currState->aarch64)
250 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
251 currState->tc, currState->el);
252 else
253 currState->vaddr = currState->vaddr_tainted;
254
255 if (currState->aarch64) {
256 switch (currState->el) {
257 case EL0:
258 case EL1:
259 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
260 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
261 break;
262 // @todo: uncomment this to enable Virtualization
263 // case EL2:
264 // assert(haveVirtualization);
265 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
266 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
267 // break;
268 case EL3:
269 assert(haveSecurity);
270 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
271 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
272 break;
273 default:
274 panic("Invalid exception level");
275 break;
276 }
277 } else {
278 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
279 MISCREG_SCTLR, currState->tc, !currState->isSecure));
280 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
281 MISCREG_TTBCR, currState->tc, !currState->isSecure));
282 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
283 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
284 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
285 }
286 sctlr = currState->sctlr;
287
288 currState->isFetch = (currState->mode == TLB::Execute);
289 currState->isWrite = (currState->mode == TLB::Write);
290
291 statRequestOrigin[REQUESTED][currState->isFetch]++;
292
293 // We only do a second stage of translation if we're not secure, or in
294 // hyp mode, the second stage MMU is enabled, and this table walker
295 // instance is the first stage.
296 currState->doingStage2 = false;
297 // @todo: for now disable this in AArch64 (HCR is not set)
298 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
299 !isStage2 && !currState->isSecure && !currState->isHyp;
300
301 bool long_desc_format = currState->aarch64 ||
302 (_haveLPAE && currState->ttbcr.eae) ||
303 _isHyp || isStage2;
304
305 if (long_desc_format) {
306 // Helper variables used for hierarchical permissions
307 currState->secureLookup = currState->isSecure;
308 currState->rwTable = true;
309 currState->userTable = true;
310 currState->xnTable = false;
311 currState->pxnTable = false;
312
313 ++statWalksLongDescriptor;
314 } else {
315 ++statWalksShortDescriptor;
316 }
317
318 if (!currState->timing) {
319 Fault fault = NoFault;
320 if (currState->aarch64)
321 fault = processWalkAArch64();
322 else if (long_desc_format)
323 fault = processWalkLPAE();
324 else
325 fault = processWalk();
326
327 // If this was a functional non-timing access restore state to
328 // how we found it.
329 if (currState->functional) {
330 delete currState;
331 currState = savedCurrState;
332 }
333 return fault;
334 }
335
336 if (pending || pendingQueue.size()) {
337 pendingQueue.push_back(currState);
338 currState = NULL;
339 pendingChange();
340 } else {
341 pending = true;
342 pendingChange();
343 if (currState->aarch64)
344 return processWalkAArch64();
345 else if (long_desc_format)
346 return processWalkLPAE();
347 else
348 return processWalk();
349 }
350
351 return NoFault;
352}
353
354void
355TableWalker::processWalkWrapper()
356{
357 assert(!currState);
358 assert(pendingQueue.size());
359 pendingChange();
360 currState = pendingQueue.front();
361
362 ExceptionLevel target_el = EL0;
363 if (currState->aarch64)
364 target_el = currEL(currState->tc);
365 else
366 target_el = EL1;
367
368 // Check if a previous walk filled this request already
369 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
370 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
371 currState->vmid, currState->isHyp, currState->isSecure, true, false,
372 target_el);
373
374 // Check if we still need to have a walk for this request. If the requesting
375 // instruction has been squashed, or a previous walk has filled the TLB with
376 // a match, we just want to get rid of the walk. The latter could happen
377 // when there are multiple outstanding misses to a single page and a
378 // previous request has been successfully translated.
379 if (!currState->transState->squashed() && !te) {
380 // We've got a valid request, lets process it
381 pending = true;
382 pendingQueue.pop_front();
383 // Keep currState in case one of the processWalk... calls NULLs it
384 WalkerState *curr_state_copy = currState;
385 Fault f;
386 if (currState->aarch64)
387 f = processWalkAArch64();
388 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
389 f = processWalkLPAE();
390 else
391 f = processWalk();
392
393 if (f != NoFault) {
394 curr_state_copy->transState->finish(f, curr_state_copy->req,
395 curr_state_copy->tc, curr_state_copy->mode);
396
397 delete curr_state_copy;
398 }
399 return;
400 }
401
402
403 // If the instruction that we were translating for has been
404 // squashed we shouldn't bother.
405 unsigned num_squashed = 0;
406 ThreadContext *tc = currState->tc;
407 while ((num_squashed < numSquashable) && currState &&
408 (currState->transState->squashed() || te)) {
409 pendingQueue.pop_front();
410 num_squashed++;
411 statSquashedBefore++;
412
413 DPRINTF(TLB, "Squashing table walk for address %#x\n",
414 currState->vaddr_tainted);
415
416 if (currState->transState->squashed()) {
417 // finish the translation which will delete the translation object
418 currState->transState->finish(
419 std::make_shared<UnimpFault>("Squashed Inst"),
420 currState->req, currState->tc, currState->mode);
421 } else {
422 // translate the request now that we know it will work
423 statWalkServiceTime.sample(curTick() - currState->startTime);
424 tlb->translateTiming(currState->req, currState->tc,
425 currState->transState, currState->mode);
426
427 }
428
429 // delete the current request
430 delete currState;
431
432 // peak at the next one
433 if (pendingQueue.size()) {
434 currState = pendingQueue.front();
435 te = tlb->lookup(currState->vaddr, currState->asid,
436 currState->vmid, currState->isHyp, currState->isSecure, true,
437 false, target_el);
438 } else {
439 // Terminate the loop, nothing more to do
440 currState = NULL;
441 }
442 }
443 pendingChange();
444
445 // if we still have pending translations, schedule more work
446 nextWalk(tc);
447 currState = NULL;
448}
449
450Fault
451TableWalker::processWalk()
452{
453 Addr ttbr = 0;
454
455 // If translation isn't enabled, we shouldn't be here
456 assert(currState->sctlr.m || isStage2);
457
458 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
459 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
460 32 - currState->ttbcr.n));
461
462 statWalkWaitTime.sample(curTick() - currState->startTime);
463
464 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
465 32 - currState->ttbcr.n)) {
466 DPRINTF(TLB, " - Selecting TTBR0\n");
467 // Check if table walk is allowed when Security Extensions are enabled
468 if (haveSecurity && currState->ttbcr.pd0) {
469 if (currState->isFetch)
470 return std::make_shared<PrefetchAbort>(
471 currState->vaddr_tainted,
472 ArmFault::TranslationLL + L1,
473 isStage2,
474 ArmFault::VmsaTran);
475 else
476 return std::make_shared<DataAbort>(
477 currState->vaddr_tainted,
478 TlbEntry::DomainType::NoAccess, currState->isWrite,
479 ArmFault::TranslationLL + L1, isStage2,
480 ArmFault::VmsaTran);
481 }
482 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
483 MISCREG_TTBR0, currState->tc, !currState->isSecure));
484 } else {
485 DPRINTF(TLB, " - Selecting TTBR1\n");
486 // Check if table walk is allowed when Security Extensions are enabled
487 if (haveSecurity && currState->ttbcr.pd1) {
488 if (currState->isFetch)
489 return std::make_shared<PrefetchAbort>(
490 currState->vaddr_tainted,
491 ArmFault::TranslationLL + L1,
492 isStage2,
493 ArmFault::VmsaTran);
494 else
495 return std::make_shared<DataAbort>(
496 currState->vaddr_tainted,
497 TlbEntry::DomainType::NoAccess, currState->isWrite,
498 ArmFault::TranslationLL + L1, isStage2,
499 ArmFault::VmsaTran);
500 }
501 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
502 MISCREG_TTBR1, currState->tc, !currState->isSecure));
503 currState->ttbcr.n = 0;
504 }
505
506 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
507 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
508 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
509 currState->isSecure ? "s" : "ns");
510
511 // Trickbox address check
512 Fault f;
513 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
514 currState->vaddr, sizeof(uint32_t), currState->isFetch,
515 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
516 if (f) {
517 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
518 if (currState->timing) {
519 pending = false;
520 nextWalk(currState->tc);
521 currState = NULL;
522 } else {
523 currState->tc = NULL;
524 currState->req = NULL;
525 }
526 return f;
527 }
528
529 Request::Flags flag = Request::PT_WALK;
530 if (currState->sctlr.c == 0) {
531 flag.set(Request::UNCACHEABLE);
532 }
533
534 bool delayed;
535 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
536 sizeof(uint32_t), flag, L1, &doL1DescEvent,
537 &TableWalker::doL1Descriptor);
538 if (!delayed) {
539 f = currState->fault;
540 }
541
542 return f;
543}
544
545Fault
546TableWalker::processWalkLPAE()
547{
548 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
549 int tsz, n;
550 LookupLevel start_lookup_level = L1;
551
552 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
553 currState->vaddr_tainted, currState->ttbcr);
554
555 statWalkWaitTime.sample(curTick() - currState->startTime);
556
557 Request::Flags flag = Request::PT_WALK;
558 if (currState->isSecure)
559 flag.set(Request::SECURE);
560
561 // work out which base address register to use, if in hyp mode we always
562 // use HTTBR
563 if (isStage2) {
564 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
565 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
566 tsz = sext<4>(currState->vtcr.t0sz);
567 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
568 } else if (currState->isHyp) {
569 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
570 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
571 tsz = currState->htcr.t0sz;
572 } else {
573 assert(_haveLPAE && currState->ttbcr.eae);
574
575 // Determine boundaries of TTBR0/1 regions
576 if (currState->ttbcr.t0sz)
577 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
578 else if (currState->ttbcr.t1sz)
579 ttbr0_max = (1ULL << 32) -
580 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
581 else
582 ttbr0_max = (1ULL << 32) - 1;
583 if (currState->ttbcr.t1sz)
584 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
585 else
586 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
587
588 // The following code snippet selects the appropriate translation table base
589 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
590 // depending on the address range supported by the translation table (ARM
591 // ARM issue C B3.6.4)
592 if (currState->vaddr <= ttbr0_max) {
593 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
594 // Check if table walk is allowed
595 if (currState->ttbcr.epd0) {
596 if (currState->isFetch)
597 return std::make_shared<PrefetchAbort>(
598 currState->vaddr_tainted,
599 ArmFault::TranslationLL + L1,
600 isStage2,
601 ArmFault::LpaeTran);
602 else
603 return std::make_shared<DataAbort>(
604 currState->vaddr_tainted,
605 TlbEntry::DomainType::NoAccess,
606 currState->isWrite,
607 ArmFault::TranslationLL + L1,
608 isStage2,
609 ArmFault::LpaeTran);
610 }
611 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
612 MISCREG_TTBR0, currState->tc, !currState->isSecure));
613 tsz = currState->ttbcr.t0sz;
614 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
615 start_lookup_level = L2;
616 } else if (currState->vaddr >= ttbr1_min) {
617 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
618 // Check if table walk is allowed
619 if (currState->ttbcr.epd1) {
620 if (currState->isFetch)
621 return std::make_shared<PrefetchAbort>(
622 currState->vaddr_tainted,
623 ArmFault::TranslationLL + L1,
624 isStage2,
625 ArmFault::LpaeTran);
626 else
627 return std::make_shared<DataAbort>(
628 currState->vaddr_tainted,
629 TlbEntry::DomainType::NoAccess,
630 currState->isWrite,
631 ArmFault::TranslationLL + L1,
632 isStage2,
633 ArmFault::LpaeTran);
634 }
635 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
636 MISCREG_TTBR1, currState->tc, !currState->isSecure));
637 tsz = currState->ttbcr.t1sz;
638 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
639 start_lookup_level = L2;
640 } else {
641 // Out of boundaries -> translation fault
642 if (currState->isFetch)
643 return std::make_shared<PrefetchAbort>(
644 currState->vaddr_tainted,
645 ArmFault::TranslationLL + L1,
646 isStage2,
647 ArmFault::LpaeTran);
648 else
649 return std::make_shared<DataAbort>(
650 currState->vaddr_tainted,
651 TlbEntry::DomainType::NoAccess,
652 currState->isWrite, ArmFault::TranslationLL + L1,
653 isStage2, ArmFault::LpaeTran);
654 }
655
656 }
657
658 // Perform lookup (ARM ARM issue C B3.6.6)
659 if (start_lookup_level == L1) {
660 n = 5 - tsz;
661 desc_addr = mbits(ttbr, 39, n) |
662 (bits(currState->vaddr, n + 26, 30) << 3);
663 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
664 desc_addr, currState->isSecure ? "s" : "ns");
665 } else {
666 // Skip first-level lookup
667 n = (tsz >= 2 ? 14 - tsz : 12);
668 desc_addr = mbits(ttbr, 39, n) |
669 (bits(currState->vaddr, n + 17, 21) << 3);
670 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
671 desc_addr, currState->isSecure ? "s" : "ns");
672 }
673
674 // Trickbox address check
675 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
676 currState->vaddr, sizeof(uint64_t), currState->isFetch,
677 currState->isWrite, TlbEntry::DomainType::NoAccess,
678 start_lookup_level);
679 if (f) {
680 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
681 if (currState->timing) {
682 pending = false;
683 nextWalk(currState->tc);
684 currState = NULL;
685 } else {
686 currState->tc = NULL;
687 currState->req = NULL;
688 }
689 return f;
690 }
691
692 if (currState->sctlr.c == 0) {
693 flag.set(Request::UNCACHEABLE);
694 }
695
696 if (currState->isSecure)
697 flag.set(Request::SECURE);
698
699 currState->longDesc.lookupLevel = start_lookup_level;
700 currState->longDesc.aarch64 = false;
701 currState->longDesc.grainSize = Grain4KB;
702
703 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
704 : (Event *) &doL2LongDescEvent;
705
706 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
707 sizeof(uint64_t), flag, start_lookup_level,
708 event, &TableWalker::doLongDescriptor);
709 if (!delayed) {
710 f = currState->fault;
711 }
712
713 return f;
714}
715
716unsigned
717TableWalker::adjustTableSizeAArch64(unsigned tsz)
718{
719 if (tsz < 25)
720 return 25;
721 if (tsz > 48)
722 return 48;
723 return tsz;
724}
725
726bool
727TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
728{
729 return (currPhysAddrRange != MaxPhysAddrRange &&
730 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
731}
732
733Fault
734TableWalker::processWalkAArch64()
735{
736 assert(currState->aarch64);
737
738 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
739 currState->vaddr_tainted, currState->tcr);
740
741 static const GrainSize GrainMapDefault[] =
742 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
743 static const GrainSize GrainMap_EL1_tg1[] =
744 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
745
746 statWalkWaitTime.sample(curTick() - currState->startTime);
747
748 // Determine TTBR, table size, granule size and phys. address range
749 Addr ttbr = 0;
750 int tsz = 0, ps = 0;
751 GrainSize tg = Grain4KB; // grain size computed from tg* field
752 bool fault = false;
753 switch (currState->el) {
754 case EL0:
755 case EL1:
756 switch (bits(currState->vaddr, 63,48)) {
757 case 0:
758 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
759 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
760 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
761 tg = GrainMapDefault[currState->tcr.tg0];
762 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
763 currState->tcr.epd0)
764 fault = true;
765 break;
766 case 0xffff:
767 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
768 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
769 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
770 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
771 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
772 currState->tcr.epd1)
773 fault = true;
774 break;
775 default:
776 // top two bytes must be all 0s or all 1s, else invalid addr
777 fault = true;
778 }
779 ps = currState->tcr.ips;
780 break;
781 case EL2:
782 case EL3:
783 switch(bits(currState->vaddr, 63,48)) {
784 case 0:
785 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
786 if (currState->el == EL2)
787 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
788 else
789 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
790 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
791 tg = GrainMapDefault[currState->tcr.tg0];
792 break;
793 default:
794 // invalid addr if top two bytes are not all 0s
795 fault = true;
796 }
797 ps = currState->tcr.ips;
798 break;
799 }
800
801 if (fault) {
802 Fault f;
803 if (currState->isFetch)
804 f = std::make_shared<PrefetchAbort>(
805 currState->vaddr_tainted,
806 ArmFault::TranslationLL + L0, isStage2,
807 ArmFault::LpaeTran);
808 else
809 f = std::make_shared<DataAbort>(
810 currState->vaddr_tainted,
811 TlbEntry::DomainType::NoAccess,
812 currState->isWrite,
813 ArmFault::TranslationLL + L0,
814 isStage2, ArmFault::LpaeTran);
815
816 if (currState->timing) {
817 pending = false;
818 nextWalk(currState->tc);
819 currState = NULL;
820 } else {
821 currState->tc = NULL;
822 currState->req = NULL;
823 }
824 return f;
825
826 }
827
828 if (tg == ReservedGrain) {
829 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
830 "DEFINED behavior takes this to mean 4KB granules\n");
831 tg = Grain4KB;
832 }
833
834 int stride = tg - 3;
835 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
836
837 // Determine starting lookup level
838 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
839 // in ARM DDI 0487A. These table values correspond to the cascading tests
840 // to compute the lookup level and are of the form
841 // (grain_size + N*stride), for N = {1, 2, 3}.
842 // A value of 64 will never succeed and a value of 0 will always succeed.
843 {
844 struct GrainMap {
845 GrainSize grain_size;
846 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
847 };
848 static const GrainMap GM[] = {
849 { Grain4KB, { 39, 30, 0, 0 } },
850 { Grain16KB, { 47, 36, 25, 0 } },
851 { Grain64KB, { 64, 42, 29, 0 } }
852 };
853
854 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
855
856 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
857 if (tg == GM[i].grain_size) {
858 lookup = GM[i].lookup_level_cutoff;
859 break;
860 }
861 }
862 assert(lookup);
863
864 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
865 if (tsz > lookup[L]) {
866 start_lookup_level = (LookupLevel) L;
867 break;
868 }
869 }
870 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
871 "Table walker couldn't find lookup level\n");
872 }
873
874 // Determine table base address
875 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
876 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
877
878 // Determine physical address size and raise an Address Size Fault if
879 // necessary
880 int pa_range = decodePhysAddrRange64(ps);
881 // Clamp to lower limit
882 if (pa_range > physAddrRange)
883 currState->physAddrRange = physAddrRange;
884 else
885 currState->physAddrRange = pa_range;
886 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
887 DPRINTF(TLB, "Address size fault before any lookup\n");
888 Fault f;
889 if (currState->isFetch)
890 f = std::make_shared<PrefetchAbort>(
891 currState->vaddr_tainted,
892 ArmFault::AddressSizeLL + start_lookup_level,
893 isStage2,
894 ArmFault::LpaeTran);
895 else
896 f = std::make_shared<DataAbort>(
897 currState->vaddr_tainted,
898 TlbEntry::DomainType::NoAccess,
899 currState->isWrite,
900 ArmFault::AddressSizeLL + start_lookup_level,
901 isStage2,
902 ArmFault::LpaeTran);
903
904
905 if (currState->timing) {
906 pending = false;
907 nextWalk(currState->tc);
908 currState = NULL;
909 } else {
910 currState->tc = NULL;
911 currState->req = NULL;
912 }
913 return f;
914
915 }
916
917 // Determine descriptor address
918 Addr desc_addr = base_addr |
919 (bits(currState->vaddr, tsz - 1,
920 stride * (3 - start_lookup_level) + tg) << 3);
921
922 // Trickbox address check
923 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
924 currState->vaddr, sizeof(uint64_t), currState->isFetch,
925 currState->isWrite, TlbEntry::DomainType::NoAccess,
926 start_lookup_level);
927 if (f) {
928 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
929 if (currState->timing) {
930 pending = false;
931 nextWalk(currState->tc);
932 currState = NULL;
933 } else {
934 currState->tc = NULL;
935 currState->req = NULL;
936 }
937 return f;
938 }
939
940 Request::Flags flag = Request::PT_WALK;
941 if (currState->sctlr.c == 0) {
942 flag.set(Request::UNCACHEABLE);
943 }
944
945 currState->longDesc.lookupLevel = start_lookup_level;
946 currState->longDesc.aarch64 = true;
947 currState->longDesc.grainSize = tg;
948
949 if (currState->timing) {
950 Event *event;
951 switch (start_lookup_level) {
952 case L0:
953 event = (Event *) &doL0LongDescEvent;
954 break;
955 case L1:
956 event = (Event *) &doL1LongDescEvent;
957 break;
958 case L2:
959 event = (Event *) &doL2LongDescEvent;
960 break;
961 case L3:
962 event = (Event *) &doL3LongDescEvent;
963 break;
964 default:
965 panic("Invalid table lookup level");
966 break;
967 }
968 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
969 event, (uint8_t*) &currState->longDesc.data,
970 currState->tc->getCpuPtr()->clockPeriod(), flag);
971 DPRINTF(TLBVerbose,
972 "Adding to walker fifo: queue size before adding: %d\n",
973 stateQueues[start_lookup_level].size());
974 stateQueues[start_lookup_level].push_back(currState);
975 currState = NULL;
976 } else if (!currState->functional) {
977 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
978 NULL, (uint8_t*) &currState->longDesc.data,
979 currState->tc->getCpuPtr()->clockPeriod(), flag);
980 doLongDescriptor();
981 f = currState->fault;
982 } else {
983 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
984 masterId);
985 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
986 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
987 port->sendFunctional(pkt);
988 doLongDescriptor();
989 delete req;
990 delete pkt;
991 f = currState->fault;
992 }
993
994 return f;
995}
996
997void
998TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
999 uint8_t texcb, bool s)
1000{
1001 // Note: tc and sctlr local variables are hiding tc and sctrl class
1002 // variables
1003 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1004 te.shareable = false; // default value
1005 te.nonCacheable = false;
1006 te.outerShareable = false;
1007 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1008 switch(texcb) {
1009 case 0: // Stongly-ordered
1010 te.nonCacheable = true;
1011 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1012 te.shareable = true;
1013 te.innerAttrs = 1;
1014 te.outerAttrs = 0;
1015 break;
1016 case 1: // Shareable Device
1017 te.nonCacheable = true;
1018 te.mtype = TlbEntry::MemoryType::Device;
1019 te.shareable = true;
1020 te.innerAttrs = 3;
1021 te.outerAttrs = 0;
1022 break;
1023 case 2: // Outer and Inner Write-Through, no Write-Allocate
1024 te.mtype = TlbEntry::MemoryType::Normal;
1025 te.shareable = s;
1026 te.innerAttrs = 6;
1027 te.outerAttrs = bits(texcb, 1, 0);
1028 break;
1029 case 3: // Outer and Inner Write-Back, no Write-Allocate
1030 te.mtype = TlbEntry::MemoryType::Normal;
1031 te.shareable = s;
1032 te.innerAttrs = 7;
1033 te.outerAttrs = bits(texcb, 1, 0);
1034 break;
1035 case 4: // Outer and Inner Non-cacheable
1036 te.nonCacheable = true;
1037 te.mtype = TlbEntry::MemoryType::Normal;
1038 te.shareable = s;
1039 te.innerAttrs = 0;
1040 te.outerAttrs = bits(texcb, 1, 0);
1041 break;
1042 case 5: // Reserved
1043 panic("Reserved texcb value!\n");
1044 break;
1045 case 6: // Implementation Defined
1046 panic("Implementation-defined texcb value!\n");
1047 break;
1048 case 7: // Outer and Inner Write-Back, Write-Allocate
1049 te.mtype = TlbEntry::MemoryType::Normal;
1050 te.shareable = s;
1051 te.innerAttrs = 5;
1052 te.outerAttrs = 1;
1053 break;
1054 case 8: // Non-shareable Device
1055 te.nonCacheable = true;
1056 te.mtype = TlbEntry::MemoryType::Device;
1057 te.shareable = false;
1058 te.innerAttrs = 3;
1059 te.outerAttrs = 0;
1060 break;
1061 case 9 ... 15: // Reserved
1062 panic("Reserved texcb value!\n");
1063 break;
1064 case 16 ... 31: // Cacheable Memory
1065 te.mtype = TlbEntry::MemoryType::Normal;
1066 te.shareable = s;
1067 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1068 te.nonCacheable = true;
1069 te.innerAttrs = bits(texcb, 1, 0);
1070 te.outerAttrs = bits(texcb, 3, 2);
1071 break;
1072 default:
1073 panic("More than 32 states for 5 bits?\n");
1074 }
1075 } else {
1076 assert(tc);
1077 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1078 currState->tc, !currState->isSecure));
1079 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1080 currState->tc, !currState->isSecure));
1081 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1082 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1083 switch(bits(texcb, 2,0)) {
1084 case 0:
1085 curr_tr = prrr.tr0;
1086 curr_ir = nmrr.ir0;
1087 curr_or = nmrr.or0;
1088 te.outerShareable = (prrr.nos0 == 0);
1089 break;
1090 case 1:
1091 curr_tr = prrr.tr1;
1092 curr_ir = nmrr.ir1;
1093 curr_or = nmrr.or1;
1094 te.outerShareable = (prrr.nos1 == 0);
1095 break;
1096 case 2:
1097 curr_tr = prrr.tr2;
1098 curr_ir = nmrr.ir2;
1099 curr_or = nmrr.or2;
1100 te.outerShareable = (prrr.nos2 == 0);
1101 break;
1102 case 3:
1103 curr_tr = prrr.tr3;
1104 curr_ir = nmrr.ir3;
1105 curr_or = nmrr.or3;
1106 te.outerShareable = (prrr.nos3 == 0);
1107 break;
1108 case 4:
1109 curr_tr = prrr.tr4;
1110 curr_ir = nmrr.ir4;
1111 curr_or = nmrr.or4;
1112 te.outerShareable = (prrr.nos4 == 0);
1113 break;
1114 case 5:
1115 curr_tr = prrr.tr5;
1116 curr_ir = nmrr.ir5;
1117 curr_or = nmrr.or5;
1118 te.outerShareable = (prrr.nos5 == 0);
1119 break;
1120 case 6:
1121 panic("Imp defined type\n");
1122 case 7:
1123 curr_tr = prrr.tr7;
1124 curr_ir = nmrr.ir7;
1125 curr_or = nmrr.or7;
1126 te.outerShareable = (prrr.nos7 == 0);
1127 break;
1128 }
1129
1130 switch(curr_tr) {
1131 case 0:
1132 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1133 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1134 te.nonCacheable = true;
1135 te.innerAttrs = 1;
1136 te.outerAttrs = 0;
1137 te.shareable = true;
1138 break;
1139 case 1:
1140 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1141 prrr.ds1, prrr.ds0, s);
1142 te.mtype = TlbEntry::MemoryType::Device;
1143 te.nonCacheable = true;
1144 te.innerAttrs = 3;
1145 te.outerAttrs = 0;
1146 if (prrr.ds1 && s)
1147 te.shareable = true;
1148 if (prrr.ds0 && !s)
1149 te.shareable = true;
1150 break;
1151 case 2:
1152 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1153 prrr.ns1, prrr.ns0, s);
1154 te.mtype = TlbEntry::MemoryType::Normal;
1155 if (prrr.ns1 && s)
1156 te.shareable = true;
1157 if (prrr.ns0 && !s)
1158 te.shareable = true;
1159 break;
1160 case 3:
1161 panic("Reserved type");
1162 }
1163
1164 if (te.mtype == TlbEntry::MemoryType::Normal){
1165 switch(curr_ir) {
1166 case 0:
1167 te.nonCacheable = true;
1168 te.innerAttrs = 0;
1169 break;
1170 case 1:
1171 te.innerAttrs = 5;
1172 break;
1173 case 2:
1174 te.innerAttrs = 6;
1175 break;
1176 case 3:
1177 te.innerAttrs = 7;
1178 break;
1179 }
1180
1181 switch(curr_or) {
1182 case 0:
1183 te.nonCacheable = true;
1184 te.outerAttrs = 0;
1185 break;
1186 case 1:
1187 te.outerAttrs = 1;
1188 break;
1189 case 2:
1190 te.outerAttrs = 2;
1191 break;
1192 case 3:
1193 te.outerAttrs = 3;
1194 break;
1195 }
1196 }
1197 }
1198 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1199 "outerAttrs: %d\n",
1200 te.shareable, te.innerAttrs, te.outerAttrs);
1201 te.setAttributes(false);
1202}
1203
1204void
1205TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1206 LongDescriptor &lDescriptor)
1207{
1208 assert(_haveLPAE);
1209
1210 uint8_t attr;
1211 uint8_t sh = lDescriptor.sh();
1212 // Different format and source of attributes if this is a stage 2
1213 // translation
1214 if (isStage2) {
1215 attr = lDescriptor.memAttr();
1216 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1217 uint8_t attr_1_0 = attr & 0x3;
1218
1219 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1220
1221 if (attr_3_2 == 0) {
1222 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1223 : TlbEntry::MemoryType::Device;
1224 te.outerAttrs = 0;
1225 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1226 te.nonCacheable = true;
1227 } else {
1228 te.mtype = TlbEntry::MemoryType::Normal;
1229 te.outerAttrs = attr_3_2 == 1 ? 0 :
1230 attr_3_2 == 2 ? 2 : 1;
1231 te.innerAttrs = attr_1_0 == 1 ? 0 :
1232 attr_1_0 == 2 ? 6 : 5;
1233 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1234 }
1235 } else {
1236 uint8_t attrIndx = lDescriptor.attrIndx();
1237
1238 // LPAE always uses remapping of memory attributes, irrespective of the
1239 // value of SCTLR.TRE
1240 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1241 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1242 !currState->isSecure);
1243 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1244 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1245 uint8_t attr_7_4 = bits(attr, 7, 4);
1246 uint8_t attr_3_0 = bits(attr, 3, 0);
1247 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1248
1249 // Note: the memory subsystem only cares about the 'cacheable' memory
1250 // attribute. The other attributes are only used to fill the PAR register
1251 // accordingly to provide the illusion of full support
1252 te.nonCacheable = false;
1253
1254 switch (attr_7_4) {
1255 case 0x0:
1256 // Strongly-ordered or Device memory
1257 if (attr_3_0 == 0x0)
1258 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1259 else if (attr_3_0 == 0x4)
1260 te.mtype = TlbEntry::MemoryType::Device;
1261 else
1262 panic("Unpredictable behavior\n");
1263 te.nonCacheable = true;
1264 te.outerAttrs = 0;
1265 break;
1266 case 0x4:
1267 // Normal memory, Outer Non-cacheable
1268 te.mtype = TlbEntry::MemoryType::Normal;
1269 te.outerAttrs = 0;
1270 if (attr_3_0 == 0x4)
1271 // Inner Non-cacheable
1272 te.nonCacheable = true;
1273 else if (attr_3_0 < 0x8)
1274 panic("Unpredictable behavior\n");
1275 break;
1276 case 0x8:
1277 case 0x9:
1278 case 0xa:
1279 case 0xb:
1280 case 0xc:
1281 case 0xd:
1282 case 0xe:
1283 case 0xf:
1284 if (attr_7_4 & 0x4) {
1285 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1286 } else {
1287 te.outerAttrs = 0x2;
1288 }
1289 // Normal memory, Outer Cacheable
1290 te.mtype = TlbEntry::MemoryType::Normal;
1291 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1292 panic("Unpredictable behavior\n");
1293 break;
1294 default:
1295 panic("Unpredictable behavior\n");
1296 break;
1297 }
1298
1299 switch (attr_3_0) {
1300 case 0x0:
1301 te.innerAttrs = 0x1;
1302 break;
1303 case 0x4:
1304 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1305 break;
1306 case 0x8:
1307 case 0x9:
1308 case 0xA:
1309 case 0xB:
1310 te.innerAttrs = 6;
1311 break;
1312 case 0xC:
1313 case 0xD:
1314 case 0xE:
1315 case 0xF:
1316 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1317 break;
1318 default:
1319 panic("Unpredictable behavior\n");
1320 break;
1321 }
1322 }
1323
1324 te.outerShareable = sh == 2;
1325 te.shareable = (sh & 0x2) ? true : false;
1326 te.setAttributes(true);
1327 te.attributes |= (uint64_t) attr << 56;
1328}
1329
1330void
1331TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1332 uint8_t sh)
1333{
1334 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1335
1336 // Select MAIR
1337 uint64_t mair;
1338 switch (currState->el) {
1339 case EL0:
1340 case EL1:
1341 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1342 break;
1343 case EL2:
1344 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1345 break;
1346 case EL3:
1347 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1348 break;
1349 default:
1350 panic("Invalid exception level");
1351 break;
1352 }
1353
1354 // Select attributes
1355 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1356 uint8_t attr_lo = bits(attr, 3, 0);
1357 uint8_t attr_hi = bits(attr, 7, 4);
1358
1359 // Memory type
1360 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1361
1362 // Cacheability
1363 te.nonCacheable = false;
1364 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1365 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1366 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1367 te.nonCacheable = true;
1368 }
1369
1370 te.shareable = sh == 2;
1371 te.outerShareable = (sh & 0x2) ? true : false;
1372 // Attributes formatted according to the 64-bit PAR
1373 te.attributes = ((uint64_t) attr << 56) |
1374 (1 << 11) | // LPAE bit
1375 (te.ns << 9) | // NS bit
1376 (sh << 7);
1377}
1378
1379void
1380TableWalker::doL1Descriptor()
1381{
1382 if (currState->fault != NoFault) {
1383 return;
1384 }
1385
1386 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1387 currState->vaddr_tainted, currState->l1Desc.data);
1388 TlbEntry te;
1389
1390 switch (currState->l1Desc.type()) {
1391 case L1Descriptor::Ignore:
1392 case L1Descriptor::Reserved:
1393 if (!currState->timing) {
1394 currState->tc = NULL;
1395 currState->req = NULL;
1396 }
1397 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1398 if (currState->isFetch)
1399 currState->fault =
1400 std::make_shared<PrefetchAbort>(
1401 currState->vaddr_tainted,
1402 ArmFault::TranslationLL + L1,
1403 isStage2,
1404 ArmFault::VmsaTran);
1405 else
1406 currState->fault =
1407 std::make_shared<DataAbort>(
1408 currState->vaddr_tainted,
1409 TlbEntry::DomainType::NoAccess,
1410 currState->isWrite,
1411 ArmFault::TranslationLL + L1, isStage2,
1412 ArmFault::VmsaTran);
1413 return;
1414 case L1Descriptor::Section:
1415 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1416 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1417 * enabled if set, do l1.Desc.setAp0() instead of generating
1418 * AccessFlag0
1419 */
1420
1421 currState->fault = std::make_shared<DataAbort>(
1422 currState->vaddr_tainted,
1423 currState->l1Desc.domain(),
1424 currState->isWrite,
1425 ArmFault::AccessFlagLL + L1,
1426 isStage2,
1427 ArmFault::VmsaTran);
1428 }
1429 if (currState->l1Desc.supersection()) {
1430 panic("Haven't implemented supersections\n");
1431 }
1432 insertTableEntry(currState->l1Desc, false);
1433 return;
1434 case L1Descriptor::PageTable:
1435 {
1436 Addr l2desc_addr;
1437 l2desc_addr = currState->l1Desc.l2Addr() |
1438 (bits(currState->vaddr, 19, 12) << 2);
1439 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1440 l2desc_addr, currState->isSecure ? "s" : "ns");
1441
1442 // Trickbox address check
1443 currState->fault = tlb->walkTrickBoxCheck(
1444 l2desc_addr, currState->isSecure, currState->vaddr,
1445 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1446 currState->l1Desc.domain(), L2);
1447
1448 if (currState->fault) {
1449 if (!currState->timing) {
1450 currState->tc = NULL;
1451 currState->req = NULL;
1452 }
1453 return;
1454 }
1455
1456 Request::Flags flag = Request::PT_WALK;
1457 if (currState->isSecure)
1458 flag.set(Request::SECURE);
1459
1460 bool delayed;
1461 delayed = fetchDescriptor(l2desc_addr,
1462 (uint8_t*)&currState->l2Desc.data,
1463 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1464 &TableWalker::doL2Descriptor);
1465 if (delayed) {
1466 currState->delayed = true;
1467 }
1468
1469 return;
1470 }
1471 default:
1472 panic("A new type in a 2 bit field?\n");
1473 }
1474}
1475
1476void
1477TableWalker::doLongDescriptor()
1478{
1479 if (currState->fault != NoFault) {
1480 return;
1481 }
1482
1483 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1484 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1485 currState->longDesc.data,
1486 currState->aarch64 ? "AArch64" : "long-desc.");
1487
1488 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1489 (currState->longDesc.type() == LongDescriptor::Page)) {
1490 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1491 "xn: %d, ap: %d, af: %d, type: %d\n",
1492 currState->longDesc.lookupLevel,
1493 currState->longDesc.data,
1494 currState->longDesc.pxn(),
1495 currState->longDesc.xn(),
1496 currState->longDesc.ap(),
1497 currState->longDesc.af(),
1498 currState->longDesc.type());
1499 } else {
1500 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1501 currState->longDesc.lookupLevel,
1502 currState->longDesc.data,
1503 currState->longDesc.type());
1504 }
1505
1506 TlbEntry te;
1507
1508 switch (currState->longDesc.type()) {
1509 case LongDescriptor::Invalid:
1510 if (!currState->timing) {
1511 currState->tc = NULL;
1512 currState->req = NULL;
1513 }
1514
1515 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1516 currState->longDesc.lookupLevel,
1517 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1518 if (currState->isFetch)
1519 currState->fault = std::make_shared<PrefetchAbort>(
1520 currState->vaddr_tainted,
1521 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1522 isStage2,
1523 ArmFault::LpaeTran);
1524 else
1525 currState->fault = std::make_shared<DataAbort>(
1526 currState->vaddr_tainted,
1527 TlbEntry::DomainType::NoAccess,
1528 currState->isWrite,
1529 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1530 isStage2,
1531 ArmFault::LpaeTran);
1532 return;
1533 case LongDescriptor::Block:
1534 case LongDescriptor::Page:
1535 {
1536 bool fault = false;
1537 bool aff = false;
1538 // Check for address size fault
1539 if (checkAddrSizeFaultAArch64(
1540 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1541 currState->longDesc.offsetBits()),
1542 currState->physAddrRange)) {
1543 fault = true;
1544 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1545 currState->longDesc.lookupLevel);
1546 // Check for access fault
1547 } else if (currState->longDesc.af() == 0) {
1548 fault = true;
1549 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1550 currState->longDesc.lookupLevel);
1551 aff = true;
1552 }
1553 if (fault) {
1554 if (currState->isFetch)
1555 currState->fault = std::make_shared<PrefetchAbort>(
1556 currState->vaddr_tainted,
1557 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1558 currState->longDesc.lookupLevel,
1559 isStage2,
1560 ArmFault::LpaeTran);
1561 else
1562 currState->fault = std::make_shared<DataAbort>(
1563 currState->vaddr_tainted,
1564 TlbEntry::DomainType::NoAccess, currState->isWrite,
1565 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1566 currState->longDesc.lookupLevel,
1567 isStage2,
1568 ArmFault::LpaeTran);
1569 } else {
1570 insertTableEntry(currState->longDesc, true);
1571 }
1572 }
1573 return;
1574 case LongDescriptor::Table:
1575 {
1576 // Set hierarchical permission flags
1577 currState->secureLookup = currState->secureLookup &&
1578 currState->longDesc.secureTable();
1579 currState->rwTable = currState->rwTable &&
1580 currState->longDesc.rwTable();
1581 currState->userTable = currState->userTable &&
1582 currState->longDesc.userTable();
1583 currState->xnTable = currState->xnTable ||
1584 currState->longDesc.xnTable();
1585 currState->pxnTable = currState->pxnTable ||
1586 currState->longDesc.pxnTable();
1587
1588 // Set up next level lookup
1589 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1590 currState->vaddr);
1591
1592 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1593 currState->longDesc.lookupLevel,
1594 currState->longDesc.lookupLevel + 1,
1595 next_desc_addr,
1596 currState->secureLookup ? "s" : "ns");
1597
1598 // Check for address size fault
1599 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1600 next_desc_addr, currState->physAddrRange)) {
1601 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1602 currState->longDesc.lookupLevel);
1603 if (currState->isFetch)
1604 currState->fault = std::make_shared<PrefetchAbort>(
1605 currState->vaddr_tainted,
1606 ArmFault::AddressSizeLL
1607 + currState->longDesc.lookupLevel,
1608 isStage2,
1609 ArmFault::LpaeTran);
1610 else
1611 currState->fault = std::make_shared<DataAbort>(
1612 currState->vaddr_tainted,
1613 TlbEntry::DomainType::NoAccess, currState->isWrite,
1614 ArmFault::AddressSizeLL
1615 + currState->longDesc.lookupLevel,
1616 isStage2,
1617 ArmFault::LpaeTran);
1618 return;
1619 }
1620
1621 // Trickbox address check
1622 currState->fault = tlb->walkTrickBoxCheck(
1623 next_desc_addr, currState->vaddr,
1624 currState->vaddr, sizeof(uint64_t),
1625 currState->isFetch, currState->isWrite,
1626 TlbEntry::DomainType::Client,
1627 toLookupLevel(currState->longDesc.lookupLevel +1));
1628
1629 if (currState->fault) {
1630 if (!currState->timing) {
1631 currState->tc = NULL;
1632 currState->req = NULL;
1633 }
1634 return;
1635 }
1636
1637 Request::Flags flag = Request::PT_WALK;
1638 if (currState->secureLookup)
1639 flag.set(Request::SECURE);
1640
1641 currState->longDesc.lookupLevel =
1642 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1643 Event *event = NULL;
1644 switch (currState->longDesc.lookupLevel) {
1645 case L1:
1646 assert(currState->aarch64);
1647 event = &doL1LongDescEvent;
1648 break;
1649 case L2:
1650 event = &doL2LongDescEvent;
1651 break;
1652 case L3:
1653 event = &doL3LongDescEvent;
1654 break;
1655 default:
1656 panic("Wrong lookup level in table walk\n");
1657 break;
1658 }
1659
1660 bool delayed;
1661 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1662 sizeof(uint64_t), flag, -1, event,
1663 &TableWalker::doLongDescriptor);
1664 if (delayed) {
1665 currState->delayed = true;
1666 }
1667 }
1668 return;
1669 default:
1670 panic("A new type in a 2 bit field?\n");
1671 }
1672}
1673
1674void
1675TableWalker::doL2Descriptor()
1676{
1677 if (currState->fault != NoFault) {
1678 return;
1679 }
1680
1681 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1682 currState->vaddr_tainted, currState->l2Desc.data);
1683 TlbEntry te;
1684
1685 if (currState->l2Desc.invalid()) {
1686 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1687 if (!currState->timing) {
1688 currState->tc = NULL;
1689 currState->req = NULL;
1690 }
1691 if (currState->isFetch)
1692 currState->fault = std::make_shared<PrefetchAbort>(
1693 currState->vaddr_tainted,
1694 ArmFault::TranslationLL + L2,
1695 isStage2,
1696 ArmFault::VmsaTran);
1697 else
1698 currState->fault = std::make_shared<DataAbort>(
1699 currState->vaddr_tainted, currState->l1Desc.domain(),
1700 currState->isWrite, ArmFault::TranslationLL + L2,
1701 isStage2,
1702 ArmFault::VmsaTran);
1703 return;
1704 }
1705
1706 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1707 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1708 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1709 */
1710 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1711 currState->sctlr.afe, currState->l2Desc.ap());
1712
1713 currState->fault = std::make_shared<DataAbort>(
1714 currState->vaddr_tainted,
1715 TlbEntry::DomainType::NoAccess, currState->isWrite,
1716 ArmFault::AccessFlagLL + L2, isStage2,
1717 ArmFault::VmsaTran);
1718 }
1719
1720 insertTableEntry(currState->l2Desc, false);
1721}
1722
1723void
1724TableWalker::doL1DescriptorWrapper()
1725{
1726 currState = stateQueues[L1].front();
1727 currState->delayed = false;
1728 // if there's a stage2 translation object we don't need it any more
1729 if (currState->stage2Tran) {
1730 delete currState->stage2Tran;
1731 currState->stage2Tran = NULL;
1732 }
1733
1734
1735 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1736 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1737
1738 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1739 doL1Descriptor();
1740
1741 stateQueues[L1].pop_front();
1742 // Check if fault was generated
1743 if (currState->fault != NoFault) {
1744 currState->transState->finish(currState->fault, currState->req,
1745 currState->tc, currState->mode);
1746 statWalksShortTerminatedAtLevel[0]++;
1747
1748 pending = false;
1749 nextWalk(currState->tc);
1750
1751 currState->req = NULL;
1752 currState->tc = NULL;
1753 currState->delayed = false;
1754 delete currState;
1755 }
1756 else if (!currState->delayed) {
1757 // delay is not set so there is no L2 to do
1758 // Don't finish the translation if a stage 2 look up is underway
1759 if (!currState->doingStage2) {
1760 statWalkServiceTime.sample(curTick() - currState->startTime);
1761 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1762 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1763 currState->transState, currState->mode);
1764 statWalksShortTerminatedAtLevel[0]++;
1765 }
1766
1767 pending = false;
1768 nextWalk(currState->tc);
1769
1770 currState->req = NULL;
1771 currState->tc = NULL;
1772 currState->delayed = false;
1773 delete currState;
1774 } else {
1775 // need to do L2 descriptor
1776 stateQueues[L2].push_back(currState);
1777 }
1778 currState = NULL;
1779}
1780
1781void
1782TableWalker::doL2DescriptorWrapper()
1783{
1784 currState = stateQueues[L2].front();
1785 assert(currState->delayed);
1786 // if there's a stage2 translation object we don't need it any more
1787 if (currState->stage2Tran) {
1788 delete currState->stage2Tran;
1789 currState->stage2Tran = NULL;
1790 }
1791
1792 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1793 currState->vaddr_tainted);
1794 doL2Descriptor();
1795
1796 // Check if fault was generated
1797 if (currState->fault != NoFault) {
1798 currState->transState->finish(currState->fault, currState->req,
1799 currState->tc, currState->mode);
1800 statWalksShortTerminatedAtLevel[1]++;
1801 }
1802 else {
1803 // Don't finish the translation if a stage 2 look up is underway
1804 if (!currState->doingStage2) {
1805 statWalkServiceTime.sample(curTick() - currState->startTime);
1806 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1807 currState->fault = tlb->translateTiming(currState->req,
1808 currState->tc, currState->transState, currState->mode);
1809 statWalksShortTerminatedAtLevel[1]++;
1810 }
1811 }
1812
1813
1814 stateQueues[L2].pop_front();
1815 pending = false;
1816 nextWalk(currState->tc);
1817
1818 currState->req = NULL;
1819 currState->tc = NULL;
1820 currState->delayed = false;
1821
1822 delete currState;
1823 currState = NULL;
1824}
1825
1826void
1827TableWalker::doL0LongDescriptorWrapper()
1828{
1829 doLongDescriptorWrapper(L0);
1830}
1831
1832void
1833TableWalker::doL1LongDescriptorWrapper()
1834{
1835 doLongDescriptorWrapper(L1);
1836}
1837
1838void
1839TableWalker::doL2LongDescriptorWrapper()
1840{
1841 doLongDescriptorWrapper(L2);
1842}
1843
1844void
1845TableWalker::doL3LongDescriptorWrapper()
1846{
1847 doLongDescriptorWrapper(L3);
1848}
1849
1850void
1851TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1852{
1853 currState = stateQueues[curr_lookup_level].front();
1854 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1855 currState->delayed = false;
1856
1857 // if there's a stage2 translation object we don't need it any more
1858 if (currState->stage2Tran) {
1859 delete currState->stage2Tran;
1860 currState->stage2Tran = NULL;
1861 }
1862
1863 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1864 currState->vaddr_tainted);
1865 doLongDescriptor();
1866
1867 stateQueues[curr_lookup_level].pop_front();
1868
1869 if (currState->fault != NoFault) {
1870 // A fault was generated
1871 currState->transState->finish(currState->fault, currState->req,
1872 currState->tc, currState->mode);
1873
1874 pending = false;
1875 nextWalk(currState->tc);
1876
1877 currState->req = NULL;
1878 currState->tc = NULL;
1879 currState->delayed = false;
1880 delete currState;
1881 } else if (!currState->delayed) {
1882 // No additional lookups required
1883 // Don't finish the translation if a stage 2 look up is underway
1884 if (!currState->doingStage2) {
1885 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1886 statWalkServiceTime.sample(curTick() - currState->startTime);
1887 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1888 currState->transState,
1889 currState->mode);
1890 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1891 }
1892
1893 pending = false;
1894 nextWalk(currState->tc);
1895
1896 currState->req = NULL;
1897 currState->tc = NULL;
1898 currState->delayed = false;
1899 delete currState;
1900 } else {
1901 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1902 panic("Max. number of lookups already reached in table walk\n");
1903 // Need to perform additional lookups
1904 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1905 }
1906 currState = NULL;
1907}
1908
1909
1910void
1911TableWalker::nextWalk(ThreadContext *tc)
1912{
1913 if (pendingQueue.size())
1914 schedule(doProcessEvent, clockEdge(Cycles(1)));
1915 else
1916 completeDrain();
1917}
1918
1919bool
1920TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1921 Request::Flags flags, int queueIndex, Event *event,
1922 void (TableWalker::*doDescriptor)())
1923{
1924 bool isTiming = currState->timing;
1925
1926 // do the requests for the page table descriptors have to go through the
1927 // second stage MMU
1928 if (currState->stage2Req) {
1929 Fault fault;
1930 flags = flags | TLB::MustBeOne;
1931
1932 if (isTiming) {
1933 Stage2MMU::Stage2Translation *tran = new
1934 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1935 currState->vaddr);
1936 currState->stage2Tran = tran;
1937 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1938 flags);
1939 fault = tran->fault;
1940 } else {
1941 fault = stage2Mmu->readDataUntimed(currState->tc,
1942 currState->vaddr, descAddr, data, numBytes, flags,
1943 currState->functional);
1944 }
1945
1946 if (fault != NoFault) {
1947 currState->fault = fault;
1948 }
1949 if (isTiming) {
1950 if (queueIndex >= 0) {
1951 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1952 stateQueues[queueIndex].size());
1953 stateQueues[queueIndex].push_back(currState);
1954 currState = NULL;
1955 }
1956 } else {
1957 (this->*doDescriptor)();
1958 }
1959 } else {
1960 if (isTiming) {
1961 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1962 currState->tc->getCpuPtr()->clockPeriod(),flags);
1963 if (queueIndex >= 0) {
1964 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1965 stateQueues[queueIndex].size());
1966 stateQueues[queueIndex].push_back(currState);
1967 currState = NULL;
1968 }
1969 } else if (!currState->functional) {
1970 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1971 currState->tc->getCpuPtr()->clockPeriod(), flags);
1972 (this->*doDescriptor)();
1973 } else {
1974 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1975 req->taskId(ContextSwitchTaskId::DMA);
1976 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1977 pkt->dataStatic(data);
1978 port->sendFunctional(pkt);
1979 (this->*doDescriptor)();
1980 delete req;
1981 delete pkt;
1982 }
1983 }
1984 return (isTiming);
1985}
1986
1987void
1988TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1989{
1990 TlbEntry te;
1991
1992 // Create and fill a new page table entry
1993 te.valid = true;
1994 te.longDescFormat = longDescriptor;
1995 te.isHyp = currState->isHyp;
1996 te.asid = currState->asid;
1997 te.vmid = currState->vmid;
1998 te.N = descriptor.offsetBits();
1999 te.vpn = currState->vaddr >> te.N;
2000 te.size = (1<<te.N) - 1;
2001 te.pfn = descriptor.pfn();
2002 te.domain = descriptor.domain();
2003 te.lookupLevel = descriptor.lookupLevel;
2004 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
2005 te.nstid = !currState->isSecure;
2006 te.xn = descriptor.xn();
2007 if (currState->aarch64)
2008 te.el = currState->el;
2009 else
2010 te.el = 1;
2011
2012 statPageSizes[pageSizeNtoStatBin(te.N)]++;
2013 statRequestOrigin[COMPLETED][currState->isFetch]++;
2014
2015 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2016 // as global
2017 te.global = descriptor.global(currState) || isStage2;
2018 if (longDescriptor) {
2019 LongDescriptor lDescriptor =
2020 dynamic_cast<LongDescriptor &>(descriptor);
2021
2022 te.xn |= currState->xnTable;
2023 te.pxn = currState->pxnTable || lDescriptor.pxn();
2024 if (isStage2) {
2025 // this is actually the HAP field, but its stored in the same bit
2026 // possitions as the AP field in a stage 1 translation.
2027 te.hap = lDescriptor.ap();
2028 } else {
2029 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2030 (currState->userTable && (descriptor.ap() & 0x1));
2031 }
2032 if (currState->aarch64)
2033 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2034 currState->longDesc.sh());
2035 else
2036 memAttrsLPAE(currState->tc, te, lDescriptor);
2037 } else {
2038 te.ap = descriptor.ap();
2039 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2040 descriptor.shareable());
2041 }
2042
2043 // Debug output
2044 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2045 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2046 te.N, te.pfn, te.size, te.global, te.valid);
2047 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2048 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2049 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2050 te.nonCacheable, te.ns);
2051 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2052 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2053 descriptor.getRawData());
2054
2055 // Insert the entry into the TLB
2056 tlb->insert(currState->vaddr, te);
2057 if (!currState->timing) {
2058 currState->tc = NULL;
2059 currState->req = NULL;
2060 }
2061}
2062
2063ArmISA::TableWalker *
2064ArmTableWalkerParams::create()
2065{
2066 return new ArmISA::TableWalker(this);
2067}
2068
2069LookupLevel
2070TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2071{
2072 switch (lookup_level_as_int) {
2073 case L1:
2074 return L1;
2075 case L2:
2076 return L2;
2077 case L3:
2078 return L3;
2079 default:
2080 panic("Invalid lookup level conversion");
2081 }
2082}
2083
2084/* this method keeps track of the table walker queue's residency, so
2085 * needs to be called whenever requests start and complete. */
2086void
2087TableWalker::pendingChange()
2088{
2089 unsigned n = pendingQueue.size();
2090 if ((currState != NULL) && (currState != pendingQueue.front())) {
2091 ++n;
2092 }
2093
2094 if (n != pendingReqs) {
2095 Tick now = curTick();
2096 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2097 pendingReqs = n;
2098 pendingChangeTick = now;
2099 }
2100}
2101
2102uint8_t
2103TableWalker::pageSizeNtoStatBin(uint8_t N)
2104{
2105 /* for statPageSizes */
2106 switch(N) {
2107 case 12: return 0; // 4K
2108 case 14: return 1; // 16K (using 16K granule in v8-64)
2109 case 16: return 2; // 64K
2110 case 20: return 3; // 1M
2111 case 21: return 4; // 2M-LPAE
2112 case 24: return 5; // 16M
2113 case 25: return 6; // 32M (using 16K granule in v8-64)
2114 case 29: return 7; // 512M (using 64K granule in v8-64)
2115 case 30: return 8; // 1G-LPAE
2116 default:
2117 panic("unknown page size");
2118 return 255;
2119 }
2120}
2121
2122void
2123TableWalker::regStats()
2124{
2125 statWalks
2126 .name(name() + ".walks")
2127 .desc("Table walker walks requested")
2128 ;
2129
2130 statWalksShortDescriptor
2131 .name(name() + ".walksShort")
2132 .desc("Table walker walks initiated with short descriptors")
2133 .flags(Stats::nozero)
2134 ;
2135
2136 statWalksLongDescriptor
2137 .name(name() + ".walksLong")
2138 .desc("Table walker walks initiated with long descriptors")
2139 .flags(Stats::nozero)
2140 ;
2141
2142 statWalksShortTerminatedAtLevel
2143 .init(2)
2144 .name(name() + ".walksShortTerminationLevel")
2145 .desc("Level at which table walker walks "
2146 "with short descriptors terminate")
2147 .flags(Stats::nozero)
2148 ;
2149 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2150 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2151
2152 statWalksLongTerminatedAtLevel
2153 .init(4)
2154 .name(name() + ".walksLongTerminationLevel")
2155 .desc("Level at which table walker walks "
2156 "with long descriptors terminate")
2157 .flags(Stats::nozero)
2158 ;
2159 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2160 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2161 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2162 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2163
2164 statSquashedBefore
2165 .name(name() + ".walksSquashedBefore")
2166 .desc("Table walks squashed before starting")
2167 .flags(Stats::nozero)
2168 ;
2169
2170 statSquashedAfter
2171 .name(name() + ".walksSquashedAfter")
2172 .desc("Table walks squashed after completion")
2173 .flags(Stats::nozero)
2174 ;
2175
2176 statWalkWaitTime
2177 .init(16)
2178 .name(name() + ".walkWaitTime")
2179 .desc("Table walker wait (enqueue to first request) latency")
2180 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2181 ;
2182
2183 statWalkServiceTime
2184 .init(16)
2185 .name(name() + ".walkCompletionTime")
2186 .desc("Table walker service (enqueue to completion) latency")
2187 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2188 ;
2189
2190 statPendingWalks
2191 .init(16)
2192 .name(name() + ".walksPending")
2193 .desc("Table walker pending requests distribution")
2194 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2195 ;
2196
2197 statPageSizes // see DDI 0487A D4-1661
2198 .init(9)
2199 .name(name() + ".walkPageSizes")
2200 .desc("Table walker page sizes translated")
2201 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2202 ;
2203 statPageSizes.subname(0, "4K");
2204 statPageSizes.subname(1, "16K");
2205 statPageSizes.subname(2, "64K");
2206 statPageSizes.subname(3, "1M");
2207 statPageSizes.subname(4, "2M");
2208 statPageSizes.subname(5, "16M");
2209 statPageSizes.subname(6, "32M");
2210 statPageSizes.subname(7, "512M");
2211 statPageSizes.subname(8, "1G");
2212
2213 statRequestOrigin
2214 .init(2,2) // Instruction/Data, requests/completed
2215 .name(name() + ".walkRequestOrigin")
2216 .desc("Table walker requests started/completed, data/inst")
2217 .flags(Stats::total)
2218 ;
2219 statRequestOrigin.subname(0,"Requested");
2220 statRequestOrigin.subname(1,"Completed");
2221 statRequestOrigin.ysubname(0,"Data");
2222 statRequestOrigin.ysubname(1,"Inst");
2223}
|